From 71f47d1354976ef26da43ed1dddfd53becc35e23 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:42:50 +0800 Subject: [PATCH 01/44] feat(*): a Raven daemon on a registered machine, reached over one watched ssh link A registered machine gets this Raven's own code as a daemon (raven/node): started by the relay the host's ssh runs there (`python -I -m raven.node --attach --dir `) when none answers on its socket under the node directory, and left on its own once idle. It answers file calls with Raven's own tools fenced to the directories each call names (fs/read, list, find, grep, stat, write, edit, pull), starts acp agents under task ids of the host's in their own process groups in the machine user's login environment, keeps each agent's frames in the task's record on disk so a connection that goes leaves the agent running and the next one takes it over from the frame the host last read (agent/attach), lists the tasks it runs or remembers, and probes the machine and an agent's command. It opens no network port; only --stdio ties the agents to the connection. `raven ops connection install-node ` puts it on a machine after one look and one question, and `--prune` removes older installs. The host keeps one link per machine (raven/node/client.py): the ssh line runs the relay, a heartbeat watches the link, a lost link parks the agents on it and is tried again on a backoff for as long as any waits, and a link that comes back attaches every task again by its id, sending what the host wrote meanwhile and any cancel asked in between. The registry's package tells a remote path from a local one and lends the tools the pool through one seam (raven/ops/remote_path.py); the machine lane stays debug-only (RAVEN_DEBUG_MACHINES=1). First of three changes on this design; the next runs acp sub-agents on a machine through this daemon, the one after reads and writes files there by address. Nothing dispatches to a machine yet. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 35 +- docs-site/docs/repo-layout.md | 1 + pyproject.toml | 3 + raven/acp_client/client.py | 243 +++- raven/acp_client/protocol.py | 5 +- raven/acp_client/remote.py | 323 ++++-- raven/cli/gateway_commands.py | 7 +- raven/cli/ops_connection_commands.py | 81 ++ raven/cli/tui_commands.py | 6 +- raven/core/runtime.py | 7 + raven/home.py | 17 + raven/node/__init__.py | 7 + raven/node/__main__.py | 52 + raven/node/bundle.py | 102 ++ raven/node/client.py | 809 +++++++++++++ raven/node/install.py | 298 +++++ raven/node/protocol.py | 129 +++ raven/node/relay.py | 125 ++ raven/node/server.py | 1277 +++++++++++++++++++++ raven/ops/connection_add.py | 40 +- raven/ops/remote_path.py | 474 ++++++++ raven/ops/transport.py | 5 +- raven/rpc/bootstrap.py | 6 +- tests/test_acp_client_teardown.py | 64 ++ tests/test_acp_remote.py | 544 ++++++--- tests/test_cli_ops_connection_commands.py | 259 +++++ tests/test_node_bundle.py | 107 ++ tests/test_node_client.py | 1005 ++++++++++++++++ tests/test_node_install.py | 611 ++++++++++ tests/test_node_relay.py | 94 ++ tests/test_node_server.py | 1104 ++++++++++++++++++ tests/test_ops_connection_add.py | 41 + tests/test_ops_remote_path.py | 318 +++++ tests/test_ops_transport.py | 3 + 34 files changed, 7853 insertions(+), 349 deletions(-) create mode 100644 raven/node/__init__.py create mode 100644 raven/node/__main__.py create mode 100644 raven/node/bundle.py create mode 100644 raven/node/client.py create mode 100644 raven/node/install.py create mode 100644 raven/node/protocol.py create mode 100644 raven/node/relay.py create mode 100644 raven/node/server.py create mode 100644 raven/ops/remote_path.py create mode 100644 tests/test_acp_client_teardown.py create mode 100644 tests/test_node_bundle.py create mode 100644 tests/test_node_client.py create mode 100644 tests/test_node_install.py create mode 100644 tests/test_node_relay.py create mode 100644 tests/test_node_server.py create mode 100644 tests/test_ops_remote_path.py diff --git a/CONTEXT.md b/CONTEXT.md index 78a3c3c3e..7b6a32ecc 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -390,6 +390,35 @@ GPU hosts, and the term covers any registered compute destination); here the rows name compute destinations with that binding as an implementation detail). +**Raven node** (`raven/node/`): +The Raven process on a registered **Machine** that owns the agents the host runs there: +a daemon (`python -I -m raven.node --serve --dir `) that the host's relay +starts over ssh when none is up (`--attach`: the relay bridges the ssh session to the +daemon's socket under `node_dir/run/` and leaves with the ssh; the daemon leaves on its +own once idle). It does three things there and nothing else: answers file calls with +Raven's own file tools (`read_file`, `list_dir`, `find`, `grep`, `write_file`, `edit_file`) +fenced to the directories each call names, so a read of 50 lines of a large file moves +those 50 lines, and fetches one file whole (`fs/pull`); starts the `acp` agents whose +entries name the machine, each under a task id of the host's, in its own process group in +the machine user's login environment, forwards their ACP frames on numbered channels of +the connection that holds them and keeps those frames in the task's record +(`node_dir/tasks//`), so a connection that goes leaves the agent running and the next +one takes it over from the frame the host last read (`agent/attach`); and answers probes of +the machine and of an agent's command. It holds no conversation, calls no model and opens +no network port. It runs exactly the host's code, checked by protocol and digest at every +connection (`raven/node/bundle.py`); `raven ops connection install-node` puts it on a +machine, under the row's `node_dir` (`~/.raven-node` when it names none), in a directory +named by the code's version and digest -- its install name, `bundle.install_name()`. The +host keeps at most one live link per machine and watches it with a heartbeat +(`raven/node/client.py`): a link silent for 15 s is lost, the agents on it wait, and the +link is tried again on a backoff for as long as any waits; a link that comes back attaches +each agent again by its task id. Only `--stdio` (tests, a fallback) ties the agents to the +connection. +_Avoid_: "node" alone (a DAG node is a `run_subagent_dag` step, and the Machine +entry already rules the bare word out for a computer); "node id" for the install +name (a **Node id** is a delegated task's address); "agent" or "server" (it +answers only the Raven that reaches it, only over ssh). + **Roster** (`format_agent_listing`): The agent table rendered as the text spliced into `spawn`'s and `run_subagent_dag`'s tool descriptions — `name [stateful, local-files, live-progress] (description)`. The model's only @@ -1633,9 +1662,9 @@ turns through its schedulers and sentinel but is an engine the loop and the asse consume, not a transport), and `core` (the L2 assembly root). `templates` is packaged data and takes no seat. Surfaces: `cli`, `rpc`, `acp` (an entrance: Raven serving as an agent for another host) and `a2a` (the same entrance for a peer agent, over -Agent2Agent rather than ACP). `browser` and `importer` are seated inner (feature -libraries consumed by surfaces, importing none themselves -- the edge is watched -by the contract now, not by a ruling note). `evolver` is not a seat at all: it left the +Agent2Agent rather than ACP). `browser`, `importer` and `node` (the **Raven node**) +are seated inner (feature libraries consumed by surfaces, importing none themselves -- +the edge is watched by the contract now, not by a ruling note). `evolver` is not a seat at all: it left the package for the repo-level `evolver/` tool (outside the wheel) that drives raven as a library, and a fifth import-linter contract keeps the runtime from importing it back. `agents/` is the same kind of non-seat: repo-level agent definitions (the A/B pilots whose A side is the diff --git a/docs-site/docs/repo-layout.md b/docs-site/docs/repo-layout.md index 18cc82947..73e4301da 100644 --- a/docs-site/docs/repo-layout.md +++ b/docs-site/docs/repo-layout.md @@ -52,6 +52,7 @@ repository's commit conventions. | `market` | PlugHub catalog, trust checks, installation, and contribution ledgers | | `mcp` | MCP server connections and tool integration | | `memory_engine` | Memory recall and consolidation, local skills, and SkillForge retrieval | +| `node` | The Raven node: Raven's own file tools and the agents it dispatches, run on a registered machine over one ssh link | | `observability` | Span semantics, attribute extraction, and usage attribution | | `ops` | Local and remote machine registry and execution transports | | `permissions` | Tool-call decisions: allow, request approval, or deny | diff --git a/pyproject.toml b/pyproject.toml index 51ac7951f..a795d1c29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -623,6 +623,9 @@ source_modules = [ # `raven.playbook`, by the CLI and RPC surfaces that read a stint's record, # and by the dispatch path that carries its ref; it imports no surface itself. "raven.stint", + # The Raven node: this Raven's own file tools run on a registered machine; + # consumed by the CLI, it imports no surface itself. + "raven.node", ] forbidden_modules = ["raven.cli", "raven.rpc", "raven.acp", "raven.a2a"] # Debt allowlist: none. The last allowlisted edge (market reaching the skill diff --git a/raven/acp_client/client.py b/raven/acp_client/client.py index 4744261f8..5b388a544 100644 --- a/raven/acp_client/client.py +++ b/raven/acp_client/client.py @@ -9,6 +9,13 @@ :class:`raven.agent.subagent.backends.cli_agent.CliAgentBackend`: own session, pgid captured at spawn, ``killpg`` on teardown. An ACP server that reparents a worker would otherwise survive ``proc.kill()`` exactly as a ``codex``-style CLI does. + +What a connection runs over is an :class:`Endpoint`: the child's pipes and +process group for an agent on this computer (:class:`_ProcessEndpoint`), or one +channel of a machine's Raven node for an agent there +(:class:`raven.node.client.ChannelEndpoint`). Everything above the endpoint -- +request correlation, the read loops, cancellation, the journal -- is the same +code either way. """ from __future__ import annotations @@ -20,7 +27,7 @@ from collections import deque from collections.abc import Awaitable, Callable from functools import partial -from typing import Any +from typing import Any, Protocol from loguru import logger @@ -69,6 +76,127 @@ moments later anyway. """ +#: Whether this platform has process groups ``killpg`` can reach. Windows has +#: neither them nor SIGKILL, and a teardown that reached for either there +#: raised before any of the cleanup below it ran, leaving the child alive. +_POSIX = hasattr(os, "killpg") + + +def _end(pgid: int, proc: Any) -> None: + """End the server and what it started: the process group on POSIX, the process itself elsewhere. + + On Windows the child is terminated by handle, which ends what it started + only insofar as the child ends it; there is no group to sweep. A process + already gone is not an error on either path. + """ + if _POSIX: + try: + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + return + try: + proc.kill() + except ProcessLookupError: + pass + + +class Endpoint(Protocol): + """What a connection runs over: where its frames go and come from, and how it ends. + + ``returncode`` is ``None`` while the far side runs. ``ended`` is the + endpoint's own sentence for how it ended, when it has a better one than + an exit code (a link to a machine that went); ``None`` otherwise. + """ + + pid: int | None + + @property + def returncode(self) -> int | None: ... + + def describe(self) -> str: ... + + async def write(self, data: bytes) -> None: ... + + async def readline(self) -> bytes: ... + + async def readerr(self) -> bytes: ... + + async def end(self) -> None: ... + + async def wait(self) -> int | None: ... + + def ended(self) -> str | None: ... + + def release(self) -> None: ... + + +class _ProcessEndpoint: + """A child process on this computer: its pipes, and the process group it leads.""" + + def __init__(self, proc: asyncio.subprocess.Process, pgid: int) -> None: + self.proc = proc + # Captured at spawn by the caller, not re-derived here: with + # start_new_session the launcher's pid doubles as the group's pgid, and + # once the launcher exits its pid can be recycled onto an unrelated + # process. + self._pgid = pgid + + @property + def pid(self) -> int | None: + return self.proc.pid + + @property + def returncode(self) -> int | None: + return self.proc.returncode + + def describe(self) -> str: + return f"pid {self.proc.pid}" + + async def write(self, data: bytes) -> None: + stdin = self.proc.stdin + if stdin is None or stdin.is_closing(): + raise BrokenPipeError("stdin is closed") + stdin.write(data) + await stdin.drain() + + async def readline(self) -> bytes: + if self.proc.stdout is None: # pragma: no cover - PIPE is always requested + return b"" + return await self.proc.stdout.readline() + + async def readerr(self) -> bytes: + if self.proc.stderr is None: # pragma: no cover - PIPE is always requested + return b"" + return await self.proc.stderr.readline() + + async def end(self) -> None: + _end(self._pgid, self.proc) + + async def wait(self) -> int | None: + return await self.proc.wait() + + def ended(self) -> str | None: + return None + + def release(self) -> None: + """Close the subprocess transport while there is still a loop to close it on. + + ``asyncio.subprocess.Process`` holds its transport until the child is + reaped, and an unreaped one is left to ``__del__`` -- which runs at + interpreter shutdown, calls ``call_soon`` on a closed loop, and prints a + ``RuntimeError`` traceback the user can do nothing about. Closing it + here is the only point at which that is still a no-op. + """ + transport = getattr(self.proc, "_transport", None) + if transport is None: + return + try: + transport.close() + except Exception as exc: # noqa: BLE001 - teardown must not raise + logger.debug("acp endpoint pid {}: releasing the transport failed: {}", self.proc.pid, exc) + + _CANCEL_SETTLE_S = 5.0 """How long a cancelled turn is given to settle with ``stopReason: cancelled``. @@ -152,19 +280,19 @@ def __init__( self, *, name: str, - proc: asyncio.subprocess.Process, - pgid: int, + endpoint: Endpoint | None = None, + proc: asyncio.subprocess.Process | None = None, + pgid: int | None = None, on_request: RequestHandler | None = None, on_notification: NotificationHandler | None = None, journal: FrameJournal | None = None, ) -> None: self.name = name - self._proc = proc - # Captured at spawn by the caller, not re-derived here: with - # start_new_session the launcher's pid doubles as the group's pgid, and - # once the launcher exits its pid can be recycled onto an unrelated - # process. - self._pgid = pgid + if endpoint is None: + if proc is None: + raise ValueError(f"acp agent {name!r}: a connection needs an endpoint or a process") + endpoint = _ProcessEndpoint(proc, proc.pid if pgid is None else pgid) + self._endpoint: Endpoint = endpoint self._on_request = on_request self._on_notification = on_notification # Every frame both ways, on disk, for the life of this connection. Held @@ -220,7 +348,7 @@ async def launch( on_notification: NotificationHandler | None = None, journal: FrameJournal | None = None, ) -> "AcpClient": - """Start the agent's ACP server and begin reading it. + """Start the agent's ACP server on this computer and begin reading it. Raises :class:`AcpConnectionError` if the process cannot be started; a server that starts but never speaks is a per-request timeout, not a @@ -266,8 +394,7 @@ async def launch( client = cls( name=name, - proc=proc, - pgid=proc.pid, + endpoint=_ProcessEndpoint(proc, proc.pid), on_request=on_request, on_notification=on_notification, journal=journal, @@ -276,6 +403,34 @@ async def launch( logger.info("acp agent {!r}: started {} (pid {})", name, argv[:1], proc.pid) return client + @classmethod + def over( + cls, + endpoint: Endpoint, + *, + name: str, + on_request: RequestHandler | None = None, + on_notification: NotificationHandler | None = None, + journal: FrameJournal | None = None, + ) -> "AcpClient": + """A connection over an endpoint somebody else opened: an agent a machine's Raven node started. + + The agent is already running where the endpoint leads; this only + begins reading it. Closing the client ends the agent through the + endpoint, as it kills a child process it launched itself. + """ + client = cls( + name=name, endpoint=endpoint, on_request=on_request, on_notification=on_notification, journal=journal + ) + client._start_loops() + logger.info("acp agent {!r}: started on {}", name, endpoint.describe()) + return client + + @property + def _proc(self) -> Any: + """The child process of an agent on this computer; ``None`` for one reached through a node.""" + return getattr(self._endpoint, "proc", None) + def _start_loops(self) -> None: self._reader_task = asyncio.create_task(self._read_stdout(), name=f"acp-read-{self.name}") self._stderr_task = asyncio.create_task(self._read_stderr(), name=f"acp-err-{self.name}") @@ -294,7 +449,7 @@ def alive(self) -> bool: Without this the pool would hand that connection out forever, failing every task in milliseconds while the process table says all is well. """ - if self._closed or self._proc.returncode is not None: + if self._closed or self._endpoint.returncode is not None: return False return self._reader_task is None or not self._reader_task.done() @@ -311,10 +466,7 @@ async def close(self) -> None: if self._closed: return self._closed = True - try: - os.killpg(self._pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError): - pass + await self._endpoint.end() for task in (self._reader_task, self._stderr_task): if task is not None: task.cancel() @@ -331,38 +483,21 @@ async def close(self) -> None: for task in done: if not task.cancelled(): task.exception() - if self._proc.returncode is None: + if self._endpoint.returncode is None: try: - await asyncio.wait_for(self._proc.wait(), timeout=_TEARDOWN_REAP_S) + await asyncio.wait_for(self._endpoint.wait(), timeout=_TEARDOWN_REAP_S) except (asyncio.TimeoutError, ProcessLookupError): logger.debug( - "acp agent {!r}: not reaped within {}s of SIGKILL; abandoning the wait", + "acp agent {!r}: not reaped within {}s of being ended; abandoning the wait", self.name, _TEARDOWN_REAP_S, ) - self._release_transport() + self._endpoint.release() self._fail_pending(AcpConnectionError(f"acp agent {self.name!r}: connection closed")) # Last, so the frames the teardown itself produced are in the file. if self._journal is not None: self._journal.close() - def _release_transport(self) -> None: - """Close the subprocess transport while there is still a loop to close it on. - - ``asyncio.subprocess.Process`` holds its transport until the child is - reaped, and an unreaped one is left to ``__del__`` -- which runs at - interpreter shutdown, calls ``call_soon`` on a closed loop, and prints a - ``RuntimeError`` traceback the user can do nothing about. Closing it - here is the only point at which that is still a no-op. - """ - transport = getattr(self._proc, "_transport", None) - if transport is None: - return - try: - transport.close() - except Exception as exc: # noqa: BLE001 - teardown must not raise - logger.debug("acp agent {!r}: releasing the transport failed: {}", self.name, exc) - async def __aenter__(self) -> "AcpClient": return self @@ -592,24 +727,17 @@ async def _send(self, frame: dict[str, Any], *, session: str | None = None) -> N """ if self._journal is not None: self._journal.note("out", frame=redact_acp_frame(frame), session=session or self._frame_session(frame)) - stdin = self._proc.stdin - if stdin is None or stdin.is_closing(): - raise AcpConnectionError(f"acp agent {self.name!r}: stdin is closed") try: - stdin.write(protocol.encode(frame)) - await stdin.drain() - except (BrokenPipeError, ConnectionResetError, RuntimeError) as exc: + await self._endpoint.write(protocol.encode(frame)) + except (OSError, RuntimeError) as exc: raise AcpConnectionError(f"acp agent {self.name!r}: write failed: {exc}") from exc # ---- read loops ------------------------------------------------------ async def _read_stdout(self) -> None: - stdout = self._proc.stdout - if stdout is None: # pragma: no cover - PIPE is always requested - return try: while True: - raw = await stdout.readline() + raw = await self._endpoint.readline() if not raw: break line = raw.decode("utf-8", "replace").strip() @@ -664,18 +792,19 @@ async def _settle_exit(self) -> None: stderr open forever must not park every caller behind it. """ try: - await asyncio.wait_for(asyncio.shield(self._proc.wait()), timeout=1.5) + await asyncio.wait_for(asyncio.shield(self._endpoint.wait()), timeout=1.5) except (asyncio.TimeoutError, ProcessLookupError): pass if self._stderr_task is not None and not self._stderr_task.done(): await asyncio.wait({self._stderr_task}, timeout=0.5) def _ended_error(self) -> AcpConnectionError: - return AcpConnectionError( - f"acp agent {self.name!r}: connection ended (exit {self._proc.returncode}); " - f"stderr tail: {self.stderr_tail(400) or ''}", - stderr=self.stderr_tail() or None, - ) + """How the connection ended: the endpoint's own sentence when it has one, else the exit and stderr.""" + rc = self._endpoint.returncode + said = self._endpoint.ended() + if said is None: + said = f"connection ended (exit {rc}); stderr tail: {self.stderr_tail(400) or ''}" + return AcpConnectionError(f"acp agent {self.name!r}: {said}", stderr=self.stderr_tail() or None, returncode=rc) async def _read_stderr(self) -> None: """Drain the child's stderr for as long as the child is alive. @@ -690,13 +819,10 @@ async def _read_stderr(self) -> None: limit, this loop exited, and the agent loop died holding a lock it never saw. So the only exits are EOF and cancellation. """ - stderr = self._proc.stderr - if stderr is None: # pragma: no cover - PIPE is always requested - return try: while True: try: - raw = await stderr.readline() + raw = await self._endpoint.readerr() except ValueError as exc: # A line past the reader's limit. It has already cleared its # buffer and resumed the transport, so the oversized line is @@ -865,6 +991,7 @@ def _fail_pending(self, exc: Exception) -> None: "is_draining", "UNHANDLED", "AcpClient", + "Endpoint", "NotificationHandler", "RequestHandler", ] diff --git a/raven/acp_client/protocol.py b/raven/acp_client/protocol.py index bc3e44bba..aa933f264 100644 --- a/raven/acp_client/protocol.py +++ b/raven/acp_client/protocol.py @@ -111,9 +111,12 @@ class AcpConnectionError(AcpError): failed with first and then prints a stack trace and a log path under it. """ - def __init__(self, message: str, *, stderr: str | None = None) -> None: + def __init__(self, message: str, *, stderr: str | None = None, returncode: int | None = None) -> None: super().__init__(message) self.stderr = stderr + # The process's exit status when it had ended, which for an agent on a + # registered machine says whether ssh (255) or the far side ended it. + self.returncode = returncode class AcpTimeoutError(AcpError): diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index 7b0b2e171..8c1ac3a6f 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -1,22 +1,26 @@ -"""An ACP sub-agent that lives on a registered machine, reached over ssh. - -The pool and :class:`~raven.acp_client.client.AcpClient` only ever start a local -process and talk to its stdio. For an agent on another machine that local -process is ``ssh``, and the far end of its stdio is the agent, so nothing above -the launch line has to change. What does change is everything that assumed the -agent shares this disk: its working directory, the MCP servers lent through a -local socket, the files a turn is credited with, and the paths handed to it. - -This module owns the three things only a remote agent needs: - -- the launch line, built from the machine registry at launch and never stored, - so no address, port or key path reaches a config file, a log line or a model; -- the session directory on the machine, made and resolved to an absolute path - before ``session/new``, because an agent refuses a cwd that does not exist - where it runs (measured 2026-10-10 on claude-agent-acp: ``-32602``); +"""An ACP sub-agent that lives on a registered machine, reached through that machine's Raven node. + +The host keeps one Raven node per machine, started over ssh +(:mod:`raven.node.client`), and the node starts the agent there: in the +machine user's login environment, in a process group of its own, with its ACP +frames forwarded on a channel of the node's connection. Above the connection +nothing knows the agent is elsewhere. What does change is everything that +assumed the agent shares this disk: its working directory, the MCP servers +lent through a local socket, the files a turn is credited with, and the paths +handed to it. + +This module owns what only a machine needs and the node cannot own: + +- the registry lookup, refused up front for a row that cannot be used, so a + dispatch fails before anything starts rather than as a network fault + minutes in; +- the ssh line that carries a node, built from the registry at launch and + never stored, so no address, port or key path reaches a config file, a log + line or a model; - one sentence for an ssh failure, naming the machine and never its address; ssh's own messages, which do name it, go to an owner-only log rather than to - the stderr the connection journals and quotes. + the stderr the connection journals and quotes; +- the one-shot commands the installer runs there (:func:`run`, :func:`push`). """ from __future__ import annotations @@ -25,32 +29,41 @@ import os import re import shlex -from collections.abc import Mapping +import subprocess +from collections.abc import Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any -from raven.ops import connections +from raven.home import MACHINE_LANE_ENV, machine_lane_enabled +from raven.ops import connections, remote_path from raven.ops.transport import TIMED_OUT_RC, TransportError, runner_from, ssh_argv, ssh_target -#: Where an agent works on its machine when its entry names nowhere else. -#: Relative to nothing on this computer: ``~`` is the registered user's home -#: there, expanded by the machine's own shell. -DEFAULT_ROOT = "~/raven-work" +#: Where an agent works on its machine when its entry names nowhere else +#: (:data:`raven.ops.remote_path.DEFAULT_ROOT`, which the file tools read from +#: without asking). +DEFAULT_ROOT = remote_path.DEFAULT_ROOT #: Options only a session that stays open needs. ``-T`` because a pseudo-tty #: rewrites the JSON stream (CRLF line ends, echoed input); keepalives because #: ``ConnectTimeout`` is over once the connection is up, and without them a #: machine that drops off the network is noticed only by TCP, minutes later. +#: The node's own heartbeat is what notices first; these are the floor under it. KEEPALIVE = ("-T", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=3") +#: The session directory every check of an agent reuses (Test, Connect, the +#: capability record), so pressing Test leaves one directory on the machine +#: rather than one per press. +CHECK_HANDLE = "raven-check" + #: What ssh's own exit code is when ssh, rather than the remote command, failed. SSH_FAILED_RC = 255 #: What a POSIX shell exits with when it cannot find the command it was given. NOT_FOUND_RC = 127 -_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +# The fields of a registry row that are the way onto the machine, never shown. +_WAY_IN = frozenset({"host", "port", "user", "key"}) _LEAF_UNSAFE = re.compile(r"[^A-Za-z0-9._-]") _LEAF_MAX = 64 @@ -79,6 +92,18 @@ def label(self) -> str: return f"{self.display_name!r} ({self.id})" return repr(self.id) + def field(self, name: str) -> Any: + """One field of the machine's row, other than the way onto it. + + For what a caller needs to know about the machine itself -- where a + Raven node is kept on it (``node_dir``), the directories its owner + listed (``paths``). The address, port, account and key stay behind + :func:`ssh_line`, :func:`run` and :func:`push`. + """ + if name in _WAY_IN: + raise KeyError(f"{name!r} is how raven reaches the machine, not a fact about it") + return self._row.get(name) + def machine(conn_id: str) -> Machine: """The registered machine ``conn_id`` names, or :class:`RemoteMachineError`. @@ -90,6 +115,13 @@ def machine(conn_id: str) -> Machine: launched on half a row fails as a network fault minutes into a dispatch. """ wanted = str(conn_id or "").strip() + if not machine_lane_enabled(): + # The one seam every way onto a machine passes through, so "off" is + # off for a dispatch, a verify, a node and a file alike. + raise RemoteMachineError( + f"machine {wanted!r}: the machine lane is debug-only and off; start raven with " + f"{MACHINE_LANE_ENV}=1 to use agents and files on registered machines" + ) found = connections.read() if found.state == connections.UNREADABLE: raise RemoteMachineError( @@ -110,45 +142,37 @@ def machine(conn_id: str) -> Machine: return Machine(id=wanted, display_name=str(row.get("display_name") or wanted), _row=dict(row)) -def remote_command(agent_command: str, *, root: str, env: Mapping[str, str]) -> str: - """The one argv element sshd hands to the machine user's shell. +def posix(command: str) -> str: + """``command`` as sshd should hand it to the machine user's shell, whatever shell that is. - Two shells read it, each once: sshd's ``$SHELL -c`` reads this string, and - the login shell it execs reads ``inner``. The login shell is what finds the - agent: measured 2026-10-10, claude, codex and qwen installed through nvm or - into ``~/.local/bin`` were on the PATH of ``bash -lic`` and of nothing - less. ``-i`` because that is where nvm's lines live (``.bashrc`` returns - early for a non-interactive shell); it costs two "no job control" lines on - stderr and nothing on stdout, which carries the protocol. + sshd runs a remote command with the user's login shell, and a tcsh, csh or + fish one cannot read a POSIX line -- ``${SHELL:-/bin/sh}`` alone stops csh + with "Bad : modifier". ``exec /bin/sh -c '...'`` reads + the same in all of them, with two adjustments for csh: a ``!`` is written + ``'\\!'``, because csh substitutes history inside single quotes even for + ``-c`` (measured 2026-10-10: ``a!b`` gave "Event not found"), and a line + break is refused, because csh cannot quote one. - ``env`` travels as ``env K=V`` on the command line because ssh forwards no - environment without ``AcceptEnv`` on every machine; the values are - therefore visible in the process list at both ends, and the caller passes - nothing secret. + The command's output starts on a line of its own. The user's shell may + print while it starts -- bash reads ``.bashrc`` for a command sshd hands + it, csh reads ``.cshrc`` for every ``-c`` -- and a greeting with no line + break would otherwise glue itself to the first line the caller reads. """ - bad = sorted(k for k in env if not _ENV_NAME.fullmatch(k)) - if bad: - raise ValueError(f"not an environment variable name: {bad}") - assigns = " ".join(shlex.quote(f"{k}={v}") for k, v in env.items()) - inner = f"exec env {assigns} {agent_command}" if assigns else f"exec {agent_command}" - where = _shell_path(root) - return f'mkdir -p {where} && cd {where} && exec "${{SHELL:-/bin/sh}}" -lic {shlex.quote(inner)}' - - -def launch_command( - target: Machine, - agent_command: str, - *, - root: str = DEFAULT_ROOT, - env: Mapping[str, str] | None = None, - connect_timeout: int = 15, - ssh_log: str | os.PathLike[str] | None = None, -) -> str: - """The launch line for ``agent_command`` on ``target``, as one shell-quoted string. - - A string because that is what the pool and the client take (they - ``shlex.split`` it); ``shlex.join`` makes the split give back exactly this - argv. Built at launch and held in memory only: it carries the address. + if "\n" in command or "\r" in command: + raise ValueError("a command for a registered machine must be one line") + return "exec /bin/sh -c " + shlex.quote(f"echo; {command}").replace("!", "'\\!'") + + +def ssh_line(target: Machine, command: str, *, ssh_log: str | os.PathLike[str], connect_timeout: int = 15) -> str: + """The ssh that keeps ``command`` running on ``target`` for as long as it is read: one shell-quoted line. + + What a Raven node is started with. A string because that is what the + connection takes (it ``shlex.split``\\ s it); ``shlex.join`` makes the + split give back exactly this argv. Built at launch and held in memory + only: it carries the address. ``command`` is sent through :func:`posix`, + so the machine user's login shell may be any shell; it carries no + variable and reads no login profile -- the node captures the login + environment itself, for the agents it starts. ``ssh_log`` (see :func:`ssh_log_path`) is where ssh writes its own messages instead of stderr, so stderr carries only what the far side @@ -156,36 +180,88 @@ def launch_command( ssh's own lines name the address. """ host, port, key, user = _target(target) - extra = KEEPALIVE + (("-E", os.fspath(ssh_log)) if ssh_log is not None else ()) + extra = KEEPALIVE + ("-E", os.fspath(ssh_log)) argv = ssh_argv(host, port, key, user=user, connect_timeout=connect_timeout, extra=extra) # ``--`` so nothing in the remote command is read as an option to ssh, # which also parses options that follow the destination. - argv += ["--", remote_command(agent_command, root=root, env=env or {})] + argv += ["--", posix(command)] return shlex.join(argv) -def session_dir(target: Machine, *, root: str = DEFAULT_ROOT, handle: str, timeout: float = 30.0) -> str: - """Make ``/`` on ``target`` and return its absolute path there. +def failure(target: Machine, log: str | os.PathLike[str], exc: BaseException) -> str | None: + """The sentence for a connection that ended, when ssh or the far shell ended it. - One ssh round trip (about 0.3 s measured). Absolute because ``session/new`` - requires it, and resolved on the machine because ``~`` and symlinks mean - something only there. + The launch ``exec``s the node, so once ssh got through the status ssh + reports is the node's own, and ssh's exits are told apart by what ssh + said: 255 with ssh's words in its log is ssh's; 255 with none is the far + side exiting 255 itself, and its stderr is the better account. 127 is the + far shell not finding the command, which it says on stderr; a process + exiting 127 on its own is left to its own words. 124 is the one-shot + runner's cap, which a launch does not have. """ - where = f"{_shell_path(root)}/{shlex.quote(leaf(handle))}" - try: - run = runner_from(target._row, cap_seconds=timeout) - except TransportError as exc: - raise RemoteMachineError(f"machine {target.label}: {exc}") from None - rc, out = run(f"mkdir -p {where} && cd {where} && pwd -P") - lines = [line.strip() for line in out.splitlines() if line.strip()] - if rc == 0 and lines and lines[-1].startswith("/"): - return lines[-1] - said = explain(target, rc, out) - if said is None: - said = f"could not make a working directory under {root!r} on machine {target.label}" - if reason := _far_side_words(target, lines): - said += f": {reason}" - raise RemoteMachineError(said) + rc = getattr(exc, "returncode", None) + if rc is None: + return None + said = read_ssh_log(log) + if rc == SSH_FAILED_RC and not _ssh_spoke(said): + return None + if rc == NOT_FOUND_RC and not _NOT_FOUND.search(str(getattr(exc, "stderr", None) or "")): + return None + if rc == TIMED_OUT_RC: + return None + return explain(target, rc, said) + + +def _ssh_spoke(log: str) -> bool: + """Whether ssh's log holds a word of its own beyond the first-connect host-key note.""" + return any(line.strip() and not line.startswith("Warning: Permanently added ") for line in log.splitlines()) + + +# How a POSIX shell says the command it was told to exec is not there. +_NOT_FOUND = re.compile(r"not found|No such file or directory|cannot execute", re.IGNORECASE) + + +def root_of(remote_cwd: str | None) -> str: + """Where an entry's agent works on its machine, as written: its ``remoteCwd`` or the default.""" + return (str(remote_cwd or "").strip() or DEFAULT_ROOT).rstrip("/") or "/" + + +def session_root(remote_cwd: str | None, handle: str) -> str: + """``/`` as written, where an agent's session works on its machine. + + The directory the node makes (``session/dir``), before the machine has + resolved it -- what a record and an announce can name on every path, a + failed and a cancelled run included. + """ + return f"{root_of(remote_cwd).rstrip('/')}/{leaf(handle)}" + + +def files_note(media: Sequence[Any], target: Machine) -> str: + """What the agent on ``target`` is told about the files that came with its task: their addresses, nothing more. + + Nothing is carried to the machine. A file is named as the sender has it, + and the agent uses it where the machine can read it -- the same path on + a shared disk, a copy already there -- or says that it could not be found + on the machine, and goes on. + """ + paths = [str(getattr(item, "path", "") or "") for item in media] + paths = [path for path in paths if path] + if not paths: + return "" + listed = "\n".join(f"- {path}" for path in paths) + return ( + "Files that came with this task, by their address on the computer that sent it. Nothing was copied here: " + "use a file where this machine can read it (the same path on a shared disk, a copy already here), and " + f"where it cannot be found on machine {target.label}, say so in your reply and go on.\n{listed}" + ) + + +def agent_missing(target: Machine) -> str: + """The sentence for an agent's command that is not on its machine.""" + return ( + f"the agent's command was not found on machine {target.label}; " + "install it there, for the user the machine is registered with" + ) def ssh_log_path(agent: str) -> Path: @@ -216,6 +292,46 @@ def read_ssh_log(path: str | os.PathLike[str], limit: int = 4000) -> str: return "" +def run(target: Machine, command: str, *, timeout: float = 60.0) -> tuple[int, str]: + """``command``, a POSIX shell line, run once on ``target``: ``(exit code, output)``. + + The registry's own one-shot runner (:func:`raven.ops.transport.runner_from`), + so a command here and a look through the machine channel reach the machine + the same way; sent through :func:`posix`, so the user's login shell may be + any shell. Blocking: call it off the event loop. + """ + try: + wrapped = posix(command) + runner = runner_from(target._row, cap_seconds=timeout) + except (TransportError, ValueError) as exc: + raise RemoteMachineError(f"machine {target.label}: {exc}") from None + return runner(wrapped) + + +def push(target: Machine, command: str, data: bytes, *, timeout: float = 300.0) -> tuple[int, str]: + """``command`` on ``target`` with ``data`` on its stdin: how a file is sent there. + + The same ssh options as :func:`run`, and the same :func:`posix` wrapping. + Output is ``(exit code, stdout, then stderr when it failed)``, the shape + :func:`run` answers in, so a failure is read by :func:`explain` either way. + Blocking: call it off the event loop. + """ + host, port, key, user = _target(target) + try: + wrapped = posix(command) + except ValueError as exc: + raise RemoteMachineError(f"machine {target.label}: {exc}") from None + argv = ssh_argv(host, port, key, user=user) + [wrapped] + try: + done = subprocess.run(argv, input=data, capture_output=True, timeout=timeout) + except subprocess.TimeoutExpired: + return TIMED_OUT_RC, "" + out = done.stdout.decode("utf-8", "replace") + if done.returncode != 0 and done.stderr: + out += "\n" + done.stderr.decode("utf-8", "replace") + return done.returncode, out + + def explain(target: Machine, rc: int, output: str) -> str | None: """One sentence for a failure ssh itself caused, or None when ssh got through. @@ -225,10 +341,7 @@ def explain(target: Machine, rc: int, output: str) -> str | None: if rc == TIMED_OUT_RC: return f"machine {target.label} did not answer in time" if rc == NOT_FOUND_RC: - return ( - f"the agent's command was not found on machine {target.label}; " - "install it there, for the user the machine is registered with" - ) + return agent_missing(target) if rc != SSH_FAILED_RC: return None for pattern, said in _SSH_FAILURES: @@ -274,21 +387,7 @@ def leaf(handle: str) -> str: return f"{safe}-{hashlib.sha256(raw.encode('utf-8')).hexdigest()[:8]}" -def _far_side_words(target: Machine, lines: list[str]) -> str: - """The machine's own last word on a command that failed past ssh, or ``""``. - - The machine's reason is the useful part ("No space left on device", - measured 2026-10-10 on a registered machine whose disk had filled), and it - names nothing of how raven got there. The one-shot runner shares stderr - with ssh, though, and ssh's lines name the address (on a first connect, - "Permanently added '[]:'"), so a line carrying it is skipped. - """ - host = str(target._row.get("host") or "").strip() - words = [line for line in lines if not (host and host in line)] - return words[-1][:200] if words else "" - - -def _shell_path(path: str) -> str: +def shell_path(path: str) -> str: """``path`` quoted for the machine's shell, leaving a leading ``~`` to it.""" path = str(path or "").strip() or DEFAULT_ROOT if path == "~": @@ -299,6 +398,14 @@ def _shell_path(path: str) -> str: return shlex.quote(path) +def shown(target: Machine, path: str) -> str: + """``path`` on ``target`` the way scp writes it, with the registry's user, address and port. + + What the file tools read back as a remote path (:mod:`raven.ops.remote_path`). + """ + return remote_path.RemotePath(machine=target.id, path=path, _row=target._row).display() + + def _target(target: Machine) -> tuple[str, int, str, str]: try: return ssh_target(target._row) @@ -307,16 +414,24 @@ def _target(target: Machine) -> tuple[str, int, str, str]: __all__ = [ + "CHECK_HANDLE", "DEFAULT_ROOT", "KEEPALIVE", "Machine", "RemoteMachineError", + "agent_missing", "explain", - "launch_command", + "failure", "leaf", "machine", + "posix", + "push", "read_ssh_log", - "remote_command", - "session_dir", + "root_of", + "run", + "session_root", + "shell_path", + "shown", + "ssh_line", "ssh_log_path", ] diff --git a/raven/cli/gateway_commands.py b/raven/cli/gateway_commands.py index c1308d6d7..ebd3e7053 100644 --- a/raven/cli/gateway_commands.py +++ b/raven/cli/gateway_commands.py @@ -1301,6 +1301,7 @@ async def _when_idle() -> None: await control.stop() from raven.acp_client.client import begin_drain from raven.acp_client.pool import close_pool + from raven.node.client import close_node_pool # Before anything tears a transport down: an ACP connection # closed first fails every pending turn with a connection error, @@ -1322,8 +1323,12 @@ async def _when_idle() -> None: await gw_teardown() # ACP agents are launched with start_new_session, so they do not # get this process's signals and outlive it unless the pool is - # closed. + # closed. The ssh of each Raven node on a registered machine is + # launched the same way, and the agents the node started end + # with it; the node pool is closed after the agents' so a + # closing agent still has its link to be ended through. await close_pool() + await close_node_pool() await agent.close_mcp() agent.stop() # Not folded into AgentLoop.stop: its other caller is the diff --git a/raven/cli/ops_connection_commands.py b/raven/cli/ops_connection_commands.py index f5787c964..dd639bf47 100644 --- a/raven/cli/ops_connection_commands.py +++ b/raven/cli/ops_connection_commands.py @@ -244,6 +244,87 @@ def doctor(as_json: bool = typer.Option(False, "--json")) -> None: raise typer.Exit(0 if fit else 1) +@connection_app.command("install-node") +def install_node( + conn_id: str = typer.Argument(..., help="The machine's id in the registry."), + node_dir: str = typer.Option( + "", "--dir", help="Where on the machine to keep Raven nodes (default ~/.raven-node); kept in the registry." + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Install without asking first."), + prune: bool = typer.Option( + False, "--prune", help="Also remove the nodes of other Raven versions from that directory." + ), +) -> None: + """Put this Raven's node on a registered machine, so files can be read and agents started there. + + The node is this Raven's own code in a small virtualenv of its own, under + ``/-``; nothing else on the machine changes. Asks + before installing; a machine that already has this node is left as it is. + """ + import asyncio + + from raven.node import install as node_install + from raven.node.client import NodeError, NodePool + + chosen = node_dir.strip().rstrip("/") + try: + plan = node_install.plan(conn_id, node_dir=chosen or None) + except node_install.InstallError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from None + if plan.present: + console.print(f"Machine {plan.target.label} already has the Raven node {plan.install_name} in {plan.node_dir}.") + else: + if (why := node_install.refusal(plan)) is not None: + console.print(f"[red]{why}[/red]") + raise typer.Exit(1) + console.print(plan.describe()) + if not yes and not typer.confirm("Install it?", default=False): + raise typer.Exit(1) + try: + done = node_install.install(plan) + except node_install.InstallError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from None + size = f" ({done.size_kb // 1024} MB)" if done.size_kb else "" + console.print(f"Installed the Raven node {done.install_name} in {done.node_dir}{size}.") + if not done.rg: + console.print( + "[yellow]ripgrep-bin was not available from the machine's package index or PyPI; " + "grep there runs Raven's own Python search, as it does on a computer without rg.[/yellow]" + ) + if prune: + try: + removed = node_install.prune(plan) + except node_install.InstallError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from None + console.print(f"Removed {len(removed)} other node(s) from {plan.node_dir}: {', '.join(removed) or 'none'}.") + if chosen and chosen != str(plan.target.field("node_dir") or "").strip().rstrip("/"): + from raven.ops.connection_add import set_fields + + try: + set_fields(conn_id, {"node_dir": chosen}) + except ValueError as exc: + console.print(f"[red]The node is installed, but {chosen} could not be recorded: {exc}[/red]") + raise typer.Exit(1) from None + console.print(f"Recorded {chosen} as where machine {plan.target.label} keeps its Raven nodes.") + + async def answer() -> dict: + pool = NodePool() + try: + return await pool.hello(conn_id) + finally: + await pool.close_all() + + try: + hello = asyncio.run(answer()) + except NodeError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from None + console.print(f"[green]The node answers: Python {hello.get('python')}, code {hello.get('digest')}.[/green]") + + # The host's `ops` group holds only the registry. The campaign commands stay in # the on-call agent's own checkout: the host needs to know which machines exist # and to write one down while the owner is in the conversation, not to run diff --git a/raven/cli/tui_commands.py b/raven/cli/tui_commands.py index 8d12564d7..fe65782ae 100644 --- a/raven/cli/tui_commands.py +++ b/raven/cli/tui_commands.py @@ -589,11 +589,15 @@ async def _start_backend() -> None: _logger.exception("tui: plugin services stop failed; continuing shutdown") # ACP agents are launched with start_new_session, so they do not get the - # terminal's signals and outlive this process unless the pool is closed. + # terminal's signals and outlive this process unless the pool is closed; + # so does the ssh of a Raven node on a registered machine, whose pool is + # closed after the agents' so a closing agent still has its link. try: from raven.acp_client.pool import close_pool + from raven.node.client import close_node_pool await close_pool() + await close_node_pool() except Exception: from loguru import logger as _logger diff --git a/raven/core/runtime.py b/raven/core/runtime.py index 8a50c2e9b..d6df814ab 100644 --- a/raven/core/runtime.py +++ b/raven/core/runtime.py @@ -163,6 +163,13 @@ def build_runtime( frozen ``RavenRuntime`` this returns is FREEZE. """ from raven.agent.tools.deliverables import DeliverableStore + from raven.node.client import get_node_pool + from raven.ops.remote_path import use_nodes + + # A file call on a registered machine is answered by that machine's Raven + # node. The tools reach the pool through the registry's package, because + # the node package imports the tools and the reverse would close a cycle. + use_nodes(get_node_pool) from raven.config.paths import get_deliverables_path if provider_pool is None: diff --git a/raven/home.py b/raven/home.py index 28c727446..1aabc70f1 100644 --- a/raven/home.py +++ b/raven/home.py @@ -26,9 +26,11 @@ from raven.contracts.path_policy import CONFIG_FILENAME, DEFAULT_HOME_DIRNAME, HOME_ENV_VAR __all__ = [ + "MACHINE_LANE_ENV", "SUBAGENT_ENV_VAR", "get_config_path", "is_subagent_process", + "machine_lane_enabled", "raven_home", "set_config_path", "subagent_role_env", @@ -83,3 +85,18 @@ def subagent_role_env() -> dict[str, str]: registry -- the failure this exists to close, and one no error reports. """ return {SUBAGENT_ENV_VAR: "1"} + + +#: Set to ``1`` to turn the machine lane on: acp sub-agents that run on a +#: registered machine, and files read there through a Raven node. Off by +#: default -- the lane is debug-only until it matures (ruled 2026-10-11), so an +#: entry that names a machine stays inert in normal operation. An environment +#: variable rather than a config field, because the model edits config through +#: ``raven_config`` and must not be able to turn the lane on itself; whoever +#: starts the process sets it. +MACHINE_LANE_ENV = "RAVEN_DEBUG_MACHINES" + + +def machine_lane_enabled() -> bool: + """Whether this process may dispatch to, and read files on, registered machines.""" + return os.environ.get(MACHINE_LANE_ENV, "").strip().lower() in ("1", "true", "yes", "on") diff --git a/raven/node/__init__.py b/raven/node/__init__.py new file mode 100644 index 000000000..55772bb86 --- /dev/null +++ b/raven/node/__init__.py @@ -0,0 +1,7 @@ +"""A Raven node: Raven's own file tools and the agents it dispatches, run on a registered machine for the Raven that started it. + +The node half is :mod:`raven.node.server`, run there as ``python -m raven.node +--stdio``; the host half is :mod:`raven.node.client` and :mod:`raven.node.install`. +This package's ``__init__`` imports nothing, so starting a node loads only what +the node itself needs. +""" diff --git a/raven/node/__main__.py b/raven/node/__main__.py new file mode 100644 index 000000000..9a37e5a10 --- /dev/null +++ b/raven/node/__main__.py @@ -0,0 +1,52 @@ +"""``python -m raven.node``: run this process as a Raven Subagent Server, or as the way to one. + +``--serve --dir D`` runs the daemon of this install under the node directory +``D``; ``--attach --dir D`` bridges this process's stdin and stdout to that +daemon, starting it when none is up -- what the host runs over ssh; +``--stdio`` serves the host on stdin and stdout directly, with agents that +end when the host hangs up (tests, and a machine with no socket to spare); +``--version`` prints what the host checks before using a node: the protocol +and the digest of this code. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m raven.node", description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--stdio", action="store_true", help="serve JSON-RPC on stdin and stdout; agents end with it") + group.add_argument("--serve", action="store_true", help="run the daemon under --dir until idle or stopped") + group.add_argument("--attach", action="store_true", help="bridge stdin and stdout to the daemon under --dir") + group.add_argument("--version", action="store_true", help="print the protocol and the code digest as JSON") + parser.add_argument("--dir", help="the node directory on this machine (run/, tasks/, log/ live under it)") + parser.add_argument("--idle", type=float, default=None, help="seconds a daemon stays with nothing to do") + args = parser.parse_args(argv) + if args.version: + from raven.node import bundle, protocol + + print(json.dumps({"protocol": protocol.PROTOCOL, "digest": bundle.digest()})) + return 0 + if args.stdio: + from raven.node.server import run_stdio + + return run_stdio() + if not args.dir: + parser.error("--serve and --attach need --dir") + root = Path(args.dir).expanduser() + if args.serve: + from raven.node.server import IDLE_EXIT_S, run_daemon + + return run_daemon(root, idle_s=IDLE_EXIT_S if args.idle is None else args.idle) + from raven.node.relay import run_attach + + return run_attach(root) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/raven/node/bundle.py b/raven/node/bundle.py new file mode 100644 index 000000000..92b323d23 --- /dev/null +++ b/raven/node/bundle.py @@ -0,0 +1,102 @@ +"""What a Raven node runs: this Raven's own source, and the digest that names it. + +A node runs the host's code exactly, so the two answer a file call alike. The +code travels as the ``raven`` package directory itself -- the same files +whether this Raven came from a release or a checkout -- and the digest of those +files is how both ends tell they hold the same code: the host computes it +before installing, the node computes it again over what arrived, and +``node/hello`` compares the two. A version number alone could not: two +checkouts of one version can differ. + +Standard library only; the node imports this too. +""" + +from __future__ import annotations + +import hashlib +import io +import tarfile +from functools import lru_cache +from pathlib import Path + +_SKIPPED_DIRS = frozenset({"__pycache__"}) +_SKIPPED_SUFFIXES = (".pyc", ".pyo") + + +def package_dir() -> Path: + """The ``raven`` package directory this process imports from.""" + import raven + + return Path(raven.__file__).resolve().parent + + +def files(root: Path | None = None) -> list[Path]: + """Every file a node needs, relative to the package directory, in a fixed order. + + Bytecode is left out: each Python writes its own, so it would make two + copies of one source differ. + """ + base = root or package_dir() + out: list[Path] = [] + for path in base.rglob("*"): + rel = path.relative_to(base) + if any(part in _SKIPPED_DIRS for part in rel.parts) or path.suffix in _SKIPPED_SUFFIXES: + continue + if path.is_file(): + out.append(rel) + return sorted(out, key=lambda p: p.as_posix()) + + +def tree_digest(root: Path) -> str: + """A short digest of the files' names and contents under ``root``; equal on two machines iff the code is.""" + hasher = hashlib.sha256() + for rel in files(root): + hasher.update(rel.as_posix().encode("utf-8") + b"\0") + hasher.update((root / rel).read_bytes()) + hasher.update(b"\0") + return hasher.hexdigest()[:12] + + +@lru_cache(maxsize=1) +def digest() -> str: + """:func:`tree_digest` of this process's own package, worked out once: the code it runs does not change.""" + return tree_digest(package_dir()) + + +def install_name() -> str: + """The directory name a node of this code is installed under: version, then digest. + + Not a "node id": that names a delegated task's address (``node_id`` on + ``spawn`` and DAG nodes), and this names an install. + """ + from raven import __version__ + + return f"{__version__}-{digest()}" + + +def tarball(root: Path | None = None) -> bytes: + """The package as a gzipped tar, every file under ``raven/``, ready to unpack into a node's ``lib``.""" + base = root or package_dir() + buffer = io.BytesIO() + # Dereferenced, so a link in the checkout arrives as the file it points at + # and the node, which has only what was sent, digests the same bytes. + with tarfile.open(fileobj=buffer, mode="w:gz", dereference=True) as archive: + for rel in files(base): + archive.add(base / rel, arcname=f"raven/{rel.as_posix()}", recursive=False) + return buffer.getvalue() + + +def pinned(names: tuple[str, ...]) -> list[str]: + """``name==version`` for each package as installed here, or the bare name when it is not.""" + from importlib.metadata import PackageNotFoundError, version + + out = [] + for name in names: + try: + out.append(f"{name}=={version(name)}") + except PackageNotFoundError: + out.append(name) + return out + + +__all__ = ["digest", "files", "install_name", "package_dir", "pinned", "tarball", "tree_digest"] diff --git a/raven/node/client.py b/raven/node/client.py new file mode 100644 index 000000000..990e8bbb3 --- /dev/null +++ b/raven/node/client.py @@ -0,0 +1,809 @@ +"""The host's side of a Raven node: one live link per registered machine. + +A node is reached over ssh (:func:`raven.acp_client.remote.ssh_line`): the +relay the ssh runs bridges it to the daemon on the machine, starting one when +none is up. It is +spoken to with the client an ACP agent is (:class:`raven.acp_client.client.AcpClient`), +because the framing is the same. The link is kept for as long as the machine +is in use: every file call to it, every agent started there and every probe +of it travels on that one connection. A link that has carried nothing past +:data:`IDLE_CLOSE_S` and runs no agent is closed the next time any node is +asked for, and its ssh goes with it. + +The link is watched from here, not by ssh: a ``node/ping`` every +:data:`HEARTBEAT_S` seconds, and one unanswered for :data:`HEARTBEAT_LOST_S` +means the link is gone -- not the agents. They run on under the machine's +daemon, which keeps their frames; each agent's endpoint here waits, holding +what the host writes meanwhile, and the link is tried again on a backoff +(:data:`RECONNECT_BACKOFF_S`) for as long as any agent is waiting. A link +that comes back attaches each agent again by its task id, from the frame the +host last read, and the turn goes on as if nothing had happened. Only a task +the machine no longer knows ends an endpoint with the sentence that says so. + +Before a node is used, ``node/hello`` has to show the same protocol and the +same code as this Raven (:func:`raven.node.bundle.digest`): a node that differs +would answer a file call differently from the tool it stands in for. Every +failure here is one sentence that names the machine, never its address, and +says what to do. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import hashlib +import os +import time +import uuid +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from functools import partial +from pathlib import Path +from typing import Any + +from loguru import logger + +from raven.acp_client import remote +from raven.acp_client.client import AcpClient +from raven.acp_client.protocol import AcpConnectionError, AcpError, AcpRemoteError, AcpTimeoutError +from raven.node import bundle, protocol +from raven.ops import remote_path + +#: Where a machine keeps its nodes when its registry row names nowhere else +#: (``node_dir``). ``~`` is the registered user's home there. +DEFAULT_NODE_DIR = "~/.raven-node" + +#: How long a link may carry nothing, with no agent on it, before the next acquire closes it. +IDLE_CLOSE_S = 600.0 + +#: How often the host pings a node it holds a link to. +HEARTBEAT_S = 5.0 +#: A ping unanswered this long means the link is lost: three heartbeats. +HEARTBEAT_LOST_S = 15.0 +#: The waits before each try at a lost link, in order; after the last, the next call tries again. +RECONNECT_BACKOFF_S = (2.0, 4.0, 8.0, 16.0) + +#: What an agent's connection reads as its exit code when its link went, not the agent. +LOST_RC = -1 + +_HELLO_TIMEOUT_S = 60.0 +_CALL_TIMEOUT_S = 120.0 +_LAUNCH_TIMEOUT_S = 60.0 +_ATTACH_TIMEOUT_S = 30.0 +#: How many frames an endpoint holds for a link that is down before it gives up on it. +_PENDING_FRAMES = 1000 +_CANCEL_TIMEOUT_S = 5.0 + +#: The node's own refusal codes, as the file tools hear of them (``NodeCallError.reason``). +_REASONS = { + protocol.CHANGED: "changed", + protocol.NOT_FOUND: "not-found", + protocol.BUSY: "busy", + protocol.UNKNOWN_TASK: "unknown-task", +} + +#: How the machine's shell ends when the node's Python is not there: 127 from +#: dash and bash 5, 126 from bash 3.2 (macOS's ``/bin/sh``), which reports an +#: ``exec`` of a missing path as "cannot execute" (measured 2026-10-10). +_MISSING_RCS = frozenset({remote.NOT_FOUND_RC, 126}) + + +class NodeError(remote_path.NodeCallError): + """A call to a machine that could not be made. The message is safe to show.""" + + +def node_dir(target: remote.Machine) -> str: + """The directory on ``target`` that holds its nodes.""" + return str(target.field("node_dir") or "").strip().rstrip("/") or DEFAULT_NODE_DIR + + +def node_python(target: remote.Machine, install_name: str) -> str: + """The Python of the node of ``install_name`` on ``target``.""" + return f"{node_dir(target)}/{install_name}/bin/python" + + +def install_hint(target: remote.Machine) -> str: + """The command that puts this Raven's node on ``target``.""" + return f"raven ops connection install-node {target.id}" + + +_TASK_CHARS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._@-") + + +def task_id(name: str) -> str: + """A task id for an agent called ``name``: its spelling the machine accepts, and a tail nobody repeats.""" + safe = "".join(ch if ch in _TASK_CHARS else "-" for ch in name.strip())[:40].strip("-.") or "agent" + return f"{safe}-{uuid.uuid4().hex[:8]}" + + +@dataclass(frozen=True) +class MachineStatus: + """What the listing says about a machine's link, with no round trip.""" + + state: str + """``connected`` (agents running), ``idle`` (a link, no agent), ``lost`` (the link is down, agents + wait for it), ``off`` (no link), or ``error``.""" + detail: str = "" + """The one sentence for ``error`` and ``lost``.""" + agents: int = 0 + version: str = "" + """The node's Raven version, when a link is up.""" + + def said(self) -> str: + """The parenthetical a listing shows, or ``""`` for a machine nothing has reached yet.""" + if self.state == "connected": + return f"connected, {self.agents} agent{'s' if self.agents != 1 else ''} running" + if self.state == "idle": + return "connected, idle" + if self.state == "lost": + tasks = f"{self.agents} task{'s continue' if self.agents != 1 else ' continues'}" + return f"link lost: {self.detail}; {tasks} on the machine; reconnecting" + if self.state == "error": + return f"connection error: {self.detail}" + return "" + + +class ChannelEndpoint: + """An agent a node started, as the far end of an :class:`AcpClient`. + + The client reads and writes it as it would a child process's pipes; what + it gets are the ``agent/frame`` and ``agent/stderr`` notifications of one + channel, and what it writes goes back the same way. ``end`` is + ``agent/cancel``: the node ends the agent's whole process tree, and the + ``agent/exit`` that follows is what settles ``wait``. A link that goes + before the agent does leaves the endpoint waiting (``detach``): the agent + runs on under the machine's daemon, what the client writes meanwhile is + held, and once the pool has the link back it attaches the task again + (``reattach``) and the held frames go out. Only a task the machine no + longer knows ends the endpoint with the sentence that says so, which the + client then raises instead of an exit code. + """ + + def __init__( + self, link: AcpClient, channel: int, *, target: remote.Machine, agent: str, task: str | None = None + ) -> None: + self._link = link + self.channel = channel + self.target = target + self.agent = agent + self.task = task or task_id(agent) + self.pid: int | None = None + self.received = 0 + """How many of the agent's frames reached here: where an attach goes on from.""" + self._frames: asyncio.Queue[bytes | None] = asyncio.Queue() + self._stderr: asyncio.Queue[bytes | None] = asyncio.Queue() + self._rc: int | None = None + self._lost: str | None = None + self._detached: str | None = None + self._pending: list[str] = [] + self._cancel_wanted = False + self._done = asyncio.Event() + + @property + def returncode(self) -> int | None: + if self._rc is not None: + return self._rc + return LOST_RC if self._lost is not None else None + + def describe(self) -> str: + pid = f", pid {self.pid}" if self.pid else "" + return f"task {self.task} on machine {self.target.label}{pid}" + + @property + def detached(self) -> str | None: + """Why the endpoint is waiting for its link, or ``None`` while it has one.""" + return self._detached + + async def write(self, data: bytes) -> None: + if self.returncode is not None: + raise BrokenPipeError(f"the agent on {self.describe()} has ended") + line = data.decode("utf-8", "replace").rstrip("\r\n") + if self._detached is not None or not self._link.alive: + # Down, or gone and not yet said so by the heartbeat: the frame waits for the next link. + self._hold(line) + return + try: + await self._link.notify(protocol.AGENT_FRAME, {"channel": self.channel, "line": line}) + except AcpConnectionError: + # The pool's heartbeat will say the link is lost; the frame waits for the next one. + self._hold(line) + + def _hold(self, line: str) -> None: + if len(self._pending) >= _PENDING_FRAMES: + raise BrokenPipeError( + f"the link to machine {self.target.label} has been down too long: {len(self._pending)} frames wait" + ) + self._pending.append(line) + + async def readline(self) -> bytes: + item = await self._frames.get() + return b"" if item is None else item + + async def readerr(self) -> bytes: + item = await self._stderr.get() + return b"" if item is None else item + + async def end(self) -> None: + """End the agent and what it started, through the node; while the link is down, as soon as it is back.""" + if self.returncode is not None: + return + if self._detached is not None: + self._cancel_wanted = True + return + with contextlib.suppress(AcpError): + await self._link.request(protocol.AGENT_CANCEL, {"channel": self.channel}, timeout=_CANCEL_TIMEOUT_S) + + async def wait(self) -> int | None: + await self._done.wait() + return self.returncode + + def ended(self) -> str | None: + return self._lost + + def release(self) -> None: + """Nothing is held: the link is the pool's.""" + + # --- fed by the pool, off the link's notifications --------------------------- + + def feed_frame(self, line: Any, n: Any = None) -> None: + if isinstance(line, str) and line and not self._done.is_set(): + self.received = n if isinstance(n, int) and not isinstance(n, bool) else self.received + 1 + self._frames.put_nowait(line.encode("utf-8")) + + def feed_stderr(self, text: Any) -> None: + if isinstance(text, str) and not self._done.is_set(): + self._stderr.put_nowait(text.encode("utf-8")) + + def exited(self, rc: Any) -> None: + if self._done.is_set(): + return + self._rc = rc if isinstance(rc, int) and not isinstance(rc, bool) else LOST_RC + self._settle() + + def lost(self, reason: str) -> None: + if self._done.is_set(): + return + self._lost = reason + self._settle() + + def detach(self, reason: str) -> None: + """The link went; the agent did not. Wait for the pool to bring a link back.""" + if not self._done.is_set(): + self._detached = reason + + async def reattach(self, link: AcpClient, channel: int) -> None: + """The task is on ``channel`` of ``link`` again: send what waited, and the cancel if one was asked.""" + self._link, self.channel = link, channel + self._detached = None + held, self._pending = self._pending, [] + for line in held: + await self._link.notify(protocol.AGENT_FRAME, {"channel": channel, "line": line}) + if self._cancel_wanted: + self._cancel_wanted = False + await self.end() + + def _settle(self) -> None: + self._frames.put_nowait(None) + self._stderr.put_nowait(None) + self._done.set() + + +@dataclass +class _Node: + client: AcpClient + target: remote.Machine + log: Path + """The ssh log of this link, where ssh's own words go.""" + hello: dict[str, Any] + channels: dict[int, ChannelEndpoint] + used: float = field(default_factory=time.monotonic) + heartbeat: asyncio.Task[None] | None = None + next_channel: int = 1 + + @property + def home(self) -> str: + """The login home on the machine, as the node reported it at its handshake.""" + return str(self.hello.get("home") or "") + + +class NodePool: + """Live links, at most one per machine, shared by every caller in this process.""" + + def __init__( + self, + *, + idle_close_s: float = IDLE_CLOSE_S, + heartbeat_s: float = HEARTBEAT_S, + lost_after_s: float = HEARTBEAT_LOST_S, + backoff_s: tuple[float, ...] = RECONNECT_BACKOFF_S, + ) -> None: + self._nodes: dict[str, _Node] = {} + self._locks: dict[str, asyncio.Lock] = {} + self._errors: dict[str, str] = {} + self._reconnects: dict[str, asyncio.Task[None]] = {} + self._parked: dict[str, list[ChannelEndpoint]] = {} + """The endpoints waiting for a machine's link to come back, by machine.""" + self._idle_close_s = idle_close_s + self._heartbeat_s = heartbeat_s + self._lost_after_s = lost_after_s + self._backoff_s = tuple(backoff_s) + self._closing = False + + # --- what a caller asks of a machine ------------------------------------------- + + async def call( + self, machine_id: str, method: str, params: dict[str, Any], *, timeout: float = _CALL_TIMEOUT_S + ) -> dict[str, Any]: + """``method`` on the node of ``machine_id``, starting one if none is live.""" + node = await self._acquire(machine_id) + label = node.target.label + try: + result = await node.client.request(method, params, timeout=timeout) + except AcpRemoteError as exc: + raise NodeError( + f"the Raven node on machine {label} refused {method}: {exc.message}", reason=_REASONS.get(exc.code) + ) from None + except AcpTimeoutError: + await self._lost( + machine_id, node, f"the Raven node on machine {label} did not answer {method} within {timeout:.0f}s" + ) + raise NodeError( + f"the Raven node on machine {label} did not answer {method} within {timeout:.0f}s" + ) from None + except AcpConnectionError as exc: + said = self._ended(node.target, node.log, exc) + await self._lost(machine_id, node, said) + raise NodeError(said) from None + node.used = time.monotonic() + return result if isinstance(result, dict) else {} + + async def hello(self, machine_id: str) -> dict[str, Any]: + """The handshake of the node of ``machine_id``: its protocol, digest, Python and home.""" + return await self.call(machine_id, protocol.HELLO, {"protocol": protocol.PROTOCOL}) + + async def home(self, machine_id: str) -> str: + """The login home on ``machine_id``, as its node reported it; starts the node if none is live.""" + node = await self._acquire(machine_id) + return node.home + + async def session_dir(self, machine_id: str, path: str) -> tuple[str, float]: + """Make ``path`` on the machine and return it resolved there, with the machine's clock as it was made.""" + got = await self.call(machine_id, protocol.SESSION_DIR, {"path": path}) + resolved = str(got.get("path") or "") + if not resolved.startswith("/"): + label = (await self._acquire(machine_id)).target.label + raise NodeError(f"the Raven node on machine {label} did not say where it made {path}") + since = got.get("since") + return resolved, float(since) if isinstance(since, (int, float)) else 0.0 + + async def probe_machine(self, machine_id: str) -> dict[str, Any]: + """One look at the machine: free disk, Python, shell, uv, load.""" + return await self.call(machine_id, protocol.PROBE_MACHINE, {}) + + async def probe_agent(self, machine_id: str, command: str, env: Mapping[str, str] | None = None) -> dict[str, Any]: + """Whether ``command`` is on the machine, on the PATH an agent there gets (``env`` may override it).""" + params: dict[str, Any] = {"command": command} + if env and isinstance(env.get("PATH"), str): + params["env"] = {"PATH": env["PATH"]} + return await self.call(machine_id, protocol.PROBE_AGENT, params) + + async def stat(self, machine_id: str, path: str, *, roots: Sequence[str], hash: bool = False) -> dict[str, Any]: + """What is at ``path`` on the machine: exists, kind, size, mtime, and its digest when asked.""" + return await self.call(machine_id, protocol.STAT, {"path": path, "roots": list(roots), "hash": hash}) + + async def tasks(self, machine_id: str) -> list[dict[str, Any]]: + """Every task the machine's daemon runs or remembers: id, state, exit code, times, frames.""" + answer = await self.call(machine_id, protocol.TASK_LIST, {}) + rows = answer.get("tasks") + return [row for row in rows if isinstance(row, dict)] if isinstance(rows, list) else [] + + async def pull_file(self, machine_id: str, remote_file: str, local: Path, *, roots: Sequence[str]) -> int: + """Bring ``remote_file`` from the machine to ``local``, in chunks, checked whole; the bytes received. + + Written beside its destination first and moved into place only when + the digest the node gave with the first chunk matches, so a file that + did not arrive whole is never left looking as if it had. + """ + local.parent.mkdir(parents=True, exist_ok=True) + part = local.with_name(local.name + ".raven-part") + hasher = hashlib.sha256() + expected: str | None = None + offset = 0 + try: + with part.open("wb") as handle: + while True: + params = {"path": remote_file, "offset": offset, "length": protocol.CHUNK, "roots": list(roots)} + got = await self.call(machine_id, protocol.PULL, params) + data = base64.b64decode(str(got.get("data") or "")) + if offset == 0 and isinstance(got.get("sha256"), str): + expected = got["sha256"] + handle.write(data) + hasher.update(data) + offset += len(data) + if got.get("eof") or not data: + break + if expected is not None and hasher.hexdigest() != expected: + label = (await self._acquire(machine_id)).target.label + raise NodeError(f"{remote_file} on machine {label} did not arrive whole; what arrived was removed") + os.replace(part, local) + finally: + part.unlink(missing_ok=True) + return offset + + async def open_channel( + self, machine_id: str, *, name: str, command: str, cwd: str, env: Mapping[str, str], task: str | None = None + ) -> ChannelEndpoint: + """Start ``command`` on the machine, in ``cwd`` with ``env`` over its login environment. + + The endpoint returned is what an :class:`AcpClient` runs over + (``AcpClient.over``). ``env`` travels on the link, never on a command + line, and the link is not journaled. ``task`` names the run on the + machine, where its record outlives this link; one is made up from + ``name`` when the caller has none. + """ + node = await self._acquire(machine_id) + label = node.target.label + channel = node.next_channel + node.next_channel += 1 + task = task or task_id(name) + endpoint = ChannelEndpoint(node.client, channel, target=node.target, agent=name, task=task) + # Registered before the launch, so a frame the agent writes at once has somewhere to go. + node.channels[channel] = endpoint + params = {"channel": channel, "task": task, "command": command, "cwd": cwd, "env": dict(env)} + try: + started = await node.client.request(protocol.AGENT_LAUNCH, params, timeout=_LAUNCH_TIMEOUT_S) + except AcpRemoteError as exc: + node.channels.pop(channel, None) + raise NodeError(self._refused_launch(node, name, exc)) from None + except AcpTimeoutError: + node.channels.pop(channel, None) + said = f"the Raven node on machine {label} did not answer {protocol.AGENT_LAUNCH} within {_LAUNCH_TIMEOUT_S:.0f}s" + await self._lost(machine_id, node, said) + raise NodeError(said) from None + except AcpConnectionError as exc: + node.channels.pop(channel, None) + said = self._ended(node.target, node.log, exc) + await self._lost(machine_id, node, said) + raise NodeError(said) from None + if isinstance(started, dict) and isinstance(started.get("pid"), int): + endpoint.pid = started["pid"] + node.used = time.monotonic() + logger.info("acp agent {!r}: started on {}", name, endpoint.describe()) + return endpoint + + def status(self, machine_id: str) -> MachineStatus: + """The link's state, with no round trip: for a listing.""" + node = self._nodes.get(machine_id) + if node is not None and node.client.alive: + agents = len(node.channels) + version = str(node.hello.get("version") or "") + return MachineStatus("connected" if agents else "idle", agents=agents, version=version) + if waiting := self._parked.get(machine_id): + said = self._errors.get(machine_id, "").removesuffix("; reconnecting") + return MachineStatus("lost", said, agents=len(waiting)) + if said := self._errors.get(machine_id): + return MachineStatus("error", said) + return MachineStatus("off") + + def waiting(self, machine_id: str) -> int: + """How many agents wait for the machine's link to come back.""" + return len(self._parked.get(machine_id, ())) + + def live(self) -> list[str]: + """The machines that have a node running for this process.""" + return [machine_id for machine_id, node in self._nodes.items() if node.client.alive] + + async def close(self, machine_id: str) -> None: + self._stop_reconnect(machine_id) + await self._drop(machine_id) + + async def close_all(self) -> None: + """Close every link. The agents on the machines run on under their daemons.""" + self._closing = True + try: + for machine_id in list(self._reconnects): + self._stop_reconnect(machine_id) + for machine_id in list(self._nodes): + await self._drop(machine_id) + finally: + self._closing = False + + # --- the link ------------------------------------------------------------------ + + async def _acquire(self, machine_id: str) -> _Node: + await self._close_idle(keep=machine_id) + lock = self._locks.setdefault(machine_id, asyncio.Lock()) + async with lock: + node = self._nodes.get(machine_id) + if node is not None and node.client.alive: + return node + if node is not None: + await self._drop(machine_id, f"the Raven node on machine {node.target.label} stopped") + try: + node = await self._launch(machine_id) + except NodeError as exc: + self._errors[machine_id] = str(exc) + raise + self._nodes[machine_id] = node + self._errors.pop(machine_id, None) + node.heartbeat = asyncio.create_task(self._heartbeat(machine_id, node), name=f"node-heartbeat-{machine_id}") + return node + + def _prepare(self, machine_id: str) -> tuple[remote.Machine, Path, str, str]: + """The machine, its ssh log (emptied), the ssh line and the install name. Blocking: reads the registry.""" + target = remote.machine(machine_id) + install_name = bundle.install_name() + log = remote.ssh_log_path(f"raven-node@{target.id}") + try: + # Emptied per launch, so a failure is read from this launch's lines. + log.write_text("", encoding="utf-8") + except OSError: + pass + # ``-I``: the login shell's PYTHONPATH or PYTHONHOME would otherwise + # reach the node, and a shadow ``raven`` on the former answers the + # handshake with another digest that no reinstall cures. Started from + # the home directory, which the ssh login lands in: the node makes no + # directory of its own. + # ``exec``: the shell that reads the line becomes the node, so the ssh + # session has one process at its far end and a signal to it reaches + # the node, not a wrapper. + # ``--attach``: the relay, which bridges this ssh to the machine's + # daemon and starts one when none is up; the daemon, not the ssh, + # owns the agents. + command = ( + f"exec {remote.shell_path(node_python(target, install_name))} -I -m raven.node --attach " + f"--dir {remote.shell_path(node_dir(target))}" + ) + try: + line = remote.ssh_line(target, command, ssh_log=log) + except ValueError as exc: + raise remote.RemoteMachineError(f"machine {target.label}: {exc}") from None + return target, log, line, install_name + + async def _launch(self, machine_id: str) -> _Node: + try: + target, log, line, install_name = await asyncio.to_thread(self._prepare, machine_id) + except remote.RemoteMachineError as exc: + raise NodeError(str(exc)) from None + channels: dict[int, ChannelEndpoint] = {} + try: + client = await AcpClient.launch( + name=f"raven-node@{target.id}", + command=line, + # Fixed rather than any caller's workspace: the ssh needs none. + cwd=str(Path.home()), + env={}, + on_notification=partial(self._notified, channels), + ) + except AcpConnectionError as exc: + raise NodeError(f"could not start ssh to machine {target.label}: {exc}") from None + try: + hello = await client.request(protocol.HELLO, {"protocol": protocol.PROTOCOL}, timeout=_HELLO_TIMEOUT_S) + except AcpConnectionError as exc: + await client.close() + raise NodeError(self._ended(target, log, exc, install_name)) from None + except AcpError as exc: + await client.close() + reason = exc.message if isinstance(exc, AcpRemoteError) else str(exc) + raise NodeError( + f"the Raven node on machine {target.label} did not complete its handshake: {reason}" + ) from None + digest = hello.get("digest") if isinstance(hello, dict) else None + if not isinstance(hello, dict) or hello.get("protocol") != protocol.PROTOCOL or digest != bundle.digest(): + await client.close() + raise NodeError( + f"the Raven node on machine {target.label} runs different code from this Raven; " + f"update it with `{install_hint(target)}`" + ) + node = _Node(client, target, log, hello, channels) + remote_path.remember_home(target.id, node.home) + return node + + async def _notified(self, channels: dict[int, ChannelEndpoint], method: str, params: dict[str, Any]) -> None: + """One notification off the link, handed to the channel it names.""" + channel = params.get("channel") + endpoint = channels.get(channel) if isinstance(channel, int) and not isinstance(channel, bool) else None + if endpoint is None: + logger.debug("raven node: {} for channel {!r}, which is not open", method, channel) + return + if method == protocol.AGENT_FRAME: + endpoint.feed_frame(params.get("line"), params.get("n")) + elif method == protocol.AGENT_STDERR: + endpoint.feed_stderr(params.get("text")) + elif method == protocol.AGENT_EXIT: + channels.pop(channel, None) + endpoint.exited(params.get("rc")) + + async def _heartbeat(self, machine_id: str, node: _Node) -> None: + """Ping the node until one ping goes unanswered; then the link is lost.""" + while True: + await asyncio.sleep(self._heartbeat_s) + try: + await node.client.request(protocol.PING, {"t": time.time()}, timeout=self._lost_after_s) + except AcpTimeoutError: + said = f"machine {node.target.label} did not answer a heartbeat within {self._lost_after_s:.0f}s" + except AcpConnectionError as exc: + said = self._ended(node.target, node.log, exc) + except AcpError: + # A refusal is an answer: the node is there. + continue + else: + continue + await self._lost(machine_id, node, said) + return + + async def _lost(self, machine_id: str, node: _Node, said: str) -> None: + """The link to ``node`` is gone: park its agents, close it, and try again on the backoff.""" + if self._nodes.get(machine_id) is not node: + # Already replaced or closed by another path. + return + logger.warning("raven node on machine {}: link lost: {}", node.target.id, said) + self._errors[machine_id] = f"{said}; reconnecting" + await self._drop(machine_id, said) + if not self._closing and machine_id not in self._reconnects: + self._reconnects[machine_id] = asyncio.create_task( + self._reconnect(machine_id), name=f"node-reconnect-{machine_id}" + ) + + async def _reconnect(self, machine_id: str) -> None: + """Try the link again on the backoff; for as long as agents wait for it, keep trying at the last delay.""" + tried = 0 + try: + if not self._backoff_s: + # No backoff at all: the link is tried again by the next call only. + self._errors[machine_id] = self._errors.get(machine_id, "").removesuffix("; reconnecting") + return + while True: + delay = self._backoff_s[min(tried, len(self._backoff_s) - 1)] + tried += 1 + await asyncio.sleep(delay) + try: + await self._acquire(machine_id) + except NodeError as exc: + self._errors[machine_id] = f"{exc}; reconnecting" + else: + logger.info("raven node on machine {}: link restored", machine_id) + await self._reattach(machine_id) + if not self._parked.get(machine_id): + return + continue + if tried >= len(self._backoff_s) and not self._parked.get(machine_id): + # Given up for now: the sentence stands until the next call tries again. + self._errors[machine_id] = self._errors.get(machine_id, "").removesuffix("; reconnecting") + return + finally: + self._reconnects.pop(machine_id, None) + + async def _reattach(self, machine_id: str) -> None: + """Every endpoint parked for the machine onto a channel of its new link, from the frame it last read.""" + node = self._nodes.get(machine_id) + waiting = self._parked.pop(machine_id, []) + if node is None or not node.client.alive: + if waiting: + self._parked[machine_id] = waiting + return + for i, endpoint in enumerate(waiting): + if endpoint.returncode is not None: + continue + channel = node.next_channel + node.next_channel += 1 + node.channels[channel] = endpoint + params = {"channel": channel, "task": endpoint.task, "after": endpoint.received} + try: + await node.client.request(protocol.AGENT_ATTACH, params, timeout=_ATTACH_TIMEOUT_S) + except AcpRemoteError as exc: + node.channels.pop(channel, None) + endpoint.lost( + f"machine {node.target.label} no longer runs {endpoint.agent!r} (task {endpoint.task}): {exc.message}" + ) + continue + except (AcpTimeoutError, AcpConnectionError) as exc: + # This link is gone too: this endpoint and every one not yet + # reached wait for the next one. + node.channels.pop(channel, None) + self._parked.setdefault(machine_id, []).extend(e for e in waiting[i:] if e.returncode is None) + said = ( + self._ended(node.target, node.log, exc) + if isinstance(exc, AcpConnectionError) + else f"the Raven node on machine {node.target.label} did not answer {protocol.AGENT_ATTACH}" + ) + await self._lost(machine_id, node, said) + return + if endpoint.returncode is None: + await endpoint.reattach(node.client, channel) + logger.info("acp agent {!r}: attached again on {}", endpoint.agent, endpoint.describe()) + + def _stop_reconnect(self, machine_id: str) -> None: + task = self._reconnects.pop(machine_id, None) + if task is not None and task is not asyncio.current_task(): + task.cancel() + + def _refused_launch(self, node: _Node, name: str, exc: AcpRemoteError) -> str: + if exc.code == protocol.NOT_FOUND: + return remote.agent_missing(node.target) + if exc.code == protocol.BUSY: + limits = node.hello.get("limits") if isinstance(node.hello.get("limits"), dict) else {} + limit = limits.get("agents") or protocol.MAX_AGENTS + return f"machine {node.target.label} already runs {limit} agents; wait for one to finish" + return f"the Raven node on machine {node.target.label} could not start {name!r}: {exc.message}" + + def _ended( + self, target: remote.Machine, log: Path, exc: AcpConnectionError, install_name: str | None = None + ) -> str: + """``exc`` in one sentence: not installed, an ssh failure, or the node's own last word.""" + if exc.returncode in _MISSING_RCS: + installed = f"Raven node {install_name}" if install_name else "this Raven's node" + return f"{installed} is not installed on machine {target.label}; install it with `{install_hint(target)}`" + if (said := remote.failure(target, log, exc)) is not None: + return said + lines = [line.strip() for line in (exc.stderr or "").splitlines() if line.strip()] + # The node's own reason, a traceback's last line, closes its stderr. + last = lines[-1][:200] if lines else "" + return f"the Raven node on machine {target.label} stopped" + (f": {last}" if last else "") + + async def _drop(self, machine_id: str, reason: str | None = None) -> None: + """Forget the link and close it. + + With a ``reason`` the link was lost: its agents are parked to wait for + the next one. Without, it was closed on purpose, and the agents on it + are told so; they run on under the machine's daemon, which a later + link can list (``tasks``). + """ + node = self._nodes.pop(machine_id, None) + if node is None: + return + if node.heartbeat is not None and node.heartbeat is not asyncio.current_task(): + node.heartbeat.cancel() + for endpoint in list(node.channels.values()): + if reason is None: + endpoint.lost(f"the link to machine {node.target.label} was closed") + else: + endpoint.detach(reason) + self._parked.setdefault(machine_id, []).append(endpoint) + node.channels.clear() + await node.client.close() + + async def _close_idle(self, *, keep: str) -> None: + now = time.monotonic() + for machine_id, node in list(self._nodes.items()): + if machine_id != keep and not node.channels and now - node.used > self._idle_close_s: + await self._drop(machine_id) + + +_POOL: NodePool | None = None + + +def get_node_pool() -> NodePool: + """The process-wide pool.""" + global _POOL + if _POOL is None: + _POOL = NodePool() + return _POOL + + +async def close_node_pool() -> None: + """Close every node this process started, and forget the pool.""" + global _POOL + pool, _POOL = _POOL, None + if pool is not None: + await pool.close_all() + + +__all__ = [ + "DEFAULT_NODE_DIR", + "HEARTBEAT_LOST_S", + "HEARTBEAT_S", + "IDLE_CLOSE_S", + "LOST_RC", + "RECONNECT_BACKOFF_S", + "ChannelEndpoint", + "MachineStatus", + "NodeError", + "NodePool", + "close_node_pool", + "get_node_pool", + "install_hint", + "node_dir", + "node_python", +] diff --git a/raven/node/install.py b/raven/node/install.py new file mode 100644 index 000000000..cbedf2c6d --- /dev/null +++ b/raven/node/install.py @@ -0,0 +1,298 @@ +"""Put this Raven's node on a registered machine. + +What lands there, all under ``//`` (``node_dir`` from the +machine's registry row, ``~/.raven-node`` when it names none): a Python 3.12 +virtualenv, the packages in :data:`raven.node.protocol.REQUIREMENTS` pinned to +this Raven's own versions, and this Raven's own code (:mod:`raven.node.bundle`). +Nothing else on the machine changes: no system package, no shell rc file, no +environment it already had. + +The Python is the machine's own when it has 3.12 or later with ``venv``; +otherwise the machine's ``uv`` makes one inside ``node_dir``. With neither, the +install stops and says so rather than fetching an installer and running it. + +Each step is one ssh call through :func:`raven.acp_client.remote.run`; a step +that fails removes the half-made node directory, so the next attempt starts +clean and :func:`plan` never mistakes a broken node for an installed one. +""" + +from __future__ import annotations + +import json +import shlex +from collections.abc import Callable +from dataclasses import dataclass + +from raven.acp_client import remote +from raven.node import bundle, protocol +from raven.node.client import node_dir as registered_node_dir + +#: Roughly what a node takes, for the free-space check: Raven's code with +#: pillow, loguru and ripgrep-bin (measured 2026-10-10), and a Python 3.12 when +#: the machine has none of its own. +_NODE_KB = 60 * 1024 +_PYTHON_KB = 80 * 1024 + +_PROBE_TIMEOUT_S = 60.0 +_STEP_TIMEOUT_S = 900.0 + + +class InstallError(RuntimeError): + """An install that could not go ahead or did not finish. Safe to show: no address.""" + + +@dataclass(frozen=True) +class Plan: + """What an install would do on one machine, found by one look at it.""" + + target: remote.Machine + node_dir: str + install_name: str + present: bool + python: str + uv: str + free_kb: int | None + + @property + def how(self) -> str: + """``"python"`` (the machine's own 3.12), ``"uv"`` (one made by uv), or ``""`` (neither).""" + if self.python: + return "python" + return "uv" if self.uv else "" + + @property + def needed_kb(self) -> int: + return _NODE_KB + (_PYTHON_KB if self.how == "uv" else 0) + + def describe(self) -> str: + """The install as a person is asked to approve it.""" + where = f"{self.node_dir}/{self.install_name}" + python = ( + f"a virtualenv of the machine's own {self.python}" + if self.how == "python" + else "a Python 3.12 made by the machine's uv, kept beside it" + ) + space = f"; {self.free_kb // (1024 * 1024)} GB free there" if self.free_kb is not None else "" + return ( + f"Install the Raven node {self.install_name} on machine {self.target.label} into {where}: {python}, " + f"{' '.join(bundle.pinned((*protocol.REQUIREMENTS, protocol.SEARCH_REQUIREMENT)))} from the " + "machine's package index, and this Raven's code. " + f"About {self.needed_kb // 1024} MB{space}. Nothing else on the machine changes." + ) + + +@dataclass(frozen=True) +class Installed: + """A node that answered with this Raven's digest after being put in place.""" + + node_dir: str + install_name: str + size_kb: int | None + rg: bool = True + """Whether ``ripgrep-bin`` went in; without it the node searches with Raven's Python fallback.""" + + +Runner = Callable[[remote.Machine, str], tuple[int, str]] +Pusher = Callable[[remote.Machine, str, bytes], tuple[int, str]] + + +def _run(target: remote.Machine, command: str) -> tuple[int, str]: + return remote.run(target, command, timeout=_STEP_TIMEOUT_S) + + +def _push(target: remote.Machine, command: str, data: bytes) -> tuple[int, str]: + return remote.push(target, command, data, timeout=_STEP_TIMEOUT_S) + + +def _vars(node_dir: str, install_name: str) -> str: + """The shell assignments every step starts with; ``~`` is left for the machine to expand.""" + return f'dir={remote.shell_path(node_dir)}; id={shlex.quote(install_name)}; node="$dir/$id"; ' + + +def probe_script(node_dir: str, install_name: str, digest: str) -> str: + """One look at the machine: is this node there, which Python and uv does it have, how much room.""" + return _vars(node_dir, install_name) + ( + f'if [ -x "$node/bin/python" ] && "$node/bin/python" -I -m raven.node --version 2>/dev/null ' + f"| grep -q {shlex.quote(f'"digest": "{digest}"')}; then echo PRESENT=1; fi; " + 'for p in python3.13 python3.12 python3; do q=$(command -v "$p" 2>/dev/null) || continue; ' + "if \"$q\" -c 'import sys, venv, ensurepip; sys.exit(0 if sys.version_info >= (3, 12) else 1)' " + '2>/dev/null; then echo "PYTHON=$q"; break; fi; done; ' + 'for u in "$(command -v uv 2>/dev/null)" "$HOME/.local/bin/uv" "$HOME/.cargo/bin/uv"; do ' + 'if [ -n "$u" ] && [ -x "$u" ]; then echo "UV=$u"; break; fi; done; ' + 'd="$dir"; while [ ! -d "$d" ]; do d=$(dirname "$d"); done; ' + 'echo "FREE_KB=$(df -Pk "$d" 2>/dev/null | awk \'NR==2 {print $4}\')"' + ) + + +def plan(machine_id: str, *, node_dir: str | None = None, run: Runner = _run) -> Plan: + """Look at ``machine_id`` once and say what installing there would do. + + ``node_dir`` is where to keep nodes when it should differ from the + machine's registry row; the caller records it once the install succeeds. + """ + try: + target = remote.machine(machine_id) + except remote.RemoteMachineError as exc: + raise InstallError(str(exc)) from None + where = (node_dir or "").strip().rstrip("/") or registered_node_dir(target) + install_name = bundle.install_name() + rc, out = run(target, probe_script(where, install_name, bundle.digest())) + if rc != 0: + raise InstallError(remote.explain(target, rc, out) or f"could not look at machine {target.label}") + seen: dict[str, str] = {} + for line in out.splitlines(): + key, sep, value = line.strip().partition("=") + if sep and key in {"PRESENT", "PYTHON", "UV", "FREE_KB"}: + seen[key] = value.strip() + free = seen.get("FREE_KB", "") + return Plan( + target=target, + node_dir=where, + install_name=install_name, + present=seen.get("PRESENT") == "1", + python=seen.get("PYTHON", ""), + uv=seen.get("UV", ""), + free_kb=int(free) if free.isdigit() else None, + ) + + +def refusal(p: Plan) -> str | None: + """Why ``p`` cannot go ahead, or ``None`` when it can.""" + if p.how == "": + return ( + f"machine {p.target.label} has neither a Python 3.12 (with venv) nor uv; " + "install one of them there, then run this again" + ) + if p.free_kb is not None and p.free_kb < p.needed_kb: + return ( + f"{p.node_dir} on machine {p.target.label} has {p.free_kb // 1024} MB free and a node needs about " + f"{p.needed_kb // 1024} MB; free some space or choose another directory with --dir" + ) + return None + + +def steps(p: Plan) -> list[tuple[str, str]]: + """``(name, command)`` for each ssh step after the code is sent, in order.""" + head = _vars(p.node_dir, p.install_name) + if p.how == "python": + venv = f'mkdir -p "$dir" && {shlex.quote(p.python)} -m venv "$node"' + else: + # Python, cache and all inside node_dir: the machine's home may be on a + # disk with no room (measured 2026-10-10: one registered machine's root + # filesystem was full). + venv = ( + 'mkdir -p "$dir" && UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python" ' + f'{shlex.quote(p.uv)} venv --quiet --seed --python 3.12 "$node"' + ) + pip = '"$node/bin/python" -m pip install --quiet --no-cache-dir --disable-pip-version-check --no-input' + requirements = " ".join(shlex.quote(r) for r in bundle.pinned(protocol.REQUIREMENTS)) + deps = f"{pip} {requirements}" + # Optional, and never fails the install: the machine's own index first (its + # owner chose it, often a nearby mirror); then the public index, for a + # mirror that does not carry it; else the node searches without rg. + (rg,) = bundle.pinned((protocol.SEARCH_REQUIREMENT,)) + search = ( + f"{{ {pip} {shlex.quote(rg)} || {pip} --extra-index-url {protocol.PUBLIC_INDEX} {shlex.quote(rg)}; }} " + ">/dev/null 2>&1 && echo RG=1 || echo RG=0" + ) + link = ( + '"$node/bin/python" -c \'import site, sys; ' + 'open(site.getsitepackages()[0] + "/raven-node.pth", "w").write(sys.argv[1] + "\\n")\' "$node/lib"' + ) + return [ + ("virtualenv", head + venv), + ("packages", head + deps), + ("search", head + search), + ("code", head + 'mkdir -p "$node/lib" && tar -xzf - -C "$node/lib"'), + ("path", head + link), + ("check", head + '"$node/bin/python" -I -m raven.node --version'), + ] + + +def prune_script(node_dir: str, install_name: str) -> str: + """Remove every other node in ``node_dir``, printing ``PRUNED=`` for each. + + A directory counts as a node only when it holds both a ``bin/python`` and + a ``lib/raven``, so the uv cache and Python beside them, and anything else + the owner keeps there, are left alone. + """ + return _vars(node_dir, install_name) + ( + 'for d in "$dir"/*; do [ "$d" = "$node" ] && continue; ' + '[ -x "$d/bin/python" ] && [ -d "$d/lib/raven" ] && rm -rf "$d" && echo "PRUNED=${d##*/}"; done; true' + ) + + +def prune(p: Plan, *, run: Runner = _run) -> tuple[str, ...]: + """Remove the nodes of other code from ``p``'s ``node_dir``; the ids removed. + + Opt-in: two Ravens of different versions may share one machine account, + and each needs its own node. + """ + rc, out = run(p.target, prune_script(p.node_dir, p.install_name)) + if rc != 0: + raise InstallError(_failed(p, "prune", rc, out)) + return tuple(line.strip()[len("PRUNED=") :] for line in out.splitlines() if line.strip().startswith("PRUNED=")) + + +def install(p: Plan, *, run: Runner = _run, push: Pusher = _push) -> Installed: + """Carry out ``p``. Raises :class:`InstallError`, after removing the half-made node.""" + if (why := refusal(p)) is not None: + raise InstallError(why) + head = _vars(p.node_dir, p.install_name) + rg = False + try: + for name, command in steps(p): + if name == "code": + rc, out = push(p.target, command, bundle.tarball()) + else: + rc, out = run(p.target, command) + if rc != 0: + raise InstallError(_failed(p, name, rc, out)) + if name == "search": + rg = "RG=1" in out + if name == "check": + _confirm_digest(p, out) + except BaseException: + # Only the node's own directory: node_dir may hold other nodes, and a + # Python uv made there is reused by the next attempt. + run(p.target, head + 'rm -rf "$node"') + raise + rc, out = run(p.target, head + 'du -sk "$node" 2>/dev/null | cut -f1') + size = out.strip().splitlines()[-1] if rc == 0 and out.strip() else "" + return Installed(p.node_dir, p.install_name, int(size) if size.isdigit() else None, rg=rg) + + +def _failed(p: Plan, step: str, rc: int, out: str) -> str: + """One sentence for a failed step: ssh's failure explained, else the machine's own last line.""" + if (said := remote.explain(p.target, rc, out)) is not None and rc != remote.NOT_FOUND_RC: + return f"installing the Raven node: {said}" + lines = [line.strip() for line in out.splitlines() if line.strip()] + last = lines[-1][:300] if lines else f"exit {rc}" + return f"installing the Raven node on machine {p.target.label} failed at the {step} step: {last}" + + +def _confirm_digest(p: Plan, out: str) -> None: + lines = [line for line in out.splitlines() if line.strip().startswith("{")] + try: + reported = json.loads(lines[-1]) if lines else {} + except ValueError: + reported = {} + if reported.get("digest") != bundle.digest() or reported.get("protocol") != protocol.PROTOCOL: + raise InstallError( + f"the Raven node installed on machine {p.target.label} does not report this Raven's code " + f"({reported.get('digest') or 'nothing'} instead of {bundle.digest()})" + ) + + +__all__ = [ + "InstallError", + "Installed", + "Plan", + "install", + "plan", + "probe_script", + "prune", + "prune_script", + "refusal", + "steps", +] diff --git a/raven/node/protocol.py b/raven/node/protocol.py new file mode 100644 index 000000000..732cccbbc --- /dev/null +++ b/raven/node/protocol.py @@ -0,0 +1,129 @@ +"""The wire between a Raven and the Raven Subagent Server on a registered machine. + +JSON-RPC 2.0, one object per line: the framing ACP uses, so the host drives a +machine with the client it drives an ACP agent with +(:class:`raven.acp_client.client.AcpClient`). On the machine a daemon answers +(:mod:`raven.node.server`); the host reaches it through a relay started over +ssh (``python -I -m raven.node --attach``), which bridges the ssh session to +the daemon's local socket and starts the daemon when none is up. The daemon +opens no network port. + +One connection per machine carries everything the host asks of it. Agents the +daemon starts for the host each get a *channel* of that connection, on which +the agent's ACP frames travel in both directions as ``agent/frame`` +notifications -- and a *task* id of the host's choosing, which outlives the +connection: an agent keeps running when the ssh goes, its frames are kept on +the machine, and a new connection picks them up from where the host stopped +reading (``agent/attach``). Requests are answered as they finish, not in the +order they arrive: a search of a large tree does not hold a heartbeat behind +it. + +Every ``fs/*`` call carries ``roots``, the directories it may touch. The host +decides them; the daemon checks them again, because only the machine can +resolve its own symlinks. +""" + +from __future__ import annotations + +#: Bumped when a method or a field changes meaning. A daemon and a host that +#: disagree refuse each other at ``node/hello`` instead of misreading a reply. +PROTOCOL = 3 + +HELLO = "node/hello" +PING = "node/ping" +SHUTDOWN = "node/shutdown" + +PROBE_MACHINE = "probe/machine" +PROBE_AGENT = "probe/agent" +SESSION_DIR = "session/dir" + +AGENT_LAUNCH = "agent/launch" +AGENT_ATTACH = "agent/attach" +AGENT_FRAME = "agent/frame" +AGENT_STDERR = "agent/stderr" +AGENT_EXIT = "agent/exit" +AGENT_CANCEL = "agent/cancel" +AGENT_CLOSE = "agent/close" +TASK_LIST = "task/list" + +READ = "fs/read" +LIST = "fs/list" +FIND = "fs/find" +GREP = "fs/grep" +STAT = "fs/stat" +WRITE = "fs/write" +EDIT = "fs/edit" +PULL = "fs/pull" + +#: Error codes of the daemon's own, beyond JSON-RPC's, for the refusals the +#: host answers with a sentence of its own rather than the daemon's message. +NOT_FOUND = -32010 +"""``agent/launch``: the command is not on the machine's PATH.""" +BUSY = -32011 +"""``agent/launch``: :data:`MAX_AGENTS` are already running.""" +CHANGED = -32012 +"""``fs/write``: the file is not as ``expect`` said it was when it was read.""" +UNKNOWN_TASK = -32014 +"""``agent/attach``: no task of that id is running or recorded on the machine.""" + +#: The most bytes one ``fs/pull`` returns per chunk, before base64. A line of +#: the protocol holds one chunk and little else. +CHUNK = 1024 * 1024 + +#: How many agents one machine runs at once; a launch past it is refused with :data:`BUSY`. +MAX_AGENTS = 8 + +#: How long a finished task's record (its frames, its exit) is kept on the +#: machine for a host to come back for. +TASK_KEEP_S = 7 * 24 * 3600 + +#: Third-party packages the daemon cannot run without, beside Raven's own +#: code: the only ones its file tools import (measured 2026-10-10), pinned to +#: the host's versions. +REQUIREMENTS = ("loguru", "pillow") + +#: Installed when the machine can get it, not required. ``ripgrep-bin`` puts the +#: same ``rg`` beside the daemon's Python that the host has beside its own, so a +#: search reads the same on both machines; without it ``grep`` runs Raven's own +#: Python search, as it does on a host with no ``rg`` (measured 2026-10-10: one +#: registered machine's package mirror did not carry it). +SEARCH_REQUIREMENT = "ripgrep-bin" + +#: Where an install looks for a package its own index does not carry. +PUBLIC_INDEX = "https://pypi.org/simple" + +__all__ = [ + "AGENT_ATTACH", + "AGENT_CANCEL", + "AGENT_CLOSE", + "AGENT_EXIT", + "AGENT_FRAME", + "AGENT_LAUNCH", + "AGENT_STDERR", + "BUSY", + "CHANGED", + "CHUNK", + "EDIT", + "FIND", + "GREP", + "HELLO", + "LIST", + "MAX_AGENTS", + "NOT_FOUND", + "PING", + "PROBE_AGENT", + "PROBE_MACHINE", + "PROTOCOL", + "PUBLIC_INDEX", + "PULL", + "READ", + "REQUIREMENTS", + "SEARCH_REQUIREMENT", + "SESSION_DIR", + "SHUTDOWN", + "STAT", + "TASK_KEEP_S", + "TASK_LIST", + "UNKNOWN_TASK", + "WRITE", +] diff --git a/raven/node/relay.py b/raven/node/relay.py new file mode 100644 index 000000000..3eecd3f8e --- /dev/null +++ b/raven/node/relay.py @@ -0,0 +1,125 @@ +"""The relay: what the host's ssh runs on the machine, bridging its session to the daemon. + +``python -I -m raven.node --attach --dir D`` connects to the daemon's socket +under ``D`` and copies bytes both ways until either side closes -- the ssh +session on one side, the daemon on the other. It parses nothing. When no +daemon answers it starts one, detached from this session so the daemon +outlives it, and waits for the socket to come up. So the host never has to +install or start a service: the first relay after a reboot brings the daemon +back, and the daemon leaves on its own once idle. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import subprocess +import sys +import time +from pathlib import Path + +from raven.node import server + +#: How long a relay waits for a daemon it started to open its socket. +START_TIMEOUT_S = 20.0 +_POLL_S = 0.2 +_BLOCK = 65536 + + +def start_daemon(root: Path, *, python: str | None = None) -> subprocess.Popen[bytes]: + """Start the daemon under ``root`` in a session of its own; its words go to ``log/serve.log``.""" + log_dir = root / server.LOG_DIR + log_dir.mkdir(parents=True, exist_ok=True) + log = (log_dir / "serve.log").open("ab") + argv = [python or sys.executable, "-I", "-m", "raven.node", "--serve", "--dir", str(root)] + # How long the daemon stays idle is the daemon's default, unless the + # environment the relay runs in says otherwise (tests end theirs quickly). + if idle := os.environ.get("RAVEN_NODE_IDLE_S", "").strip(): + argv += ["--idle", idle] + try: + return subprocess.Popen( # noqa: S603 - this install's own Python, no caller input + argv, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=log, + start_new_session=True, + close_fds=True, + cwd=str(Path.home()), + ) + finally: + log.close() + + +async def connect( + root: Path, *, python: str | None = None, timeout_s: float = START_TIMEOUT_S +) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: + """A connection to the daemon under ``root``, started here when none answers.""" + sock = server.socket_path(root) + deadline = time.monotonic() + timeout_s + started = False + while True: + try: + return await asyncio.open_unix_connection(str(sock), limit=_BLOCK) + except (FileNotFoundError, ConnectionRefusedError, OSError): + if not started: + start_daemon(root, python=python) + started = True + elif time.monotonic() > deadline: + raise ConnectionError(f"the Raven daemon under {root} did not answer within {timeout_s:.0f}s") from None + await asyncio.sleep(_POLL_S) + + +async def bridge( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter, stdin: asyncio.StreamReader, out: object +) -> None: + """Copy bytes between the session (``stdin``/``out``) and the daemon until either side ends.""" + + async def inward() -> None: + while True: + data = await stdin.read(_BLOCK) + if not data: + return + writer.write(data) + await writer.drain() + + async def outward() -> None: + while True: + data = await reader.read(_BLOCK) + if not data: + return + out.write(data) # type: ignore[attr-defined] + out.flush() # type: ignore[attr-defined] + + pumps = [asyncio.create_task(inward()), asyncio.create_task(outward())] + try: + await asyncio.wait(pumps, return_when=asyncio.FIRST_COMPLETED) + finally: + for pump in pumps: + pump.cancel() + with contextlib.suppress(OSError, ConnectionError): + writer.close() + await writer.wait_closed() + + +def run_attach(root: Path) -> int: + """Bridge this process's stdin and stdout to the daemon under ``root``; 1 when it cannot be reached.""" + out = sys.stdout.buffer + sys.stdout = sys.stderr + + async def main() -> int: + try: + reader, writer = await connect(root) + except ConnectionError as exc: + sys.stderr.write(f"{exc}\n") + return 1 + loop = asyncio.get_running_loop() + stdin = asyncio.StreamReader(limit=_BLOCK) + await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(stdin), sys.stdin) + await bridge(reader, writer, stdin, out) + return 0 + + return asyncio.run(main()) + + +__all__ = ["START_TIMEOUT_S", "bridge", "connect", "run_attach", "start_daemon"] diff --git a/raven/node/server.py b/raven/node/server.py new file mode 100644 index 000000000..2a95a63e5 --- /dev/null +++ b/raven/node/server.py @@ -0,0 +1,1277 @@ +"""The machine side: a daemon that runs the host's agents and answers its file calls, outliving any one connection. + +Started on a registered machine by the host's relay (``python -I -m raven.node +--attach --dir ``, over ssh) when none is up, and spoken to over a +unix socket under that directory; or, for tests and as a fallback, on this +process's stdin and stdout (``--stdio``). A request is a JSON-RPC 2.0 object +on one line; its answer is one line back. Each file call runs the very tool +the host would run on its own disk -- ``read_file``, ``list_dir``, ``find``, +``grep`` -- so a read of a remote file comes back exactly as a local one +would: the same line numbers, the same limits, the same footer. + +An agent the host starts here (``agent/launch``) runs in its own process group, +in the login environment this daemon captures once at its first need, under a +*task* id of the host's. Its ACP frames travel as ``agent/frame`` +notifications on a channel of the connection that asked, and are kept in the +task's record on disk as they arrive -- so when that connection goes, the +agent keeps running, and the next connection takes the task over from the +frame the host last read (``agent/attach``). Ending a task ends its whole +process tree, the commands the agent put in the background or in a session +of its own included. Only ``--stdio`` ties the agents to the connection. + +Deliberately small. It imports the file tools and nothing that talks to a +model, reads a config or opens a network port, so a machine needs Python +3.12 and the packages in :data:`raven.node.protocol.REQUIREMENTS` beside +Raven's own code, not the whole of Raven's dependencies (about 1 GB, +measured 2026-10-10). + +Requests run concurrently and are answered as they finish; what an agent +writes is forwarded as it arrives. One lock per connection orders its lines. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import fcntl +import hashlib +import json +import os +import platform +import re +import shlex +import shutil +import signal +import stat as filemode +import subprocess +import sys +import tempfile +import time +import uuid +from collections.abc import Awaitable, Callable +from functools import partial +from pathlib import Path +from typing import Any, BinaryIO + +from raven.node import bundle, protocol + +_PARSE_ERROR = -32700 +_INVALID_REQUEST = -32600 +_METHOD_NOT_FOUND = -32601 +_INVALID_PARAMS = -32602 +_INTERNAL_ERROR = -32603 + +#: A request line the node will read; a file call is a few hundred bytes, a +#: chunk of a file sent for writing about 1.4 MB. +_LINE_LIMIT = 2 * 1024 * 1024 + +#: What an agent may put on one stdout line before the node stops reading it: +#: the frame the ACP stdio protocol allows, and what the host reads. +_AGENT_LINE_LIMIT = 8 * 1024 * 1024 + +#: Login shells whose ``-lic`` reads the profile and rc chain an interactive +#: session does and prints nothing of its own on the way: the two the host +#: drives for its own agents (``raven.agent.subagent.backends.env``). A user +#: of any other shell gets the environment ssh started this node with. +_DRIVABLE_SHELLS = frozenset({"bash", "zsh"}) +#: Enough PATH to find ``env`` when ssh gave the node none. +_BOOTSTRAP_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +_CAPTURE_TIMEOUT_S = 15.0 + +#: How long an agent gets to leave on its own after its stdin closes, before +#: its group is ended. +_CLOSE_GRACE_S = 3.0 +#: How long the node waits for an ended group to be reaped on its way out. +_END_WAIT_S = 2.0 +#: How long the last answers get to reach a host that has hung up. +_ANSWER_DRAIN_S = 1.0 + + +class _RefusedError(Exception): + """A request answered with a JSON-RPC error rather than a result.""" + + def __init__(self, code: int, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _text(params: dict[str, Any], name: str, *, default: str | None = None) -> str: + value = params.get(name, default) + if not isinstance(value, str) or not value.strip(): + raise _RefusedError(_INVALID_PARAMS, f"{name} must be a non-empty string") + return value + + +def _number(params: dict[str, Any], name: str, default: int | None = None) -> int | None: + value = params.get(name, default) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise _RefusedError(_INVALID_PARAMS, f"{name} must be a whole number") + return value + + +def _flag(params: dict[str, Any], name: str) -> bool: + value = params.get(name, False) + if not isinstance(value, bool): + raise _RefusedError(_INVALID_PARAMS, f"{name} must be true or false") + return value + + +def _argv(params: dict[str, Any]) -> list[str]: + try: + argv = shlex.split(_text(params, "command")) + except ValueError as exc: + raise _RefusedError(_INVALID_PARAMS, f"command cannot be parsed: {exc}") from None + if not argv: + raise _RefusedError(_INVALID_PARAMS, "command is empty") + return argv + + +def _env(params: dict[str, Any]) -> dict[str, str]: + extra = params.get("env") or {} + if not isinstance(extra, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in extra.items()): + raise _RefusedError(_INVALID_PARAMS, "env must be an object of strings") + return dict(extra) + + +def _roots(params: dict[str, Any]) -> tuple[Path, ...]: + """The directories this call may touch, as given, made absolute on this machine. + + Required and never empty: the file tools read an empty fence as "no fence", + so a call that named none would reach the whole disk. + """ + raw = params.get("roots") + if not isinstance(raw, list) or not raw or not all(isinstance(r, str) and r.strip() for r in raw): + raise _RefusedError(_INVALID_PARAMS, "roots must be a non-empty list of directories") + out = [] + for root in raw: + path = Path(root.strip()).expanduser() + if not path.is_absolute(): + raise _RefusedError(_INVALID_PARAMS, f"root {root!r} is not an absolute directory") + out.append(path) + return tuple(out) + + +def _tool(cls: Any, params: dict[str, Any]) -> Any: + """One of Raven's file tools, fenced to this call's roots. + + Relative paths resolve against the home directory, as a path after the + colon in ``host:path`` does for scp. ``follow_binding=False``: there is no + turn here whose working directory could widen the fence. + """ + return cls(workspace=Path.home(), allowed_dirs=_roots(params), follow_binding=False) + + +def _answer(value: Any) -> dict[str, Any]: + """A tool's return value as the wire carries it: the text, an image's blocks, a write's diff.""" + if isinstance(value, str): + return {"text": value} + out: dict[str, Any] = { + "text": value.model_text, + "display": value.display_text, + "ok": bool(value.ok), + "blocks": list(getattr(value, "blocks", None) or []), + } + if getattr(value, "diff", None) is not None: + out["diff"] = value.diff + change = getattr(value, "file_change", None) + if change is not None: + out["file_change"] = {"path": change.path, "after": change.after, "before": change.before} + return out + + +def _fenced(params: dict[str, Any]) -> Path: + """The call's path placed on this machine, inside its roots, whether or not it exists yet. + + The same resolution the file tools make (symlinks followed, the roots + checked after), for the calls that are not a tool: a stat, a chunk, a + change listing. + """ + from raven.agent.tools.filesystem import resolve_path + + try: + return resolve_path(_text(params, "path"), Path.home(), _roots(params)) + except PermissionError as exc: + raise _RefusedError(_INVALID_PARAMS, str(exc)) from None + + +def _sha256(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(protocol.CHUNK), b""): + hasher.update(block) + return hasher.hexdigest() + + +def _as_it_is(path: Path) -> str: + """How ``fs/write``'s ``expect`` names a file's state: its digest, or ``absent``.""" + return _sha256(path) if path.is_file() else "absent" + + +async def _hello(params: dict[str, Any]) -> dict[str, Any]: + asked = params.get("protocol") + if asked is not None and asked != protocol.PROTOCOL: + raise _RefusedError(_INVALID_REQUEST, f"this node speaks protocol {protocol.PROTOCOL}, not {asked!r}") + from raven import __version__ + from raven.agent.tools.file_search import _resolve_rg + + return { + "protocol": protocol.PROTOCOL, + "version": __version__, + "digest": bundle.digest(), + "python": platform.python_version(), + "os": platform.system(), + "home": str(Path.home()), + "pid": os.getpid(), + # Whether ``grep`` runs rg here or Raven's own Python search: the two + # differ on regex corners, and the host may want to say which ran. + "rg": _resolve_rg() is not None, + "limits": {"agents": protocol.MAX_AGENTS, "line": _LINE_LIMIT}, + } + + +async def _ping(params: dict[str, Any]) -> dict[str, Any]: + return {"t": params.get("t")} + + +async def _read(params: dict[str, Any]) -> dict[str, Any]: + from raven.agent.tools.filesystem import ReadFileTool + + tool = _tool(ReadFileTool, params) + path = _text(params, "path") + out = _answer(await tool.execute(path, offset=_number(params, "offset", 1) or 1, limit=_number(params, "limit"))) + # The whole file's digest beside the page read, so a later write of the + # whole file can say what it expects to replace (``fs/write``, ``expect``). + try: + placed = tool._resolve(path) + except PermissionError: + return out + if placed.is_file(): + out["sha256"] = _sha256(placed) + return out + + +async def _list(params: dict[str, Any]) -> dict[str, Any]: + from raven.agent.tools.filesystem import ListDirTool + + tool = _tool(ListDirTool, params) + return _answer( + await tool.execute( + _text(params, "path"), recursive=_flag(params, "recursive"), max_entries=_number(params, "max_entries") + ) + ) + + +async def _find(params: dict[str, Any]) -> dict[str, Any]: + from raven.agent.tools.file_search import FindTool + + tool = _tool(FindTool, params) + return _answer( + await tool.execute( + _text(params, "pattern"), path=_text(params, "path", default="."), limit=_number(params, "limit") + ) + ) + + +async def _grep(params: dict[str, Any]) -> dict[str, Any]: + from raven.agent.tools.file_search import GrepTool + + tool = _tool(GrepTool, params) + glob = params.get("glob") + if glob is not None and not isinstance(glob, str): + raise _RefusedError(_INVALID_PARAMS, "glob must be a string") + mode = params.get("output_mode", "content") + if not isinstance(mode, str): + raise _RefusedError(_INVALID_PARAMS, "output_mode must be a string") + return _answer( + await tool.execute( + _text(params, "pattern"), + path=_text(params, "path", default="."), + glob=glob, + output_mode=mode, + case_insensitive=_flag(params, "case_insensitive"), + context=_number(params, "context", 0) or 0, + limit=_number(params, "limit"), + ) + ) + + +async def _stat(params: dict[str, Any]) -> dict[str, Any]: + """What is at a path: whether it exists, what kind, how big, when it changed, and on request its digest.""" + path = _fenced(params) + try: + st = path.stat() + except FileNotFoundError: + return {"exists": False, "kind": "missing", "size": 0, "mtime": None} + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not stat {path}: {exc.strerror or exc}") from None + if filemode.S_ISDIR(st.st_mode): + kind = "dir" + elif filemode.S_ISREG(st.st_mode): + kind = "file" + else: + kind = "other" + out: dict[str, Any] = {"exists": True, "kind": kind, "size": st.st_size, "mtime": st.st_mtime} + if kind == "file" and _flag(params, "hash"): + out["sha256"] = _sha256(path) + return out + + +async def _write(params: dict[str, Any]) -> dict[str, Any]: + """The host's ``write_file`` on this machine. + + ``expect`` is the file as the host last read it -- its digest, or + ``absent`` -- and a file that is not so any more is left alone, with + :data:`raven.node.protocol.CHANGED`. The check and the write are one + stretch with no await between them, and this node's requests interleave + only at awaits, so what was checked is what is written over. + """ + from raven.agent.tools.filesystem import WriteFileTool + + tool = _tool(WriteFileTool, params) + path = _text(params, "path") + content = params.get("content") + if not isinstance(content, str): + raise _RefusedError(_INVALID_PARAMS, "content must be a string") + mode = params.get("mode", "overwrite") + if not isinstance(mode, str): + raise _RefusedError(_INVALID_PARAMS, "mode must be a string") + expect = params.get("expect") + if expect is not None: + if not isinstance(expect, str): + raise _RefusedError(_INVALID_PARAMS, "expect must be a digest or 'absent'") + try: + placed = tool._resolve(path) + except PermissionError as exc: + return {"text": f"Error: {exc}"} + if _as_it_is(placed) != expect.strip().lower(): + raise _RefusedError(protocol.CHANGED, f"{placed} changed after it was read; nothing was written") + return _answer(await tool.execute(path, content, mode=mode)) + + +async def _edit(params: dict[str, Any]) -> dict[str, Any]: + """The host's ``edit_file`` on this machine: one read, the replacement, one write, with nothing between.""" + from raven.agent.tools.filesystem import EditFileTool + + tool = _tool(EditFileTool, params) + new_text = params.get("new_text") + if not isinstance(new_text, str): + raise _RefusedError(_INVALID_PARAMS, "new_text must be a string") + return _answer( + await tool.execute( + _text(params, "path"), _text(params, "old_text"), new_text, replace_all=_flag(params, "replace_all") + ) + ) + + +async def _pull(params: dict[str, Any]) -> dict[str, Any]: + """One chunk of a file the host fetches; the first chunk carries the whole file's digest.""" + path = _fenced(params) + if not path.is_file(): + raise _RefusedError(_INVALID_PARAMS, f"{path} is not a file on this machine") + offset = _number(params, "offset", 0) or 0 + length = _number(params, "length", protocol.CHUNK) or protocol.CHUNK + if offset < 0 or not 0 < length <= protocol.CHUNK: + raise _RefusedError(_INVALID_PARAMS, f"offset must be a whole number and length between 1 and {protocol.CHUNK}") + try: + size = path.stat().st_size + with path.open("rb") as handle: + handle.seek(offset) + data = handle.read(length) + out: dict[str, Any] = { + "data": base64.b64encode(data).decode("ascii"), + "total": size, + "eof": offset + len(data) >= size, + } + if offset == 0: + out["sha256"] = _sha256(path) + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not read {path}: {exc.strerror or exc}") from None + return out + + +async def _session_dir(params: dict[str, Any]) -> dict[str, Any]: + """Make the session directory the host names and say where it is, as this machine resolves it. + + The name is the host's: it knows the agent's root and the handle's safe + spelling. ``since`` is this machine's clock as the directory is made, so a + later listing of what changed can be asked for what came after. + """ + path = Path(_text(params, "path")).expanduser() + try: + path.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not make {path}: {exc.strerror or exc}") from None + return {"path": str(path.resolve()), "since": time.time()} + + +async def _probe_machine(params: dict[str, Any]) -> dict[str, Any]: + """One look at this machine: room, Python, shell, uv, load.""" + disk: dict[str, int | None] = {} + for where in (Path.home(), Path.cwd()): + try: + disk[str(where)] = shutil.disk_usage(where).free // 1024 + except OSError: + disk[str(where)] = None + try: + load: list[float] | None = [round(x, 2) for x in os.getloadavg()] + except (AttributeError, OSError): + load = None + return { + "disk_free_kb": disk, + "python": platform.python_version(), + "shell": os.environ.get("SHELL", ""), + "uv": shutil.which("uv") or "", + "load": load, + } + + +# --- what the node starts --------------------------------------------------------- + + +def capture_login_env() -> dict[str, str]: + """This machine user's login environment, as a shell session would have it. + + Taken the way the host takes its own: one ``-lic env -0`` of the login + shell, so an rc file's exports (nvm, a local bin dir) are in it and + whatever the rc file prints is not. Over the environment ssh started this + node with, not a minimal base: that environment is already the login one + PAM and sshd made (an ``/etc/environment`` PATH holding the machine's + conda, measured 2026-10-12), and the profile only adds to it. A login + shell this cannot drive, or a capture that fails, leaves it as it is. + """ + shell = os.environ.get("SHELL", "").strip() + if not shell or os.path.basename(shell) not in _DRIVABLE_SHELLS: + return dict(os.environ) + base = dict(os.environ) + base.setdefault("PATH", _BOOTSTRAP_PATH) + try: + done = subprocess.run( + [shell, "-lic", "env -0"], + env=base, + capture_output=True, + stdin=subprocess.DEVNULL, + timeout=_CAPTURE_TIMEOUT_S, + # ``-i`` reaches for a controlling terminal; a session of its own + # gives it none to take over. + start_new_session=True, + ) + except (OSError, subprocess.SubprocessError): + return dict(os.environ) + captured = { + key: value + for key, sep, value in (entry.partition("=") for entry in done.stdout.decode("utf-8", "replace").split("\0")) + if key and sep + } + if done.returncode != 0 or not captured.get("PATH"): + return dict(os.environ) + return captured + + +def _descendants(pid: int) -> list[int]: + """Every process under ``pid`` by parent chain, as ``ps`` sees them now.""" + ps = shutil.which("ps") or "/bin/ps" + try: + done = subprocess.run([ps, "-A", "-o", "pid=,ppid="], capture_output=True, text=True, timeout=5) + except (OSError, subprocess.SubprocessError): + return [] + children: dict[int, list[int]] = {} + for line in done.stdout.splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit(): + children.setdefault(int(parts[1]), []).append(int(parts[0])) + out: list[int] = [] + stack = [pid] + while stack: + for child in children.get(stack.pop(), []): + out.append(child) + stack.append(child) + return out + + +def _end_tree(pid: int) -> None: + """End ``pid``, the process group it leads and every process under it, in that group or out of it. + + The descendants are found before the first signal: afterwards they are + init's, and the tree that said whose they were is gone. Out of the group + is the case that matters: an agent may put a command in a session of its + own -- Claude Code's shell tool does (measured 2026-10-12: a loop it + started outlived the group's SIGKILL) -- and that session's group goes + with it. The node's own group is never signalled, whatever is found in + it: a descendant there is ended by itself. + """ + strays = _descendants(pid) + own = os.getpgrp() + if pid != own: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(pid, signal.SIGKILL) + for stray in strays: + try: + group = os.getpgid(stray) + except (ProcessLookupError, PermissionError): + continue + if group not in (pid, own): + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(group, signal.SIGKILL) + with contextlib.suppress(ProcessLookupError, PermissionError): + os.kill(stray, signal.SIGKILL) + with contextlib.suppress(ProcessLookupError, PermissionError): + os.kill(pid, signal.SIGKILL) + + +#: Under the node directory the host registered (``--dir``): the socket and +#: lock of each install under ``run/``, the task records under ``tasks/``, the +#: daemon's own log under ``log/``. +RUN_DIR, TASKS_DIR, LOG_DIR = "run", "tasks", "log" + +#: How long a daemon with no connection and no running task waits before it +#: leaves; the next relay starts it again. +IDLE_EXIT_S = 600.0 +_IDLE_CHECK_S = 15.0 +#: What a task's record says its agent ended with when the daemon that ran it +#: went first: the host reads it the way it reads a lost link. +LOST_RC = -1 + +_TASK_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._@-]{0,119}$") + + +#: The most bytes a unix socket's path may have on the systems a node runs on (macOS: 104). +_SOCKET_PATH_MAX = 100 + + +def socket_path(root: Path) -> Path: + """Where the daemon of this install listens for ``root``: under it, or in the temp directory when that is too long. + + A unix socket's path is short by the system's rule, and a node directory + is wherever the owner put it; the fallback is named after ``root`` so a + relay and a daemon given the same directory meet, whichever they use. + """ + under = root / RUN_DIR / f"{bundle.install_name()}.sock" + if len(os.fsencode(under)) <= _SOCKET_PATH_MAX: + return under + tag = hashlib.sha256(os.fsencode(root.resolve())).hexdigest()[:16] + return Path(tempfile.gettempdir()) / f"raven-node-{tag}.sock" + + +def pid_path(root: Path) -> Path: + """Where the daemon of this install writes its pid under ``root``.""" + return root / RUN_DIR / f"{bundle.install_name()}.pid" + + +def _task_id(params: dict[str, Any]) -> str: + raw = params.get("task") + if raw is None: + return f"t-{uuid.uuid4().hex[:12]}" + if not isinstance(raw, str) or not _TASK_ID.match(raw): + raise _RefusedError(_INVALID_PARAMS, "task must be a short name of letters, digits, '.', '_', '@' or '-'") + return raw + + +class _StreamOut: + """Lines out to one host, over stdout or a socket; one lock orders them.""" + + def __init__(self, out: BinaryIO | asyncio.StreamWriter) -> None: + self._out = out + self._lock = asyncio.Lock() + + async def write_line(self, line: bytes) -> None: + async with self._lock: + if isinstance(self._out, asyncio.StreamWriter): + if self._out.is_closing(): + raise ConnectionError("the host's connection is closed") + self._out.write(line) + await self._out.drain() + else: + self._out.write(line) + self._out.flush() + + +class _Task: + """One agent the daemon started for a host, under the host's task id; it outlives the connection that asked. + + What the agent writes goes to the task's record on disk as it arrives + (``frames.jsonl``, ``stderr.log``) and, while a connection holds the + task on a channel, to that connection. A connection that goes leaves the + agent running and the record growing; ``attach`` replays the record from + the frame the host last read and takes over from there. + """ + + def __init__( + self, task_id: str, proc: asyncio.subprocess.Process, daemon: Daemon, record: Path, *, command: str, cwd: str + ) -> None: + self.id = task_id + self.proc = proc + self.daemon = daemon + self.record = record + self.command = command + self.cwd = cwd + self.started = time.time() + self.ended: float | None = None + self.rc: int | None = None + self.frames = 0 + self.subscriber: tuple[Node, int] | None = None + self._send_lock = asyncio.Lock() + self._frames_file = (record / "frames.jsonl").open("ab") + self._stderr_file = (record / "stderr.log").open("ab") + self.write_record("running") + self._forwarders = [ + asyncio.create_task(self._forward(proc.stdout, frames=True)), + asyncio.create_task(self._forward(proc.stderr, frames=False)), + ] + self._exit_task = asyncio.create_task(self._exit()) + + @property + def channel(self) -> int | None: + return self.subscriber[1] if self.subscriber is not None else None + + def write_record(self, state: str) -> None: + data = { + "task": self.id, + "command": self.command, + "cwd": self.cwd, + "pid": self.proc.pid, + "state": state, + "rc": self.rc, + "started": self.started, + "ended": self.ended, + "frames": self.frames, + } + _write_json(self.record / "task.json", data) + + def summary(self) -> dict[str, Any]: + return { + "task": self.id, + "state": "running" if self.rc is None else "exited", + "rc": self.rc, + "pid": self.proc.pid, + "command": self.command, + "cwd": self.cwd, + "started": self.started, + "ended": self.ended, + "frames": self.frames, + "attached": self.subscriber is not None, + } + + async def _tell(self, node: Node, method: str, params: dict[str, Any]) -> None: + """One notification to the connection holding this task; a connection that is gone loses the task.""" + try: + await node.notify(method, params) + except (ConnectionError, OSError, ValueError): + if self.subscriber is not None and self.subscriber[0] is node: + self.subscriber = None + node.channels.pop(params.get("channel"), None) + + async def _forward(self, stream: asyncio.StreamReader | None, *, frames: bool) -> None: + if stream is None: # pragma: no cover - PIPE is always requested + return + while True: + try: + raw = await stream.readline() + except ValueError: + # Past the reader's limit; the reader has dropped it and goes on. + continue + if not raw: + return + text = raw.decode("utf-8", "replace").rstrip("\r\n") + if not text: + continue + async with self._send_lock: + handle = self._frames_file if frames else self._stderr_file + handle.write(text.encode("utf-8") + b"\n") + handle.flush() + if frames: + self.frames += 1 + if self.subscriber is not None: + node, channel = self.subscriber + if frames: + await self._tell( + node, protocol.AGENT_FRAME, {"channel": channel, "line": text, "n": self.frames} + ) + else: + await self._tell(node, protocol.AGENT_STDERR, {"channel": channel, "text": text}) + + async def _exit(self) -> None: + rc = await self.proc.wait() + # After the output is drained, so the last frame lands before the + # exit; bounded, because a grandchild the agent left behind can hold + # the pipes open past the agent itself. + _done, pending = await asyncio.wait(self._forwarders, timeout=_ANSWER_DRAIN_S) + for task in pending: + task.cancel() + async with self._send_lock: + self.rc = rc + self.ended = time.time() + self._frames_file.close() + self._stderr_file.close() + self.write_record("exited") + if self.subscriber is not None: + node, channel = self.subscriber + self.subscriber = None + node.channels.pop(channel, None) + await self._tell(node, protocol.AGENT_EXIT, {"channel": channel, "rc": rc}) + self.daemon.touch() + + async def attach(self, node: Node, channel: int, after: int) -> dict[str, Any]: + """Hand the task to ``node``'s ``channel``: the frames past the ``after``-th first, then whatever comes.""" + async with self._send_lock: + replayed = await _replay(self.record, node, channel, after) + if self.rc is None: + old = self.subscriber + if old is not None and old[0] is not node: + old[0].channels.pop(old[1], None) + self.subscriber = (node, channel) + else: + await node.notify(protocol.AGENT_EXIT, {"channel": channel, "rc": self.rc}) + return {"pid": self.proc.pid, "frames": self.frames, "replayed": replayed, "rc": self.rc} + + async def write(self, line: str) -> None: + if self.proc.stdin is None or self.proc.stdin.is_closing(): + return + self.proc.stdin.write(line.encode("utf-8") + b"\n") + try: + await self.proc.stdin.drain() + except (BrokenPipeError, ConnectionResetError): + pass + + def end(self) -> None: + """End the agent and everything it started, in its group or out of it.""" + if self.proc.returncode is not None: + return + if hasattr(os, "killpg"): + _end_tree(self.proc.pid) + else: # pragma: no cover - a node runs on POSIX + with contextlib.suppress(ProcessLookupError): + self.proc.kill() + + async def close(self) -> int | None: + """Close the agent's stdin and give it a moment to leave, then end it.""" + if self.proc.stdin is not None and not self.proc.stdin.is_closing(): + self.proc.stdin.close() + try: + return await asyncio.wait_for(self.proc.wait(), timeout=_CLOSE_GRACE_S) + except asyncio.TimeoutError: + self.end() + return await self.proc.wait() + + +def _write_json(path: Path, data: dict[str, Any]) -> None: + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(data), encoding="utf-8") + os.replace(tmp, path) + + +async def _replay(record: Path, node: Node, channel: int, after: int) -> int: + """Send ``node`` the frames of ``record`` past the ``after``-th, numbered; how many went.""" + sent = 0 + frames = record / "frames.jsonl" + if not frames.is_file(): + return 0 + with frames.open("rb") as handle: + for n, raw in enumerate(handle, 1): + if n <= after: + continue + text = raw.decode("utf-8", "replace").rstrip("\r\n") + if text: + await node.notify(protocol.AGENT_FRAME, {"channel": channel, "line": text, "n": n}) + sent += 1 + return sent + + +class Daemon: + """What outlives a connection: the tasks, the login environment, the records under ``root``. + + ``ephemeral`` is the ``--stdio`` way: one connection, whose end ends the + agents too, with records in a directory nobody comes back to. + """ + + def __init__(self, root: Path, *, ephemeral: bool = False) -> None: + self.root = root + self.ephemeral = ephemeral + self.tasks: dict[str, _Task] = {} + self.connections: set[Node] = set() + self._env: dict[str, str] | None = None + self._env_lock = asyncio.Lock() + self.last_activity = time.monotonic() + (root / TASKS_DIR).mkdir(parents=True, exist_ok=True) + + def touch(self) -> None: + self.last_activity = time.monotonic() + + async def env(self) -> dict[str, str]: + """The login environment every agent here starts from, captured at the first need.""" + async with self._env_lock: + if self._env is None: + self._env = await asyncio.to_thread(capture_login_env) + return self._env + + def running(self) -> list[_Task]: + return [task for task in self.tasks.values() if task.rc is None] + + def record_dir(self, task_id: str) -> Path: + return self.root / TASKS_DIR / task_id + + async def launch(self, node: Node, channel: int, params: dict[str, Any]) -> _Task: + task_id = _task_id(params) + known = self.tasks.get(task_id) + if known is not None and known.rc is None: + raise _RefusedError(_INVALID_PARAMS, f"task {task_id!r} is already running; attach to it") + if len(self.running()) >= protocol.MAX_AGENTS: + raise _RefusedError(protocol.BUSY, f"{protocol.MAX_AGENTS} agents are already running on this machine") + argv = _argv(params) + env = {**await self.env(), **_env(params)} + found = shutil.which(argv[0], path=env.get("PATH")) + if found is None: + raise _RefusedError(protocol.NOT_FOUND, f"{argv[0]!r} is not on this machine's PATH") + cwd = Path(_text(params, "cwd")).expanduser() + record = self.record_dir(task_id) + try: + cwd.mkdir(parents=True, exist_ok=True) + if record.exists(): + shutil.rmtree(record) + record.mkdir(parents=True) + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not make {cwd}: {exc.strerror or exc}") from None + try: + proc = await asyncio.create_subprocess_exec( + found, + *argv[1:], + cwd=str(cwd), + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + limit=_AGENT_LINE_LIMIT, + ) + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not start {argv[0]!r}: {exc.strerror or exc}") from None + task = _Task(task_id, proc, self, record, command=shlex.join(argv), cwd=str(cwd)) + task.subscriber = (node, channel) + self.tasks[task_id] = task + self.touch() + return task + + async def attach(self, node: Node, channel: int, task_id: str, after: int) -> dict[str, Any]: + """``task_id`` onto ``node``'s ``channel``: a running task is taken over, a finished one replayed.""" + task = self.tasks.get(task_id) + if task is not None: + return await task.attach(node, channel, after) + record = self.record_dir(task_id) + info = _read_record(record) + if info is None: + raise _RefusedError(protocol.UNKNOWN_TASK, f"no task {task_id!r} is running or recorded on this machine") + if info.get("state") == "running": + # The daemon that ran it is gone, and the agent with it: the + # record says so from now on, so no host reads a run that ended + # as one still going. + info["state"], info["rc"], info["ended"] = "lost", LOST_RC, info.get("ended") or time.time() + _write_json(record / "task.json", info) + replayed = await _replay(record, node, channel, after) + rc = info.get("rc") if isinstance(info.get("rc"), int) else LOST_RC + await node.notify(protocol.AGENT_EXIT, {"channel": channel, "rc": rc}) + return {"pid": info.get("pid"), "frames": info.get("frames"), "replayed": replayed, "rc": rc} + + def list_tasks(self) -> list[dict[str, Any]]: + """Every task this machine knows: the running ones, and the records of finished ones.""" + out: dict[str, dict[str, Any]] = {task_id: task.summary() for task_id, task in self.tasks.items()} + tasks_dir = self.root / TASKS_DIR + for record in sorted(tasks_dir.iterdir()) if tasks_dir.is_dir() else []: + if record.name in out or not record.is_dir(): + continue + info = _read_record(record) + if info is not None: + out[record.name] = { + "task": record.name, + "state": info.get("state"), + "rc": info.get("rc"), + "pid": info.get("pid"), + "command": info.get("command"), + "cwd": info.get("cwd"), + "started": info.get("started"), + "ended": info.get("ended"), + "frames": info.get("frames"), + "attached": False, + } + return sorted(out.values(), key=lambda row: row.get("started") or 0) + + def sweep(self) -> int: + """Remove the records of tasks that ended longer ago than :data:`protocol.TASK_KEEP_S`; how many.""" + gone = 0 + tasks_dir = self.root / TASKS_DIR + for record in list(tasks_dir.iterdir()) if tasks_dir.is_dir() else []: + if record.name in self.tasks: + continue + info = _read_record(record) + ended = info.get("ended") if info else None + if info is None or info.get("state") == "running": + continue + if isinstance(ended, (int, float)) and time.time() - ended > protocol.TASK_KEEP_S: + shutil.rmtree(record, ignore_errors=True) + gone += 1 + return gone + + async def end_all(self) -> None: + for task in self.running(): + task.end() + for task in self.running(): + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(task.proc.wait(), timeout=_END_WAIT_S) + + +def _read_record(record: Path) -> dict[str, Any] | None: + try: + data = json.loads((record / "task.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return data if isinstance(data, dict) else None + + +class Node: + """One connection to a host: its channels onto the daemon's tasks, and the one way out. + + Without a ``daemon`` it is a connection of its own, the ``--stdio`` way: + an ephemeral daemon in a temporary directory, whose agents end with it. + """ + + def __init__(self, out: BinaryIO | asyncio.StreamWriter, daemon: Daemon | None = None) -> None: + self._out = _StreamOut(out) + self.daemon = daemon or Daemon(Path(tempfile.mkdtemp(prefix="raven-node-")), ephemeral=True) + self.channels: dict[int, _Task] = {} + self.stopping = False + self.stopped = asyncio.Event() + self.daemon.connections.add(self) + self.daemon.touch() + + @property + def agents(self) -> dict[int, _Task]: + """The tasks this connection holds, by channel.""" + return self.channels + + async def send(self, message: dict[str, Any]) -> None: + line = json.dumps(message, ensure_ascii=False).encode("utf-8") + b"\n" + await self._out.write_line(line) + + async def notify(self, method: str, params: dict[str, Any]) -> None: + await self.send({"jsonrpc": "2.0", "method": method, "params": params}) + + async def env(self) -> dict[str, str]: + return await self.daemon.env() + + async def end_all(self) -> None: + await self.daemon.end_all() + + def hang_up(self) -> None: + """Close the way out, when it is a socket: the daemon is leaving and takes the connection with it.""" + out = self._out._out + if isinstance(out, asyncio.StreamWriter) and not out.is_closing(): + out.close() + + async def detach_all(self) -> None: + """This connection is over: its tasks go on, held by nobody until the next connection attaches.""" + for channel, task in list(self.channels.items()): + if task.subscriber is not None and task.subscriber[0] is self: + task.subscriber = None + self.channels.clear() + self.daemon.connections.discard(self) + self.daemon.touch() + + def _channel(self, params: dict[str, Any], *, free: bool) -> int: + channel = _number(params, "channel") + if channel is None: + raise _RefusedError(_INVALID_PARAMS, "channel must be a whole number") + if free and channel in self.channels: + raise _RefusedError(_INVALID_PARAMS, f"channel {channel} is in use") + if not free and channel not in self.channels: + raise _RefusedError(_INVALID_PARAMS, f"no agent on channel {channel}") + return channel + + def _task(self, params: dict[str, Any]) -> _Task: + return self.channels[self._channel(params, free=False)] + + async def hello(self, params: dict[str, Any]) -> dict[str, Any]: + answer = await _hello(params) + answer["mode"] = "stdio" if self.daemon.ephemeral else "daemon" + answer["tasks"] = len(self.daemon.running()) + return answer + + async def launch(self, params: dict[str, Any]) -> dict[str, Any]: + channel = self._channel(params, free=True) + task = await self.daemon.launch(self, channel, params) + self.channels[channel] = task + return {"pid": task.proc.pid, "task": task.id} + + async def attach(self, params: dict[str, Any]) -> dict[str, Any]: + channel = self._channel(params, free=True) + task_id = _task_id({"task": params.get("task", "")}) + after = _number(params, "after", 0) or 0 + if after < 0: + raise _RefusedError(_INVALID_PARAMS, "after must be a whole number") + answer = await self.daemon.attach(self, channel, task_id, after) + task = self.daemon.tasks.get(task_id) + if task is not None and task.rc is None: + self.channels[channel] = task + return answer + + async def frame(self, params: dict[str, Any]) -> None: + line = params.get("line") + channel = params.get("channel") + task = self.channels.get(channel) if isinstance(channel, int) else None + if task is not None and isinstance(line, str): + await task.write(line) + + async def cancel(self, params: dict[str, Any]) -> dict[str, Any]: + self._task(params).end() + return {} + + async def close(self, params: dict[str, Any]) -> dict[str, Any]: + return {"rc": await self._task(params).close()} + + async def tasks(self, params: dict[str, Any]) -> dict[str, Any]: + return {"tasks": self.daemon.list_tasks()} + + async def probe_agent(self, params: dict[str, Any]) -> dict[str, Any]: + """Whether an agent's command is on this machine, on the PATH an agent here gets.""" + argv = _argv(params) + path = _env(params).get("PATH") or (await self.env()).get("PATH") + return {"found": shutil.which(argv[0], path=path)} + + async def shutdown(self, params: dict[str, Any]) -> dict[str, Any]: + """The host is leaving: a stdio node ends its agents with it; a daemon's tasks go on.""" + self.stopping = True + if self.daemon.ephemeral: + await self.daemon.end_all() + return {} + + +_HANDLERS: dict[str, Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]] = { + protocol.PING: _ping, + protocol.PROBE_MACHINE: _probe_machine, + protocol.SESSION_DIR: _session_dir, + protocol.READ: _read, + protocol.LIST: _list, + protocol.FIND: _find, + protocol.GREP: _grep, + protocol.STAT: _stat, + protocol.WRITE: _write, + protocol.EDIT: _edit, + protocol.PULL: _pull, +} + + +def _node_handlers(node: Node) -> dict[str, Callable[[dict[str, Any]], Awaitable[Any]]]: + return { + protocol.HELLO: node.hello, + protocol.AGENT_LAUNCH: node.launch, + protocol.AGENT_ATTACH: node.attach, + protocol.AGENT_FRAME: node.frame, + protocol.AGENT_CANCEL: node.cancel, + protocol.AGENT_CLOSE: node.close, + protocol.TASK_LIST: node.tasks, + protocol.PROBE_AGENT: node.probe_agent, + protocol.SHUTDOWN: node.shutdown, + } + + +async def handle(frame: Any, node: Node | None = None) -> dict[str, Any] | None: + """The answer to one decoded line, or ``None`` when nothing is owed (a notification). + + ``node`` holds the methods that need the connection's state -- its + channels, the daemon behind it -- and is absent only where none of them + is asked for. + """ + if not isinstance(frame, dict) or not isinstance(frame.get("method"), str): + return None + method = frame["method"] + request_id = frame.get("id") + params = frame.get("params") + handler = _HANDLERS.get(method) or (_node_handlers(node).get(method) if node is not None else None) + if handler is None and method == protocol.HELLO: + handler = _hello + if request_id is None: + # A notification: answered by doing, never by a frame. + if handler is not None and isinstance(params, dict): + with contextlib.suppress(Exception): + await handler(params) + return None + try: + if handler is None: + raise _RefusedError(_METHOD_NOT_FOUND, f"unknown method {method!r}") + params = params or {} + if not isinstance(params, dict): + raise _RefusedError(_INVALID_PARAMS, "params must be an object") + return {"jsonrpc": "2.0", "id": request_id, "result": await handler(params)} + except _RefusedError as exc: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": exc.code, "message": exc.message}} + except Exception as exc: # noqa: BLE001 - the host gets a reason; the daemon keeps serving + message = f"{type(exc).__name__}: {exc}"[:500] + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": _INTERNAL_ERROR, "message": message}} + + +async def _serve_one(node: Node, text: bytes) -> None: + """Answer one line, on a task of its own, so a slow call holds nothing behind it.""" + try: + frame = json.loads(text) + except ValueError: + answer: dict[str, Any] | None = { + "jsonrpc": "2.0", + "id": None, + "error": {"code": _PARSE_ERROR, "message": "not a JSON object"}, + } + else: + answer = await handle(frame, node) + if answer is not None: + with contextlib.suppress(OSError, ValueError, ConnectionError): + await node.send(answer) + if node.stopping: + node.stopped.set() + + +async def serve( + reader: asyncio.StreamReader, out: BinaryIO | asyncio.StreamWriter, daemon: Daemon | None = None +) -> None: + """Answer one host until it hangs up or says shutdown; then its tasks are left running, or ended (stdio).""" + node = Node(out, daemon) + answering: set[asyncio.Task[None]] = set() + stopped = asyncio.ensure_future(node.stopped.wait()) + try: + while True: + reading = asyncio.ensure_future(reader.readline()) + done, _ = await asyncio.wait({reading, stopped}, return_when=asyncio.FIRST_COMPLETED) + if reading not in done: + reading.cancel() + return + try: + line = reading.result() + except ValueError: + # Longer than any request; the reader has dropped it and goes on. + continue + except (ConnectionError, OSError): + return + if not line: + return + text = line.strip() + if not text: + continue + task = asyncio.create_task(_serve_one(node, text)) + answering.add(task) + task.add_done_callback(answering.discard) + finally: + stopped.cancel() + if answering: + # The answers already under way get a moment to go out; a host + # that hung up is not waited for. + _, late = await asyncio.wait(answering, timeout=_ANSWER_DRAIN_S) + for task in late: + task.cancel() + if node.daemon.ephemeral: + await node.end_all() + else: + await node.detach_all() + + +def run_stdio() -> int: + """Serve on this process's stdin and stdout; return when the host hangs up, the agents ended with it.""" + out = sys.stdout.buffer + # stdout is the protocol. Anything that prints by accident -- a library, a + # stray debug line -- goes to stderr instead of corrupting a frame. + sys.stdout = sys.stderr + + async def main() -> None: + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader(limit=_LINE_LIMIT) + await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(reader), sys.stdin) + await serve(reader, out) + + asyncio.run(main()) + return 0 + + +async def _connection(daemon: Daemon, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + await serve(reader, writer, daemon) + finally: + with contextlib.suppress(OSError, ConnectionError): + writer.close() + await writer.wait_closed() + + +async def run_daemon_async(root: Path, *, idle_s: float = IDLE_EXIT_S) -> int: + """Serve on the socket under ``root`` until idle for ``idle_s`` or told to stop; 0 when another daemon already does.""" + run_dir = root / RUN_DIR + run_dir.mkdir(parents=True, exist_ok=True) + os.chmod(run_dir, 0o700) + sock = socket_path(root) + lock = (run_dir / f"{bundle.install_name()}.lock").open("w") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + lock.close() + return 0 + with contextlib.suppress(FileNotFoundError): + sock.unlink() + pid_path(root).write_text(str(os.getpid()), encoding="utf-8") + daemon = Daemon(root) + daemon.sweep() + server = await asyncio.start_unix_server(partial(_connection, daemon), path=str(sock), limit=_LINE_LIMIT) + os.chmod(sock, 0o600) + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + with contextlib.suppress(NotImplementedError, RuntimeError): + loop.add_signal_handler(sig, stop.set) + + async def _idle_watch() -> None: + while not stop.is_set(): + await asyncio.sleep(min(_IDLE_CHECK_S, max(0.1, idle_s / 2))) + quiet = not daemon.connections and not daemon.running() + if quiet and time.monotonic() - daemon.last_activity > idle_s: + stop.set() + + watcher = asyncio.create_task(_idle_watch()) + try: + await stop.wait() + finally: + watcher.cancel() + server.close() + # Told to stop (a signal) with agents running: they end with the + # daemon, as a deliberate stop should, and their exits go out on the + # connections still open before those are closed; an idle exit has + # neither. ``wait_closed`` waits for the connections, so they are hung + # up first. + await daemon.end_all() + await asyncio.sleep(0) + for node in list(daemon.connections): + node.hang_up() + with contextlib.suppress(OSError, asyncio.TimeoutError): + await asyncio.wait_for(server.wait_closed(), timeout=_ANSWER_DRAIN_S) + with contextlib.suppress(FileNotFoundError): + sock.unlink() + with contextlib.suppress(FileNotFoundError): + pid_path(root).unlink() + lock.close() + return 0 + + +def run_daemon(root: Path, *, idle_s: float = IDLE_EXIT_S) -> int: + """Run the daemon of this install under ``root`` in this process; see :func:`run_daemon_async`.""" + sys.stdout = sys.stderr + return asyncio.run(run_daemon_async(root, idle_s=idle_s)) + + +__all__ = [ + "IDLE_EXIT_S", + "LOST_RC", + "Daemon", + "Node", + "capture_login_env", + "handle", + "run_daemon", + "run_daemon_async", + "pid_path", + "run_stdio", + "serve", + "socket_path", +] diff --git a/raven/ops/connection_add.py b/raven/ops/connection_add.py index ed3aefe10..af7ba3cf0 100644 --- a/raven/ops/connection_add.py +++ b/raven/ops/connection_add.py @@ -113,19 +113,15 @@ def probe(row: dict[str, Any], *, timeout: float = 30.0, isolate_key: bool = Fal authenticated, and the registry would then hold a path that stops working the day that agent or config changes. """ - from raven.ops.transport import TIMED_OUT_RC, TransportError, make_ssh_runner, runner_from + from raven.ops.transport import TIMED_OUT_RC, TransportError, make_ssh_runner, runner_from, ssh_target where = "this computer" if transport_of(row) == LOCAL else f"{row.get('user')}@{row.get('host')}:{row.get('port')}" try: if isolate_key and transport_of(row) == SSH: - runner = make_ssh_runner( - str(row.get("host") or ""), - int(row.get("port") or 22), - os.path.expanduser(str(row.get("key") or "")), - user=str(row.get("user") or "root"), - identities_only=True, - cap_seconds=timeout, - ) + # The row read the one way every other route onto a machine reads + # it; only the key restriction is added. + host, port, key, user = ssh_target(row) + runner = make_ssh_runner(host, port, key, user=user, identities_only=True, cap_seconds=timeout) else: runner = runner_from(row, cap_seconds=timeout) code, out = runner(PROBE_SCRIPT) @@ -200,6 +196,32 @@ def write(row: dict[str, Any]) -> Path: return path +def set_fields(conn_id: str, fields: dict[str, Any]) -> Path: + """Change ``fields`` on the one machine listed as ``conn_id``, keeping every other row as it was. + + For what is learned about a machine after it was added -- where its Raven + node is kept (``node_dir``), set by ``raven ops connection install-node + --dir``. Refuses on the same grounds as :func:`write`, and when the id is + not listed exactly once. + """ + path = connections.store_path() + found = connections.read() + if found.state == UNREADABLE: + raise ValueError(f"{found.detail}\nFix or move that file before changing it.") + if found.detail: + raise ValueError( + f"{found.detail}\nChanging a machine here would rewrite the file without them. " + "Give those entries an id (or remove them) first." + ) + rows = list(found.rows) + hits = [r for r in rows if str(r.get("id") or "").strip() == conn_id] + if len(hits) != 1: + raise ValueError(f"machine {conn_id!r} is listed {len(hits)} times in {path}; it has to be listed once") + hits[0].update(fields) + path.write_text(json.dumps({"connections": rows}, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return path + + ALIAS_MARK = "# raven connection {conn_id} (managed; rewritten on every add)" diff --git a/raven/ops/remote_path.py b/raven/ops/remote_path.py new file mode 100644 index 000000000..1bdab6359 --- /dev/null +++ b/raven/ops/remote_path.py @@ -0,0 +1,474 @@ +"""A file on a registered machine, written the way scp writes one. + +scp(1) (OpenSSH 9.9p2) has two forms, and only two: + +- short, ``[user@]host:[path]``. It has no port field: everything after the + first colon is the path, so ``root@203.0.113.7:49273:/a`` is a file named + ``49273:/a``, on port 22. +- URI, ``scp://[user@]host[:port][/path]``, the one form that carries a port. + The first slash after ``host[:port]`` only separates, so + ``scp://h:49273/a`` is ``a`` under the login home and ``scp://h:49273//a`` + is ``/a``. + +A path after the colon that does not start with ``/`` is relative to the +login home on that machine, as scp reads it. + +A remote path names exactly one registered machine or none: a machine's id +names its row; an address names the row with that host and port -- the +URI's port, else 22 -- and never the one a local ssh config would choose, +so one path means one machine on every computer. A user written in the path +must be the row's. Anything else is refused before any ssh, with the way a +local file whose name holds a colon is written instead (``./a:b``). + +Shown, a remote path uses the registry's own user, host and port, in the +form scp reads back: ``user@host:/abs/path`` on port 22, else +``scp://user@host:port//abs/path`` -- the port only fits the URI form. + +Reads the registry, and for :func:`reading` and :func:`writing` the agent +entries' ``remoteCwd``; never ssh, never a node. The machine lane is debug-only +(:func:`raven.home.machine_lane_enabled`): with it off, :func:`target` names +no machine for any path, and the file tools read every path as they always +did. +""" + +from __future__ import annotations + +import posixpath +import re +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +from raven.home import machine_lane_enabled + +#: Where an agent works on its machine when its entry names nowhere else, and +#: so the one directory every registered machine reads from without asking. +#: ``~`` is the registered user's home there, expanded by the machine. +DEFAULT_ROOT = "~/raven-work" + +#: The reading tools a remote path reaches, and so the ones its permission +#: check is about. +READING_TOOLS = frozenset({"read_file", "list_dir", "find", "grep"}) + +#: The writing tools a remote path reaches. Freed by the same roots as a read, +#: asked about on their own account elsewhere. +WRITING_TOOLS = frozenset({"write_file", "edit_file"}) + +# The tools whose path names a file, so the directory a call reaches is the file's. +_FILE_TOOLS = frozenset({"read_file", "write_file", "edit_file"}) + +#: The URI form's scheme. ``sftp://`` reads the same way, and is accepted as +#: one more spelling of it. +_URI = re.compile(r"^(?:scp|sftp)://", re.IGNORECASE) +_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://") +_DRIVE = re.compile(r"^[A-Za-z]:(?:[\\/]|$)") +_PORT = re.compile(r"^[0-9]{1,5}$") + + +class RemotePathError(ValueError): + """A path that looks remote but names no registered machine. The message names no address it did not get.""" + + +class NodeCallError(RuntimeError): + """A file call a machine's Raven node could not answer (``raven.node.client.NodeError``). Safe to show. + + ``reason`` names the node's own refusal when it gave one -- ``changed`` + (a write the file's change forbade), ``corrupt``, ``not-found``, ``busy`` + -- so a caller can say the right sentence for a refusal it expects. A + name rather than the node's code, so the file tools need not import the + node's protocol: the node imports them. + """ + + def __init__(self, message: str, *, reason: str | None = None) -> None: + super().__init__(message) + self.reason = reason + + +# How a file call reaches a machine's Raven node: the process-wide pool, +# handed over at assembly (``raven.core.runtime``). Kept here, in the registry's +# package, so the file tools reach the node without importing the node package, +# which imports them. +_NODES: Callable[[], Any] | None = None + + +def use_nodes(get_pool: Callable[[], Any]) -> None: + """Hand over how this process reaches a machine's node: ``raven.node.client.get_node_pool``.""" + global _NODES + _NODES = get_pool + + +def nodes() -> Any: + """This process's node pool, or ``None`` when none was handed over (a process that reads no machine).""" + return _NODES() if _NODES is not None else None + + +@dataclass(frozen=True) +class RemotePath: + """A path on one registered machine.""" + + machine: str + """The machine's id in the registry.""" + path: str + """The path on the machine: absolute, or relative to the login home (``""`` is the home itself).""" + _row: dict[str, Any] = field(repr=False, compare=False) + + @property + def is_absolute(self) -> bool: + return self.path.startswith("/") + + def absolute(self, home: str) -> str: + """This path as the machine names it, given the login ``home`` there; ``..`` folded away. + + Only the login user's own ``~`` is expanded here; ``~other`` is left + to the machine, and stays relative to this check. + """ + return absolute(self.path, home) + + def display(self, path: str | None = None) -> str: + """``path`` (default: this one) on this machine, the way scp writes it back. + + Port 22 gives the short form; any other port the URI form, the only + one that can say it. A relative path stays relative, as scp reads it. + """ + where = self.path if path is None else path + user = str(self._row.get("user") or "root") + host = str(self._row.get("host") or "").strip() + if ":" in host: + host = f"[{host}]" + port = int(self._row.get("port") or 22) + if port == 22: + return f"{user}@{host}:{where}" + return f"scp://{user}@{host}:{port}/{where}" + + +def absolute(path: str, home: str) -> str: + """``path`` on a machine whose login home is ``home``, as an absolute path with ``..`` folded away.""" + raw = str(path or "").strip() + if raw in ("", "~"): + return posixpath.normpath(home) if home else "" + if raw.startswith("~/"): + raw = raw[2:] + elif raw.startswith("~"): + return raw + if raw.startswith("/"): + return posixpath.normpath(raw) + return posixpath.normpath(posixpath.join(home, raw)) if home else raw + + +def within(path: str, root: str) -> bool: + """Whether the absolute ``path`` is ``root`` or under it, compared as text after folding ``..``.""" + if not path.startswith("/") or not root.startswith("/"): + return False + path, root = posixpath.normpath(path), posixpath.normpath(root) + return path == root or path.startswith(root.rstrip("/") + "/") + + +# The login home on each machine, as its Raven node reported it. Learned when a +# node starts (``raven.node.client``) and kept for the life of the process, so +# a permission check can place a ``~`` path without opening ssh to ask. +_HOMES: dict[str, str] = {} + + +def remember_home(machine: str, home: str) -> None: + """Record ``home`` as the login home on ``machine``.""" + if machine and home.startswith("/"): + _HOMES[machine] = posixpath.normpath(home) + + +def known_home(machine: str) -> str: + """The login home on ``machine`` if a node has reported it in this process, else ``""``.""" + return _HOMES.get(machine, "") + + +# The digest of each remote file as it was last read in this process, by +# machine and absolute path (``raven.agent.tools.remote_files``), so a write +# that replaces the whole file can say what it expects to replace. +_READ: dict[tuple[str, str], str] = {} + + +def remember_read(machine: str, path: str, digest: str) -> None: + """Record ``digest`` as what ``path`` on ``machine`` held when it was read.""" + if machine and path.startswith("/") and digest: + _READ[(machine, path)] = digest + + +def last_read(machine: str, path: str) -> str | None: + """The digest ``path`` on ``machine`` had when this process last read it, if it did.""" + return _READ.get((machine, path)) + + +def forget_read(machine: str, path: str) -> None: + """Drop what was read of ``path`` on ``machine``: it was written, or it changed.""" + _READ.pop((machine, path), None) + + +def target(text: str) -> RemotePath | None: + """:func:`parse` against the registry as it is now; ``None`` for every path while the lane is off.""" + if not machine_lane_enabled() or not looks_remote(text): + return None + from raven.ops.connections import load + + return parse(text, load()) + + +def reading_roots(where: RemotePath) -> list[str]: + """The directories on ``where``'s machine a read needs no question for (C30). + + The default remote root, the ``remoteCwd`` of every acp agent entry that + names this machine, and the registry row's ``paths``: where this Raven's + agents work there and where the owner said its work lives. As written, + ``~`` included; :func:`absolute` places them. + """ + roots = [DEFAULT_ROOT] + try: + from raven.config.loader import load_config + + entries = load_config().subagents.agents + except Exception: # noqa: BLE001 -- a config that cannot load names no extra root + entries = [] + for entry in entries: + if str(getattr(entry, "machine", None) or "").strip() == where.machine: + roots.append(str(getattr(entry, "remote_cwd", None) or "").strip() or DEFAULT_ROOT) + paths = where._row.get("paths") + if isinstance(paths, list): + roots += [str(p) for p in paths if str(p).startswith("/")] + return list(dict.fromkeys(roots)) + + +def _placed(path: str, home: str) -> str: + """``path`` absolute when the home is known; else in its ``~/`` spelling, which compares as text.""" + if home or path.startswith("/"): + return absolute(path, home) + raw = path.strip() + if raw in ("", "~"): + return "~" + rest = raw[2:] if raw.startswith("~/") else raw + return "~/" + posixpath.normpath(rest).lstrip("/") if not rest.startswith("~") else raw + + +def _under(path: str, root: str) -> bool: + if path.startswith("/") and root.startswith("/"): + return within(path, root) + if path.startswith("~") and root.startswith("~"): + return path == root or path.startswith(root.rstrip("/") + "/") + return False + + +@dataclass(frozen=True) +class Reading: + """What one call on a remote path reaches, for its permission check.""" + + target: RemotePath + directory: str + """The directory the call reaches: the file's for a tool that names a file, else the path itself.""" + inside: bool + """Whether that is under a :func:`reading_roots` directory, so no question is asked.""" + + +def reading(tool: str, params: dict[str, Any] | None) -> Reading | None: + """What ``tool``'s reading call reaches on a registered machine, or ``None``. + + ``None`` for any other tool, a local path, and a path that names no + machine -- that call reaches nothing and the tool says why. Placed with + the login home a node has reported in this process, when one has; else + compared as written, which can only ask where it might not have. + """ + return _reach(tool, params) if tool in READING_TOOLS else None + + +def writing(tool: str, params: dict[str, Any] | None) -> Reading | None: + """What ``tool``'s writing call reaches on a registered machine, or ``None``. + + The roots that free a read free a write too: the directories this + Raven's agents work in there, and the row's own paths. Elsewhere a write + asks on its own account, apart from any read that was allowed + (``raven.permissions.builtin.session_keys``). + """ + return _reach(tool, params) if tool in WRITING_TOOLS else None + + +def _reach(tool: str, params: dict[str, Any] | None) -> Reading | None: + path = (params or {}).get("path") + if not isinstance(path, str): + return None + try: + where = target(path) + except RemotePathError: + return None + if where is None: + return None + home = known_home(where.machine) + place = _placed(where.path, home) + directory = posixpath.dirname(place) or place if tool in _FILE_TOOLS else place + inside = any(_under(place, _placed(root, home)) for root in reading_roots(where)) + return Reading(where, directory, inside) + + +def fence(tool: str, where: RemotePath, place: str, home: str) -> str: + """The one directory a node may touch for this call: the root it is under, else what was approved. + + ``place`` is the call's absolute path on the machine. Under a + :func:`reading_roots` directory the fence is that root, so a link + between two directories inside it still reads; otherwise it is the + directory the permission check asked about, and nothing beyond it. + """ + for root in reading_roots(where): + placed = absolute(root, home) + if within(place, placed): + return placed + return (posixpath.dirname(place) or "/") if tool in _FILE_TOOLS else place + + +def looks_remote(text: str) -> bool: + """Whether ``text`` is written as a remote path, before asking which machine it names. + + Local, as scp reads it: anything starting with ``/``, ``./``, ``../`` or + ``~``; a Windows drive (``C:\\`` or ``C:/``); a URL of another scheme; + and any path whose first ``/`` comes before its first ``:``. + """ + raw = str(text or "").strip() + if not raw or raw.startswith(("/", "./", "../", "~")) or _DRIVE.match(raw): + return False + if _URI.match(raw): + return True + if _SCHEME.match(raw): + return False + colon = raw.find(":") + if colon <= 0: + return False + slash = raw.find("/") + return slash == -1 or colon < slash + + +def parse(text: str, rows: list[dict[str, Any]]) -> RemotePath | None: + """The remote path ``text`` names, or ``None`` when it is a local path. + + ``rows`` is the registry (``raven.ops.connections.load()``). Raises + :class:`RemotePathError` for a path written as remote that names no + registered machine, or names one with another user or port. + """ + raw = str(text or "").strip() + if not looks_remote(raw): + return None + if _URI.match(raw): + user, host, port, path = _split_uri(raw) + else: + user, host, path = _split_short(raw) + port = None + row = _row_for(host, port, rows, raw) + registered_user = str(row.get("user") or "root") + if user is not None and user != registered_user: + raise RemotePathError( + f"{raw!r} logs in as {user!r}, and machine {_label(row)} is registered for another user; " + "write the path without the user, or with the registered one" + ) + return RemotePath(machine=str(row.get("id")).strip(), path=path, _row=dict(row)) + + +def _split_uri(raw: str) -> tuple[str | None, str, int | None, str]: + rest = _URI.sub("", raw, count=1) + authority, slash, path = rest.partition("/") + user, _, hostport = authority.rpartition("@") if "@" in authority else (None, "", authority) + host, port = _host_port(hostport, raw) + # The first slash only separated; what follows is relative to the home + # unless it starts with another slash (``scp://h//abs``). + return (user or None), host, port, (path if slash else "") + + +def _split_short(raw: str) -> tuple[str | None, str, str]: + if "@[" in raw or raw.startswith("["): + close = raw.find("]") + if close == -1 or raw[close + 1 : close + 2] != ":": + raise RemotePathError(f"{raw!r} opens an address with '[' and does not close it before the ':'") + head, path = raw[: close + 1], raw[close + 2 :] + else: + head, _, path = raw.partition(":") + user, _, host = head.rpartition("@") if "@" in head else (None, "", head) + return (user or None), host.strip("[]"), path + + +def _host_port(hostport: str, raw: str) -> tuple[str, int | None]: + if hostport.startswith("["): + close = hostport.find("]") + if close == -1: + raise RemotePathError(f"{raw!r} opens an address with '[' and does not close it") + host, tail = hostport[1:close], hostport[close + 1 :] + if not tail: + return host, None + if not tail.startswith(":"): + raise RemotePathError(f"{raw!r} has something after the address that is not a port") + port_text = tail[1:] + else: + host, sep, port_text = hostport.partition(":") + if not sep: + return host, None + if not _PORT.match(port_text) or not 0 < int(port_text) < 65536: + raise RemotePathError(f"{raw!r} names port {port_text!r}, which is not a port") + return host, int(port_text) + + +def _row_for(host: str, port: int | None, rows: list[dict[str, Any]], raw: str) -> dict[str, Any]: + wanted = host.strip() + if not wanted: + raise RemotePathError(_unknown(raw)) + by_id = [r for r in rows if str(r.get("id") or "").strip() == wanted] + if by_id: + row = by_id[0] + if port is not None and port != int(row.get("port") or 22): + raise RemotePathError( + f"{raw!r} names machine {_label(row)} on a port it is not registered on; " + "write the path without the port" + ) + return row + address = wanted.lower() + reach = 22 if port is None else port + hits = [ + r for r in rows if str(r.get("host") or "").strip().lower() == address and int(r.get("port") or 22) == reach + ] + if len(hits) == 1: + return hits[0] + if len(hits) > 1: + names = ", ".join(_label(r) for r in hits) + raise RemotePathError(f"{raw!r} matches more than one registered machine ({names}); write the machine's id") + raise RemotePathError(_unknown(raw)) + + +def _unknown(raw: str) -> str: + return ( + f"{raw!r} is written as a path on another machine, and no registered machine matches it " + "(by its id, or by its address and port: the port written after the address in scp://, else 22). " + "For a file on this computer whose name holds a colon, write it as ./name or as an absolute path" + ) + + +def _label(row: dict[str, Any]) -> str: + conn_id = str(row.get("id") or "").strip() + name = str(row.get("display_name") or "").strip() + return f"{name!r} ({conn_id})" if name and name != conn_id else repr(conn_id) + + +__all__ = [ + "DEFAULT_ROOT", + "READING_TOOLS", + "WRITING_TOOLS", + "NodeCallError", + "Reading", + "RemotePath", + "RemotePathError", + "absolute", + "fence", + "forget_read", + "known_home", + "last_read", + "looks_remote", + "nodes", + "parse", + "reading", + "reading_roots", + "remember_home", + "remember_read", + "target", + "use_nodes", + "within", + "writing", +] diff --git a/raven/ops/transport.py b/raven/ops/transport.py index 0be1cdd48..43b8468fd 100644 --- a/raven/ops/transport.py +++ b/raven/ops/transport.py @@ -222,7 +222,10 @@ def run(cmd: str) -> tuple[int, str]: host, port, key, user=user, connect_timeout=connect_timeout, identities_only=identities_only ) + [cmd] try: - proc = subprocess.run(argv, capture_output=True, text=True, timeout=cap_seconds) + # Its own stdin, never the caller's: ssh reads stdin for the far + # side, and under ``raven acp`` the caller's stdin carries ACP + # frames, which a one-shot ssh would otherwise swallow. + proc = subprocess.run(argv, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=cap_seconds) except subprocess.TimeoutExpired as exc: got = exc.stdout.decode(errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "") return TIMED_OUT_RC, got diff --git a/raven/rpc/bootstrap.py b/raven/rpc/bootstrap.py index dbc6bfe58..419f60e39 100644 --- a/raven/rpc/bootstrap.py +++ b/raven/rpc/bootstrap.py @@ -600,11 +600,15 @@ async def teardown() -> None: except Exception: logger.exception("serve: browser close failed; continuing shutdown") # ACP agents are launched with start_new_session, so they do not get the - # terminal's signals and outlive this process unless the pool is closed. + # terminal's signals and outlive this process unless the pool is closed; + # so does the ssh of a Raven node on a registered machine, whose pool is + # closed after the agents' so a closing agent still has its link. try: from raven.acp_client.pool import close_pool + from raven.node.client import close_node_pool await close_pool() + await close_node_pool() except Exception: logger.exception("serve: acp pool close failed; continuing shutdown") diff --git a/tests/test_acp_client_teardown.py b/tests/test_acp_client_teardown.py new file mode 100644 index 000000000..16cf5ef64 --- /dev/null +++ b/tests/test_acp_client_teardown.py @@ -0,0 +1,64 @@ +"""AcpClient.close ends the server wherever raven runs. + +On POSIX the whole process group goes, as it always did. Windows has neither +process groups ``killpg`` can reach nor SIGKILL, and a teardown that reached +for either raised before any of the cleanup after it ran -- the child stayed +alive, the pipes stayed open -- on every path that closes a client, the node +installer's handshake included. +""" + +from __future__ import annotations + +import os + +import pytest + +from raven.acp_client import client as acp_client + + +class _Proc: + def __init__(self, *, gone: bool = False) -> None: + self.gone = gone + self.killed = 0 + + def kill(self) -> None: + if self.gone: + raise ProcessLookupError + self.killed += 1 + + +def test_on_posix_the_process_group_is_killed(monkeypatch): + swept: list[tuple[int, int]] = [] + monkeypatch.setattr(acp_client, "_POSIX", True) + monkeypatch.setattr(os, "killpg", lambda pgid, sig: swept.append((pgid, sig)), raising=False) + proc = _Proc() + + acp_client._end(4242, proc) + + assert swept == [(4242, acp_client.signal.SIGKILL)] + assert proc.killed == 0 + + +def test_a_group_already_gone_is_not_an_error(monkeypatch): + def gone(pgid, sig): + raise ProcessLookupError + + monkeypatch.setattr(acp_client, "_POSIX", True) + monkeypatch.setattr(os, "killpg", gone, raising=False) + + acp_client._end(4242, _Proc()) + + +def test_without_process_groups_the_child_itself_is_killed(monkeypatch): + monkeypatch.setattr(acp_client, "_POSIX", False) + monkeypatch.setattr(os, "killpg", lambda pgid, sig: pytest.fail("killpg has no meaning here"), raising=False) + proc = _Proc() + + acp_client._end(4242, proc) + + assert proc.killed == 1 + acp_client._end(4242, _Proc(gone=True)) + + +def test_the_platform_is_read_once_from_what_os_offers(): + assert acp_client._POSIX is hasattr(os, "killpg") diff --git a/tests/test_acp_remote.py b/tests/test_acp_remote.py index c6a1ffe7d..5b7df595d 100644 --- a/tests/test_acp_remote.py +++ b/tests/test_acp_remote.py @@ -1,8 +1,8 @@ -"""raven.acp_client.remote: the launch line, session directory and failure sentences. +"""raven.acp_client.remote: the machine, the ssh line that carries its node, and the failure sentences. -The quoting tests run the remote command through real shells on this computer -(``sh -c`` standing in for sshd, ``$SHELL -lic`` for the machine's login shell) -with ``HOME`` pointed at a temp dir, so a developer's own rc files take no part. +The quoting tests run what is sent through real shells on this computer +(``sh -c`` standing in for sshd, each login shell for a machine user's) with +``HOME`` pointed at a temp dir, so a developer's own rc files take no part. """ from __future__ import annotations @@ -33,6 +33,12 @@ } +@pytest.fixture(autouse=True) +def _lane_on(monkeypatch: pytest.MonkeyPatch) -> None: + """These tests are about reaching a machine; the switch that allows it is tested on its own.""" + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + + def _registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rows: list | str) -> None: path = tmp_path / "connections.json" path.write_text(rows if isinstance(rows, str) else json.dumps({"connections": rows}), encoding="utf-8") @@ -43,124 +49,118 @@ def _box() -> Machine: return Machine(id="box", display_name="Lab box", _row=dict(ROW)) -# Prints what the agent would see: its argv, its cwd and the variables asked for. -_PROBE = ( - "import json, os, sys; " - "print(json.dumps({'argv': sys.argv[1:], 'cwd': os.getcwd(), " - "'env': {k: os.environ.get(k) for k in ('RAVEN_SUBAGENT', 'ODD')}}))" -) +# Prints what the far side would see: its argv and its cwd. +_PROBE = "import json, os, sys; print(json.dumps({'argv': sys.argv[1:], 'cwd': os.getcwd()}))" + + +# What is sent is run the way sshd runs it, under a POSIX shell; a host +# without one (Windows) keeps the tests of what is sent and what is said. +_posix_shell = pytest.mark.skipif(not Path("/bin/sh").exists(), reason="runs what is sent under /bin/sh") def _run_as_sshd(command: str, home: Path) -> subprocess.CompletedProcess[str]: - """What sshd does with the one argv element it is sent: ``$SHELL -c`` it.""" + """What sshd does with the one argv element it is sent: ``$SHELL -c`` it, in the user's home.""" env = {"HOME": str(home), "SHELL": "/bin/sh", "PATH": os.environ.get("PATH", "/usr/bin:/bin")} - return subprocess.run(["/bin/sh", "-c", command], capture_output=True, text=True, env=env, cwd=home, timeout=60) + return subprocess.run( + ["/bin/sh", "-c", command], + capture_output=True, + text=True, + input="", + env=env, + cwd=home, + timeout=60, + ) -# --- the launch line ------------------------------------------------------- +# --- the ssh line ------------------------------------------------------------- -def test_the_launch_line_reaches_the_machine_with_the_registry_s_options_and_stays_open(): - argv = shlex.split(remote.launch_command(_box(), "qwen --acp", root="~/raven-work")) +def test_the_ssh_line_reaches_the_machine_with_the_registry_s_options_and_stays_open(tmp_path: Path): + log = tmp_path / "box.log" + argv = shlex.split(remote.ssh_line(_box(), "node --stdio", ssh_log=log)) assert argv[0] == "ssh" assert argv[argv.index("-i") + 1] == KEY assert argv[argv.index("-p") + 1] == str(PORT) assert "BatchMode=yes" in argv assert "StrictHostKeyChecking=accept-new" in argv - # Kept open for the life of the agent: no tty, and keepalives so a machine - # that drops off the network is noticed. + # Kept open for the life of the node: no tty, and keepalives under the + # node's own heartbeat so a machine that drops off the network is noticed. assert "-T" in argv assert "ServerAliveInterval=15" in argv assert "ServerAliveCountMax=3" in argv + # ssh's own messages go to their log, not to the stderr the connection quotes. + assert argv[argv.index("-E") + 1] == str(log) dest = argv.index(f"worker@{HOST}") assert argv[dest + 1] == "--" assert len(argv) == dest + 3, "the remote command is one argv element" # Options come before the destination, where ssh reads them. - assert argv.index("-T") < dest + assert argv.index("-T") < dest and argv.index("-E") < dest -def test_ssh_s_own_messages_go_to_their_log_not_to_stderr(tmp_path: Path): - log = tmp_path / "box.log" - argv = shlex.split(remote.launch_command(_box(), "agent", ssh_log=log)) - - assert argv[argv.index("-E") + 1] == str(log) - assert argv.index("-E") < argv.index(f"worker@{HOST}") - assert "-E" not in shlex.split(remote.launch_command(_box(), "agent")) +def test_the_ssh_line_carries_no_variable_and_reads_no_profile(tmp_path: Path): + # The node captures the login environment itself, for the agents it + # starts; what carries it is one plain line any login shell reads. + line = remote.ssh_line(_box(), "~/.raven-node/x/bin/python -I -m raven.node --stdio", ssh_log=tmp_path / "l") - -def test_each_agent_gets_an_owner_only_ssh_log(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr("raven.config.paths.get_logs_dir", lambda: tmp_path / "logs") - - first = remote.ssh_log_path("claude_code@box") - second = remote.ssh_log_path("qwen@box") - - assert first != second and first.parent == second.parent == tmp_path / "logs" / "ssh" - assert first.parent.stat().st_mode & 0o777 == 0o700 - assert remote.read_ssh_log(first) == "" - first.write_text("ssh: connect to host x port 1: Operation timed out\n", encoding="utf-8") - assert "timed out" in remote.read_ssh_log(first) + sent = shlex.split(line)[-1] + assert sent == "exec /bin/sh -c 'echo; ~/.raven-node/x/bin/python -I -m raven.node --stdio'" + assert "-lic" not in line and "mkdir" not in line and "read -r" not in line -def test_the_remote_command_runs_the_agent_with_its_arguments_intact(tmp_path: Path): +@_posix_shell +def test_the_ssh_line_s_command_runs_with_its_arguments_intact_from_the_login_directory(tmp_path: Path): # The trap measured 2026-10-10: `ssh host -- bash -lic 'qwen --acp'` loses # `--acp` to the far shell's re-parse and starts qwen interactive. - agent = f"{shlex.quote(sys.executable)} -c {shlex.quote(_PROBE)} --acp 'two words'" - done = _run_as_sshd(remote.remote_command(agent, root="~/raven-work", env={}), tmp_path) + command = f"{shlex.quote(sys.executable)} -c {shlex.quote(_PROBE)} --stdio 'two words'" + sent = shlex.split(remote.ssh_line(_box(), command, ssh_log=tmp_path / "box.log"))[-1] - assert done.returncode == 0, done.stderr - seen = json.loads(done.stdout.strip().splitlines()[-1]) - assert seen["argv"] == ["--acp", "two words"] - - -def test_the_agent_starts_in_the_root_which_the_machine_s_shell_expands_and_makes(tmp_path: Path): - agent = f"{shlex.quote(sys.executable)} -c {shlex.quote(_PROBE)}" - done = _run_as_sshd(remote.remote_command(agent, root="~/raven work/agents", env={}), tmp_path) + done = _run_as_sshd(sent, tmp_path) assert done.returncode == 0, done.stderr - seen = json.loads(done.stdout.strip().splitlines()[-1]) - assert Path(seen["cwd"]).resolve() == (tmp_path / "raven work" / "agents").resolve() + lines = done.stdout.splitlines() + assert lines[0] == "", "the far side's output starts on a line of its own" + seen = json.loads(lines[-1]) + assert seen["argv"] == ["--stdio", "two words"] + assert Path(seen["cwd"]).resolve() == tmp_path.resolve(), "where the login lands; nothing is made" -def test_variables_reach_the_agent_with_their_values_unchanged(tmp_path: Path): - odd = "a b 'c' \"d\" $HOME `id` \\e" - agent = f"{shlex.quote(sys.executable)} -c {shlex.quote(_PROBE)}" - done = _run_as_sshd( - remote.remote_command(agent, root=str(tmp_path / "work"), env={"RAVEN_SUBAGENT": "1", "ODD": odd}), - tmp_path, - ) +def test_each_agent_gets_an_owner_only_ssh_log(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr("raven.config.paths.get_logs_dir", lambda: tmp_path / "logs") - assert done.returncode == 0, done.stderr - seen = json.loads(done.stdout.strip().splitlines()[-1]) - assert seen["env"] == {"RAVEN_SUBAGENT": "1", "ODD": odd} + first = remote.ssh_log_path("raven-node@box") + second = remote.ssh_log_path("raven-node@other") + assert first != second and first.parent == second.parent == tmp_path / "logs" / "ssh" + assert first.parent.stat().st_mode & 0o777 == 0o700 + assert remote.read_ssh_log(first) == "" + first.write_text("ssh: connect to host x port 1: Operation timed out\n", encoding="utf-8") + assert "timed out" in remote.read_ssh_log(first) -def test_the_agent_is_found_on_the_login_shell_s_path(tmp_path: Path): - # Measured 2026-10-10: agents installed through nvm or into ~/.local/bin - # were on the PATH of a login shell and of nothing less. - bin_dir = tmp_path / "agentbin" - bin_dir.mkdir() - agent = bin_dir / "fake-agent" - agent.write_text("#!/bin/sh\necho found-by-login-path\n", encoding="utf-8") - agent.chmod(0o755) - (tmp_path / ".profile").write_text('PATH="$HOME/agentbin:$PATH"; export PATH\n', encoding="utf-8") - done = _run_as_sshd(remote.remote_command("fake-agent", root="~/raven-work", env={}), tmp_path) +def test_an_entry_that_lost_its_address_is_refused_by_name_not_as_a_network_fault(tmp_path: Path): + # machine() refuses such a row up front; this is the guard behind it, for a + # Machine built from a row that changed after it was read. + bare = Machine(id="box", display_name="Lab box", _row={"id": "box"}) - assert done.returncode == 0, done.stderr - assert "found-by-login-path" in done.stdout + with pytest.raises(RemoteMachineError, match=r"'Lab box' \(box\): .*has no address"): + remote.ssh_line(bare, "node", ssh_log=tmp_path / "l") + with pytest.raises(RemoteMachineError, match=r"'Lab box' \(box\): .*has no address"): + remote.push(bare, "true", b"") -def test_a_variable_name_the_shell_cannot_assign_is_refused(): - with pytest.raises(ValueError, match="not an environment variable name"): - remote.remote_command("agent", root="~/raven-work", env={"A-B": "1"}) +def test_a_line_break_in_what_would_be_sent_is_refused_before_any_ssh(tmp_path: Path): + with pytest.raises(ValueError, match="must be one line"): + remote.posix("echo a\necho b") + with pytest.raises(ValueError, match="must be one line"): + remote.ssh_line(_box(), "node\n--stdio", ssh_log=tmp_path / "l") def test_an_absolute_root_is_quoted_and_a_bare_tilde_is_left_to_the_shell(): - assert remote._shell_path("/srv/raven work") == "'/srv/raven work'" - assert remote._shell_path("~") == "~" - assert remote._shell_path("~/") == "~" - assert remote._shell_path("") == "~/raven-work" + assert remote.shell_path("/srv/raven work") == "'/srv/raven work'" + assert remote.shell_path("~") == "~" + assert remote.shell_path("~/") == "~" + assert remote.shell_path("") == "~/raven-work" # --- the machine ------------------------------------------------------------ @@ -197,92 +197,57 @@ def test_a_machine_that_cannot_be_used_is_refused_with_the_reason(tmp_path, monk assert HOST not in str(caught.value) -# --- the session directory ------------------------------------------------- - - -def _local_runner(home: Path, sent: list[str]): - def runner_from(row, *, cap_seconds=None): - def run(cmd: str) -> tuple[int, str]: - sent.append(cmd) - done = _run_as_sshd(cmd, home) - return done.returncode, done.stdout + (f"\n{done.stderr}" if done.returncode else "") - - return run - - return runner_from - - -def test_the_session_directory_is_made_on_the_machine_and_comes_back_absolute(tmp_path, monkeypatch): - sent: list[str] = [] - monkeypatch.setattr(remote, "runner_from", _local_runner(tmp_path, sent)) - - got = remote.session_dir(_box(), root="~/raven-work", handle="reviewer-1") - - assert got.startswith("/") - assert Path(got).resolve() == (tmp_path / "raven-work" / "reviewer-1").resolve() - assert Path(got).is_dir() - assert len(sent) == 1, "one round trip" - - -def test_a_relative_root_comes_back_absolute_from_where_the_login_lands(tmp_path, monkeypatch): - # session/new takes an absolute cwd; a root written relative to the - # registered user's home is resolved there, not on this computer. - monkeypatch.setattr(remote, "runner_from", _local_runner(tmp_path, [])) - - got = remote.session_dir(_box(), root="work", handle="t1") - - assert got.startswith("/") - assert Path(got).resolve() == (tmp_path / "work" / "t1").resolve() - - -def test_two_handles_get_two_directories_even_when_made_safe_alike(tmp_path, monkeypatch): - monkeypatch.setattr(remote, "runner_from", _local_runner(tmp_path, [])) - - first = remote.session_dir(_box(), root="~/raven-work", handle="a b") - second = remote.session_dir(_box(), root="~/raven-work", handle="a_b") +def test_a_machine_named_only_by_its_id_is_called_that(): + assert Machine(id="box", display_name="box", _row={}).label == "'box'" - assert first != second - assert Path(first).parent == Path(second).parent +def test_a_machine_answers_for_itself_but_never_for_the_way_onto_it(): + box = Machine(id="box", display_name="Lab box", _row={**ROW, "node_dir": "/srv/raven-node", "paths": ["/data"]}) -def test_a_session_directory_that_cannot_be_made_says_where_and_not_how(tmp_path, monkeypatch): - def runner_from(row, *, cap_seconds=None): - return lambda cmd: ( - 255, - f"\nssh: connect to host {HOST} port {PORT}: Operation timed out", - ) + assert box.field("node_dir") == "/srv/raven-node" + assert box.field("paths") == ["/data"] + assert box.field("missing") is None + for way_in in ("host", "port", "user", "key"): + with pytest.raises(KeyError, match="how raven reaches the machine"): + box.field(way_in) - monkeypatch.setattr(remote, "runner_from", runner_from) - with pytest.raises(RemoteMachineError) as caught: - remote.session_dir(_box(), handle="t1") +# --- where an agent works on its machine, as written ---------------------------------------- - said = str(caught.value) - assert "did not answer within the connection timeout" in said - assert "'Lab box' (box)" in said - assert HOST not in said and str(PORT) not in said +@pytest.mark.parametrize( + ("remote_cwd", "want"), + [ + (None, "~/raven-work"), + ("", "~/raven-work"), + (" ", "~/raven-work"), + ("~/raven-work/", "~/raven-work"), + ("/srv/agents/", "/srv/agents"), + ("~", "~"), + ("/", "/"), + ], +) +def test_the_root_is_the_entry_s_or_the_default(remote_cwd, want): + assert remote.root_of(remote_cwd) == want -def test_a_directory_the_machine_will_not_make_gives_its_reason_without_ssh_s_words(tmp_path, monkeypatch): - # Measured 2026-10-10: a registered machine whose disk had filled. On a - # first connect the runner's output also carries ssh's own warning, which - # names the address. - def runner_from(row, *, cap_seconds=None): - return lambda cmd: ( - 1, - f"\nWarning: Permanently added '[{HOST}]:{PORT}' (ED25519) to the list of known hosts.\n" - "mkdir: cannot create directory '/srv/x': No space left on device", - ) - monkeypatch.setattr(remote, "runner_from", runner_from) +@pytest.mark.parametrize( + ("root", "handle", "want"), + [ + ("~/raven-work", "t1", "~/raven-work/t1"), + (None, "reviewer", "~/raven-work/reviewer"), + ("", "t1", "~/raven-work/t1"), + ("/srv/agents/", "t1", "/srv/agents/t1"), + ("~", "t1", "~/t1"), + ("/", "t1", "/t1"), + ], +) +def test_the_session_root_is_the_root_and_the_handle_as_written(root, handle, want): + assert remote.session_root(root, handle) == want - with pytest.raises(RemoteMachineError) as caught: - remote.session_dir(_box(), root="/srv/x", handle="t1") - said = str(caught.value) - assert said.startswith("could not make a working directory under '/srv/x' on machine 'Lab box' (box): ") - assert said.endswith("No space left on device") - assert HOST not in said and str(PORT) not in said +def test_the_session_root_uses_the_safe_leaf_of_a_handle(): + assert remote.session_root("~/raven-work", "a b") == f"~/raven-work/{remote.leaf('a b')}" # --- failures ---------------------------------------------------------------- @@ -316,6 +281,58 @@ def test_a_failure_past_ssh_is_not_ssh_s_to_explain(): assert remote.explain(_box(), 0, "") is None +def test_a_missing_agent_is_one_sentence_that_points_at_the_machine(): + assert remote.agent_missing(_box()) == ( + "the agent's command was not found on machine 'Lab box' (box); " + "install it there, for the user the machine is registered with" + ) + + +def _ended(rc: int | None, stderr: str = "") -> Exception: + from raven.acp_client.protocol import AcpConnectionError + + return AcpConnectionError("gone", stderr=stderr, returncode=rc) + + +@pytest.mark.parametrize( + ("rc", "log", "stderr", "said"), + [ + # ssh's own failures, in its own words. + (255, f"ssh: connect to host {HOST} port {PORT}: Connection refused\n", "", "could not be reached"), + (255, f"worker@{HOST}: Permission denied (publickey).\n", "", "refused the login"), + (255, "client_loop: send disconnect: Broken pipe\n", "", "ended before the agent answered"), + # ssh got through and said nothing: 255 is the far side's own exit. + (255, "", "FATAL: config file is unreadable\n", None), + (255, f"Warning: Permanently added '[{HOST}]:{PORT}' (ED25519) to the list of known hosts.\n", "", None), + # The far shell could not find the command, in its own words; a + # process exiting 127 for a reason of its own is left to its words. + (127, "", "bash: line 1: exec: qwen: not found\n", "was not found on machine"), + (127, "", "sh: 1: exec: /opt/agent: cannot execute\n", "was not found on machine"), + (127, "", "module loader exited 127\n", None), + # 124 is the one-shot runner's cap, which a launch does not have. + (124, "", "", None), + (1, "", "boom\n", None), + (None, "", "", None), + ], +) +def test_a_failure_is_ssh_s_only_when_ssh_said_so(tmp_path, rc, log, stderr, said): + path = tmp_path / "box.log" + path.write_text(log, encoding="utf-8") + + got = remote.failure(_box(), path, _ended(rc, stderr)) + + if said is None: + assert got is None + else: + assert got is not None and said in got + assert HOST not in got + + +def test_a_failure_with_no_log_to_read_is_judged_on_the_exit_alone(tmp_path): + assert remote.failure(_box(), tmp_path / "never-written.log", _ended(255)) is None + assert remote.failure(_box(), tmp_path / "never-written.log", _ended(124)) is None + + # --- leaf ---------------------------------------------------------------------- @@ -332,28 +349,215 @@ def test_an_unsafe_handle_becomes_one_component_that_cannot_climb(handle): assert remote.leaf(handle) == got, "the same handle always lands in the same place" -def test_a_machine_named_only_by_its_id_is_called_that(): - assert Machine(id="box", display_name="box", _row={}).label == "'box'" +# --- any login shell --------------------------------------------------------- +# +# A machine user's login shell may be tcsh or csh (HPC clusters still hand +# them out), which cannot read ``${SHELL:-/bin/sh}``; and a profile that +# prints had its words glued to the first line read. Each test runs what is +# sent the way sshd does for a user whose login shell that is. A shell this +# computer does not have is skipped: the Linux CI has no tcsh. +_SHELLS = ["/bin/sh", "/bin/dash", "/bin/bash", "/bin/zsh", "/bin/tcsh", "/bin/csh"] -def test_an_entry_that_lost_its_address_is_refused_by_name_not_as_a_network_fault(): - # machine() refuses such a row up front; this is the guard behind it, for a - # Machine built from a row that changed after it was read. - bare = Machine(id="box", display_name="Lab box", _row={"id": "box"}) - with pytest.raises(RemoteMachineError, match=r"'Lab box' \(box\): .*has no address"): - remote.launch_command(bare, "agent") - with pytest.raises(RemoteMachineError, match=r"'Lab box' \(box\): .*has no address"): - remote.session_dir(bare, handle="t1") +def _shell(path: str) -> str: + if not os.access(path, os.X_OK): + pytest.skip(f"{path} is not on this computer") + return path -def test_a_failure_whose_only_words_are_ssh_s_gives_no_reason_rather_than_the_address(tmp_path, monkeypatch): - def runner_from(row, *, cap_seconds=None): - return lambda cmd: (1, f"\nWarning: Permanently added '[{HOST}]:{PORT}' (ED25519) to the list of known hosts.") +def _as_user_of(shell: str, command: str, home: Path) -> subprocess.CompletedProcess[bytes]: + """What sshd does for a user whose login shell is ``shell``: ``shell -c`` the command, in their home.""" + return subprocess.run( + [shell, "-c", command], + capture_output=True, + input=b"", + env={"HOME": str(home), "SHELL": shell, "PATH": "/usr/bin:/bin"}, + cwd=home, + timeout=60, + ) - monkeypatch.setattr(remote, "runner_from", runner_from) + +@_posix_shell +@pytest.mark.parametrize("shell", _SHELLS) +def test_a_command_reads_the_same_under_every_login_shell(tmp_path, shell): + shell = _shell(shell) + command = "printf '%s\\n' 'a!b' \"it's\" '$HOME' && [ ! -d /nonexistent ] && echo done" + + done = _as_user_of(shell, remote.posix(command), tmp_path) + + assert done.returncode == 0, done.stderr + assert done.stdout.decode().splitlines() == ["", "a!b", "it's", "$HOME", "done"] + + +@_posix_shell +@pytest.mark.parametrize("shell", ["/bin/sh", "/bin/bash", "/bin/zsh", "/bin/tcsh", "/bin/csh"]) +def test_what_a_profile_prints_never_joins_the_first_frame(tmp_path, shell): + shell = _shell(shell) + for rc in (".profile", ".bash_profile", ".bashrc", ".zprofile", ".zshrc"): + (tmp_path / rc).write_text("printf 'Welcome to the cluster'\n", encoding="utf-8") + (tmp_path / ".cshrc").write_text("echo -n 'Welcome to the cluster'\n", encoding="utf-8") + frame = '{"jsonrpc":"2.0","id":1,"result":{}}' + + done = _as_user_of(shell, remote.posix(f"printf '%s\\n' {shlex.quote(frame)}"), tmp_path) + + assert done.returncode == 0, done.stderr + lines = done.stdout.decode().splitlines() + # bash and zsh read their rc files only for a command sshd itself hands + # them, csh for every -c; whichever printed, the frame is a line of its own. + assert frame in lines, lines + assert lines[0] in ("", "Welcome to the cluster"), lines + + +def test_what_is_sent_is_one_quoted_sh_line_with_csh_s_history_character_escaped(): + # The shape, pinned where no csh is installed to run it (the Linux CI). + assert remote.posix("echo a!b") == "exec /bin/sh -c 'echo; echo a'\\!'b'" + + +# --- the lane is debug-only ------------------------------------------------------------ + + +def test_with_the_lane_off_a_machine_is_refused_before_the_registry_is_read(tmp_path, monkeypatch): + _registry(tmp_path, monkeypatch, [ROW]) + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + monkeypatch.setattr(remote.connections, "read", lambda: pytest.fail("the registry was read")) with pytest.raises(RemoteMachineError) as caught: - remote.session_dir(_box(), root="/srv/x", handle="t1") + remote.machine("box") + + assert str(caught.value) == ( + "machine 'box': the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1 " + "to use agents and files on registered machines" + ) + + +@pytest.mark.parametrize("value", ["1", "true", "YES", " on "]) +def test_the_lane_switch_reads_the_usual_spellings(monkeypatch, value): + from raven.home import machine_lane_enabled + + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", value) + assert machine_lane_enabled() + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "0") + assert not machine_lane_enabled() + + +# --- one command, and a file sent ---------------------------------------------------- + + +def test_run_is_the_registry_s_own_runner_capped_at_the_timeout(monkeypatch): + seen: dict = {} + + def runner_from(row, *, cap_seconds): + seen.update(row=row, cap=cap_seconds) + return lambda command: (0, f"ran {command}") + + monkeypatch.setattr(remote, "runner_from", runner_from) + + assert remote.run(_box(), "echo hi", timeout=12.0) == (0, f"ran {remote.posix('echo hi')}") + assert seen == {"row": ROW, "cap": 12.0} + + +def test_run_on_a_row_the_runner_cannot_use_is_one_sentence(monkeypatch): + from raven.ops.transport import TransportError + + def runner_from(row, *, cap_seconds): + raise TransportError("no key to log in with") + + monkeypatch.setattr(remote, "runner_from", runner_from) + + with pytest.raises(RemoteMachineError, match=r"^machine 'Lab box' \(box\): no key to log in with$"): + remote.run(_box(), "true") + + +def _ssh_here(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str) -> Path: + """An ``ssh`` on PATH that runs ``body`` with the remote command as ``$cmd``.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + ssh = bin_dir / "ssh" + ssh.write_text( + f'#!/bin/sh\nprintf "%s\\n" "$@" > "{tmp_path}/argv"\nfor cmd; do :; done\n' + body, + encoding="utf-8", + ) + ssh.chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}") + return tmp_path / "argv" + + +@_posix_shell +def test_push_hands_the_data_to_the_command_on_the_machine(tmp_path, monkeypatch): + argv = _ssh_here(tmp_path, monkeypatch, 'sh -c "$cmd"\n') + + rc, out = remote.push(_box(), "wc -c && echo done", b"0123456789", timeout=30.0) + + assert (rc, out.split()) == (0, ["10", "done"]) + sent = argv.read_text(encoding="utf-8").splitlines() + from raven.ops.transport import ssh_argv + + assert sent == ssh_argv(HOST, PORT, KEY, user="worker")[1:] + [remote.posix("wc -c && echo done")] + + +@_posix_shell +def test_a_push_that_failed_keeps_what_the_machine_said(tmp_path, monkeypatch): + _ssh_here(tmp_path, monkeypatch, 'echo partial; echo "tar: Unexpected EOF in archive" >&2; exit 2\n') + + assert remote.push(_box(), "tar -xzf -", b"x") == (2, "partial\n\ntar: Unexpected EOF in archive\n") + + +@_posix_shell +def test_a_push_that_succeeded_drops_ssh_s_chatter(tmp_path, monkeypatch): + _ssh_here(tmp_path, monkeypatch, f"echo \"Warning: Permanently added '[{HOST}]:{PORT}'\" >&2; echo ok\n") + + assert remote.push(_box(), "true", b"") == (0, "ok\n") + + +@_posix_shell +def test_a_push_that_runs_out_of_time_reads_as_timed_out(tmp_path, monkeypatch): + from raven.ops.transport import TIMED_OUT_RC + + _ssh_here(tmp_path, monkeypatch, "sleep 5\n") + + assert remote.push(_box(), "true", b"", timeout=0.3) == (TIMED_OUT_RC, "") + + +def test_a_push_to_a_row_without_an_address_is_one_sentence(): + box = Machine(id="box", display_name="Lab box", _row={"id": "box"}) + + with pytest.raises(RemoteMachineError, match=r"^machine 'Lab box' \(box\): "): + remote.push(box, "true", b"") + + +def test_run_and_push_refuse_a_command_of_two_lines_before_any_ssh(monkeypatch): + monkeypatch.setattr(remote, "runner_from", lambda row, *, cap_seconds=None: pytest.fail("no ssh for this")) + monkeypatch.setattr(remote.subprocess, "run", lambda *a, **k: pytest.fail("no ssh for this")) + + for call in (lambda: remote.run(_box(), "echo a\necho b"), lambda: remote.push(_box(), "cat\ncat", b"")): + with pytest.raises(RemoteMachineError, match=r"^machine 'Lab box' \(box\): .*must be one line$"): + call() + + +# --- what the file tools read back ------------------------------------------------------ + + +def test_a_path_on_the_machine_is_shown_the_way_scp_writes_it(): + assert ( + remote.shown(_box(), "/home/worker/raven-work/t1") == f"scp://worker@{HOST}:{PORT}//home/worker/raven-work/t1" + ) + + +def test_the_files_note_names_what_came_with_the_task_by_address_and_nothing_else(): + from types import SimpleNamespace + + target = SimpleNamespace(label="'Lab box' (box)") + media = [ + SimpleNamespace(path="/Users/me/sales.csv"), + SimpleNamespace(path=""), + SimpleNamespace(path="/tmp/a b.png"), + ] + + note = remote.files_note(media, target) - assert str(caught.value) == "could not make a working directory under '/srv/x' on machine 'Lab box' (box)" + assert note.startswith("Files that came with this task, by their address on the computer that sent it.") + assert "Nothing was copied here" in note and "say so in your reply and go on" in note + assert note.endswith("- /Users/me/sales.csv\n- /tmp/a b.png") + assert "machine 'Lab box' (box)" in note + assert remote.files_note([], target) == "" and remote.files_note([SimpleNamespace(path=None)], target) == "" diff --git a/tests/test_cli_ops_connection_commands.py b/tests/test_cli_ops_connection_commands.py index a841e1428..fa453bc26 100644 --- a/tests/test_cli_ops_connection_commands.py +++ b/tests/test_cli_ops_connection_commands.py @@ -293,3 +293,262 @@ def test_a_local_machine_gets_no_alias(store, reached, home): assert out.exit_code == 0, out.output assert not (home / ".ssh" / "config").exists() + + +# --- install-node ------------------------------------------------------------------- +# +# The install itself is raven.node.install's (tests/test_node_install.py); what the +# command adds is the order of things: look, refuse or ask, install, record where, +# and only then say the node answers. + + +class _Node: + """raven.node.install and the node's answer, stood in for, with what was asked of them.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, registered: str = "") -> None: + from raven.acp_client.remote import Machine + from raven.node import install as node_install + from raven.node.client import NodeError, NodePool + + self.present = False + self.refusal: str | None = None + self.plan_error: str | None = None + self.install_error: str | None = None + self.prune_error: str | None = None + self.hello_error: str | None = None + self.rg = True + self.pruned: tuple[str, ...] = () + self.asked: list[tuple[str, object]] = [] + + def plan(conn_id: str, *, node_dir: str | None = None) -> node_install.Plan: + self.asked.append(("plan", node_dir)) + if self.plan_error: + raise node_install.InstallError(self.plan_error) + row = {"id": conn_id, "node_dir": registered} if registered else {"id": conn_id} + return node_install.Plan( + target=Machine(id=conn_id, display_name="Lab box", _row=row), + node_dir=node_dir or registered or "~/.raven-node", + install_name="0.2.4-abcdef012345", + present=self.present, + python="/usr/bin/python3.12", + uv="", + free_kb=None, + ) + + def install(p: node_install.Plan) -> node_install.Installed: + self.asked.append(("install", p.node_dir)) + if self.install_error: + raise node_install.InstallError(self.install_error) + return node_install.Installed(p.node_dir, p.install_name, 52 * 1024, rg=self.rg) + + def prune(p: node_install.Plan) -> tuple[str, ...]: + self.asked.append(("prune", p.node_dir)) + if self.prune_error: + raise node_install.InstallError(self.prune_error) + return self.pruned + + async def hello(pool: NodePool, machine_id: str) -> dict: + self.asked.append(("hello", machine_id)) + if self.hello_error: + raise NodeError(self.hello_error) + return {"python": "3.12.3", "digest": "abcdef012345"} + + monkeypatch.setattr(node_install, "plan", plan) + monkeypatch.setattr(node_install, "refusal", lambda p: self.refusal) + monkeypatch.setattr(node_install, "install", install) + monkeypatch.setattr(node_install, "prune", prune) + monkeypatch.setattr(NodePool, "hello", hello) + + def steps(self) -> list[str]: + return [step for step, _ in self.asked] + + +@pytest.fixture +def node(monkeypatch): + return _Node(monkeypatch) + + +def install_node(*args, answer: str | None = None): + return CliRunner().invoke(connection_app, ["install-node", *args], input=answer) + + +def said(result) -> str: + """The command's output with rich's line wrapping undone.""" + return " ".join(result.output.split()) + + +def test_install_node_asks_before_installing_and_then_checks_the_node_answers(node): + out = install_node("box", answer="y\n") + + assert out.exit_code == 0, out.output + assert "Install it?" in out.output + assert node.steps() == ["plan", "install", "hello"] + assert "Installed the Raven node 0.2.4-abcdef012345 in ~/.raven-node (52 MB)." in said(out) + assert "The node answers: Python 3.12.3, code abcdef012345." in said(out) + + +def test_install_node_does_nothing_the_owner_declined(node): + out = install_node("box", answer="n\n") + + assert out.exit_code == 1 + assert node.steps() == ["plan"] + + +def test_install_node_yes_installs_without_asking(node): + out = install_node("box", "--yes") + + assert out.exit_code == 0, out.output + assert "Install it?" not in out.output + assert node.steps() == ["plan", "install", "hello"] + + +def test_a_machine_that_already_has_the_node_is_left_as_it_is(node): + node.present = True + + out = install_node("box") + + assert out.exit_code == 0, out.output + assert node.steps() == ["plan", "hello"] + assert "Machine 'Lab box' (box) already has the Raven node 0.2.4-abcdef012345 in ~/.raven-node." in said(out) + + +@pytest.mark.parametrize( + ("knob", "value"), + [ + ("plan_error", "machine 'box' is not registered; add it with `raven ops connection add`"), + ("refusal", "machine 'Lab box' (box) has neither a Python 3.12 (with venv) nor uv"), + ("install_error", "installing the Raven node on machine 'Lab box' (box) failed at the packages step: no"), + ("hello_error", "the Raven node on machine 'Lab box' (box) stopped"), + ], +) +def test_install_node_stops_at_the_first_thing_that_fails_and_says_it(node, knob, value): + setattr(node, knob, value) + + out = install_node("box", "--yes") + + assert out.exit_code == 1 + assert value in said(out) + assert "The node answers" not in out.output + + +def test_a_node_without_rg_says_how_search_runs_there(node): + node.rg = False + + out = install_node("box", "--yes") + + assert out.exit_code == 0, out.output + assert "grep there runs Raven's own Python search" in said(out) + + +def test_install_node_prunes_only_when_asked(node): + node.pruned = ("0.2.3-aaaaaaaaaaaa",) + + plain = install_node("box", "--yes") + pruned = install_node("box", "--yes", "--prune") + + assert "prune" not in [step for step, _ in node.asked[:3]] + assert "Removed 1 other node(s) from ~/.raven-node: 0.2.3-aaaaaaaaaaaa." in said(pruned) + assert plain.exit_code == pruned.exit_code == 0 + + +def test_a_prune_with_nothing_to_remove_says_none(node): + node.present = True + + out = install_node("box", "--prune") + + assert "Removed 0 other node(s) from ~/.raven-node: none." in said(out) + assert node.steps() == ["plan", "prune", "hello"] + + +def test_a_prune_that_fails_stops_the_command(node): + node.prune_error = "installing the Raven node on machine 'Lab box' (box) failed at the prune step: busy" + + out = install_node("box", "--yes", "--prune") + + assert out.exit_code == 1 + assert "failed at the prune step: busy" in said(out) + + +def _box_row(store, **extra): + store.write_text( + json.dumps({"connections": [{"id": "box", "host": "203.0.113.7", "port": 22, "key": "/k", **extra}]}), + encoding="utf-8", + ) + + +def test_a_directory_chosen_with_dir_is_kept_in_the_registry(node, store): + _box_row(store, note="kept") + + out = install_node("box", "--yes", "--dir", "/data/raven-node/") + + assert out.exit_code == 0, out.output + assert node.asked[0] == ("plan", "/data/raven-node") + (row,) = json.loads(store.read_text(encoding="utf-8"))["connections"] + assert row["node_dir"] == "/data/raven-node" and row["note"] == "kept" + assert "Recorded /data/raven-node as where machine 'Lab box' (box) keeps its Raven nodes." in said(out) + + +def test_the_directory_already_on_record_is_not_written_again(monkeypatch, store): + node = _Node(monkeypatch, registered="/data/raven-node") + _box_row(store, node_dir="/data/raven-node") + before = store.read_text(encoding="utf-8") + + out = install_node("box", "--yes", "--dir", "/data/raven-node") + + assert out.exit_code == 0, out.output + assert node.steps() == ["plan", "install", "hello"] + assert "Recorded" not in out.output + assert store.read_text(encoding="utf-8") == before + + +def test_a_directory_that_cannot_be_recorded_is_said_after_the_install(node, store): + store.write_text(json.dumps({"connections": [{"id": "box", "host": "h"}, {"id": "box", "host": "h2"}]})) + + out = install_node("box", "--yes", "--dir", "/data/raven-node") + + assert out.exit_code == 1 + assert "The node is installed, but /data/raven-node could not be recorded" in said(out) + assert "listed 2 times" in said(out) + assert node.steps() == ["plan", "install"] + + +@pytest.mark.slow +def test_install_node_closes_the_live_node_it_asked(monkeypatch, tmp_path): + # The handshake at the end runs through the real pool, launch and teardown, + # over a stand-in ssh; only the install itself is stood in for. A teardown + # that raised here would leave the node running and skip the CLI's own + # report of the failure. + from pathlib import Path + + from raven.node import bundle + from raven.node import install as node_install + from tests.test_node_client import Machine + + if not Path("/bin/sh").exists(): + pytest.skip("the stand-in ssh hands the launch line to a POSIX shell") + box = Machine(tmp_path, monkeypatch) + box.node() + present = node_install.plan.__wrapped__ if hasattr(node_install.plan, "__wrapped__") else None + del present + + def plan(conn_id, *, node_dir=None): + from raven.acp_client.remote import machine + + target = machine(conn_id) + return node_install.Plan( + target=target, + node_dir="~/.raven-node", + install_name=bundle.install_name(), + present=True, + python="/usr/bin/python3", + uv="", + free_kb=None, + ) + + monkeypatch.setattr(node_install, "plan", plan) + + out = install_node("box") + + assert out.exit_code == 0, out.output + assert f"The node answers: Python {__import__('platform').python_version()}, code {bundle.digest()}." in said(out) + assert len(box.launches()) == 1 diff --git a/tests/test_node_bundle.py b/tests/test_node_bundle.py new file mode 100644 index 000000000..afa8cfe61 --- /dev/null +++ b/tests/test_node_bundle.py @@ -0,0 +1,107 @@ +"""raven.node.bundle: the code a Raven node runs, and the digest that names it. + +The digest is how a host and a node tell they hold the same code, so these pin +what it covers -- every file's name and bytes, not the bytecode each Python +writes for itself -- and that what is sent unpacks to the same digest on the +other side. +""" + +from __future__ import annotations + +import io +import tarfile +from importlib.metadata import version +from pathlib import Path + +import pytest + +from raven.node import bundle + + +def _tree(root: Path, files: dict[str, bytes]) -> Path: + for rel, data in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return root + + +@pytest.fixture +def pkg(tmp_path: Path) -> Path: + return _tree( + tmp_path / "raven", + { + "__init__.py": b"x = 1\n", + "sub/mod.py": b"y = 2\n", + "sub/data.json": b"{}\n", + "sub/__pycache__/mod.cpython-312.pyc": b"\x00bytecode", + "stale.pyc": b"\x00", + "old.pyo": b"\x00", + }, + ) + + +def test_the_files_are_the_source_in_a_fixed_order_without_bytecode(pkg): + assert [p.as_posix() for p in bundle.files(pkg)] == ["__init__.py", "sub/data.json", "sub/mod.py"] + + +def test_the_digest_follows_the_code_and_nothing_else(pkg): + first = bundle.tree_digest(pkg) + assert len(first) == 12 and first == bundle.tree_digest(pkg) + + (pkg / "sub" / "__pycache__" / "other.cpython-313.pyc").write_bytes(b"another Python's bytecode") + assert bundle.tree_digest(pkg) == first + + (pkg / "sub" / "mod.py").write_bytes(b"y = 3\n") + edited = bundle.tree_digest(pkg) + assert edited != first + + (pkg / "sub" / "mod.py").rename(pkg / "sub" / "moved.py") + assert bundle.tree_digest(pkg) != edited, "a file's name is part of the code" + + +def test_two_trees_whose_bytes_only_line_up_differently_do_not_share_a_digest(tmp_path): + # Each name and each file's bytes are closed off, so moving a boundary -- + # bytes from one file into the next, or from a name into its contents -- + # never reads as the same code. + assert bundle.tree_digest(_tree(tmp_path / "a", {"a": b"x", "b": b"y"})) != bundle.tree_digest( + _tree(tmp_path / "b", {"a": b"xb\x00y"}) + ) + assert bundle.tree_digest(_tree(tmp_path / "c", {"a": b"bc"})) != bundle.tree_digest( + _tree(tmp_path / "d", {"ab": b"c"}) + ) + + +def test_what_is_sent_unpacks_under_raven_to_the_same_digest(pkg, tmp_path): + try: + (pkg / "linked.py").symlink_to(pkg / "sub" / "mod.py") + except (OSError, NotImplementedError): + pytest.skip("making a symlink needs a privilege this account lacks") + + data = bundle.tarball(pkg) + + out = tmp_path / "unpacked" + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + names = archive.getnames() + archive.extractall(out, filter="data") + assert sorted(names) == ["raven/__init__.py", "raven/linked.py", "raven/sub/data.json", "raven/sub/mod.py"] + assert not (out / "raven" / "linked.py").is_symlink(), "the node gets the file a link points at" + assert bundle.tree_digest(out / "raven") == bundle.tree_digest(pkg) + + +def test_this_raven_is_named_by_its_version_and_the_digest_of_its_own_package(): + from raven import __version__ + + assert bundle.package_dir().joinpath("node", "bundle.py").is_file() + assert bundle.digest() == bundle.tree_digest(bundle.package_dir()) + assert bundle.install_name() == f"{__version__}-{bundle.digest()}" + names = {p.as_posix() for p in bundle.files()} + assert {"__init__.py", "node/server.py", "agent/tools/filesystem.py"} <= names + assert not any(name.endswith((".pyc", ".pyo")) or "__pycache__" in name for name in names) + + +def test_requirements_are_pinned_to_the_versions_this_raven_runs_with(): + assert bundle.pinned(("loguru", "raven-no-such-package")) == [ + f"loguru=={version('loguru')}", + "raven-no-such-package", + ] diff --git a/tests/test_node_client.py b/tests/test_node_client.py new file mode 100644 index 000000000..3c04c95df --- /dev/null +++ b/tests/test_node_client.py @@ -0,0 +1,1005 @@ +"""raven.node.client: one live link per registered machine, and the agents it carries. + +The launch tests need no second computer. A stand-in ``ssh`` on PATH does what +sshd does with the one remote-command argument it is sent -- hands it to the +user's shell, in that user's home -- so the real ssh line and the real node +run: the machine's node directory holds a ``bin/python`` that is this test +run's own interpreter. Only the network is missing. The stand-in also fails +the way ssh fails, writing its own address-bearing words to the log the ssh +line names (``-E``), which is how the tests show those words never reach a +message. + +Every failure is one sentence that names the machine and what to do, never the +address it is reached at. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import signal +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from raven.acp_client import remote +from raven.acp_client.client import AcpClient +from raven.acp_client.protocol import AcpConnectionError, AcpRemoteError, AcpTimeoutError +from raven.agent.subagent.backends import env as backend_env +from raven.agent.tools.filesystem import ReadFileTool +from raven.node import bundle, protocol +from raven.node import client as node_client +from raven.node.client import DEFAULT_NODE_DIR, ChannelEndpoint, NodeError, NodePool, _Node + +HOST = "203.0.113.7" +PORT = 58717 + +_FAKE_SSH = """#!{python} +import json, os, sys +here = os.path.dirname(os.path.abspath(__file__)) +args = sys.argv[1:] +with open(os.path.join(here, "calls"), "a") as f: + f.write(json.dumps(args) + "\\n") +mode = open(os.path.join(here, "mode")).read().strip() +home = open(os.path.join(here, "home")).read().strip() +if mode == "unreachable": + said = "ssh: connect to host {host} port {port}: Operation timed out" + log = args[args.index("-E") + 1] if "-E" in args else None + if log: + open(log, "a").write(said + "\\n") + else: + sys.stderr.write(said + "\\n") + sys.exit(255) +command = args[args.index("--") + 1] if "--" in args else args[-1] +os.chdir(home) +os.execve("/bin/sh", ["/bin/sh", "-c", command], {{"HOME": home, "SHELL": "/bin/sh", "PATH": os.environ["PATH"]}}) +""" + +# A node that misbehaves in one named way, for the handshake's refusals. +_STUB_NODE = """ +import json, sys +mode, digest, proto = sys.argv[1], sys.argv[2], int(sys.argv[3]) +if mode == "crash": + sys.stderr.write("Traceback (most recent call last):\\n File x\\nModuleNotFoundError: No module named 'loguru'\\n") + sys.exit(1) +for line in sys.stdin: + line = line.strip() + if not line: + continue + request = json.loads(line) + if mode == "silent" or (mode == "hello-only" and request["method"] != "node/hello"): + continue + answer = {"jsonrpc": "2.0", "id": request["id"]} + if mode == "refuse": + answer["error"] = {"code": -32600, "message": "this node speaks protocol 1, not 2"} + elif mode == "other-protocol": + answer["result"] = {"protocol": proto + 1, "digest": digest} + elif mode == "other-code": + answer["result"] = {"protocol": proto, "digest": "000000000000"} + elif mode == "not-an-object": + answer["result"] = ["node/hello"] + else: + answer["result"] = {"protocol": proto, "digest": digest, "python": "3.12.0", "home": "", "rg": False} + print(json.dumps(answer), flush=True) +""" + +# An agent the node starts for the tests: answers every request with its +# params, says one thing on stderr, leaves on `quit` with the code asked for, +# and on `spawn` starts a child that outlives it unless its group is ended. +_ECHO_AGENT = """ +import json, subprocess, sys +sys.stderr.write("agent: started\\n") +sys.stderr.flush() +for line in sys.stdin: + line = line.strip() + if not line: + continue + request = json.loads(line) + method = request.get("method") + if method == "quit": + sys.exit(int(request["params"]["rc"])) + if method == "spawn": + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) + print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": {"pid": child.pid}}), flush=True) + continue + print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": {"echo": request.get("params")}}), flush=True) +""" + + +class Machine: + """The stand-in machine: its home, its ssh, and what was asked of it.""" + + def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch, rows: list[dict[str, Any]] | None = None) -> None: + self.root = root + self.home = root / "box-home" + self.home.mkdir() + self.bin = root / "fakebin" + self.bin.mkdir() + ssh = self.bin / "ssh" + ssh.write_text(_FAKE_SSH.format(python=sys.executable, host=HOST, port=PORT), encoding="utf-8") + ssh.chmod(0o755) + (self.bin / "home").write_text(str(self.home), encoding="utf-8") + self.mode("ok") + self.registry = root / "connections.json" + self.rows(rows or [row()]) + monkeypatch.setenv("RAVEN_CONNECTIONS", str(self.registry)) + # The lane is debug-only; these tests are about what it does when on. + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + path = f"{self.bin}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}" + monkeypatch.setattr(backend_env, "_LOGIN_ENV", {**os.environ, "PATH": path}) + monkeypatch.setattr("raven.config.paths.get_logs_dir", lambda: root / "logs") + + def rows(self, rows: list[dict[str, Any]]) -> None: + self.registry.write_text(json.dumps({"connections": rows}), encoding="utf-8") + + def mode(self, mode: str) -> None: + (self.bin / "mode").write_text(mode, encoding="utf-8") + + def node(self, where: Path | None = None, *, stub: str | None = None) -> Path: + """Put a node of this Raven's code under ``where`` (the default node dir).""" + base = where or self.home / DEFAULT_NODE_DIR.removeprefix("~/") + python = base / bundle.install_name() / "bin" / "python" + python.parent.mkdir(parents=True) + if stub is None: + body = f'exec {sys.executable} "$@"\n' + else: + script = self.root / "stub_node.py" + script.write_text(_STUB_NODE, encoding="utf-8") + body = f"exec {sys.executable} {script} {stub} {bundle.digest()} {protocol.PROTOCOL}\n" + python.write_text("#!/bin/sh\n" + body, encoding="utf-8") + python.chmod(0o755) + return python + + def agent(self) -> str: + """The command of an agent on this machine: the echo agent above.""" + script = self.root / "echo_agent.py" + script.write_text(_ECHO_AGENT, encoding="utf-8") + return f"{sys.executable} {script}" + + def launches(self) -> list[list[str]]: + path = self.bin / "calls" + if not path.exists(): + return [] + calls = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + return [argv for argv in calls if "-T" in argv] + + +def row(**extra: Any) -> dict[str, Any]: + return { + "id": "box", + "display_name": "Lab box", + "host": HOST, + "port": PORT, + "user": "worker", + "key": "/keys/id_test", + **extra, + } + + +@pytest.fixture +def box(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Machine: + if not Path("/bin/sh").exists(): + pytest.skip("the stand-in ssh hands the ssh line to a POSIX shell") + return Machine(tmp_path, monkeypatch) + + +@pytest.fixture +async def pool(): + nodes = NodePool() + yield nodes + await nodes.close_all() + + +def _no_address(message: str) -> None: + assert HOST not in message and str(PORT) not in message and "worker@" not in message + + +async def _until(check: Any, *, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + while not check(): + assert time.monotonic() < deadline, "did not happen in time" + await asyncio.sleep(0.02) + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + try: + # Reaped by nobody yet: a zombie answers the signal but is gone. + done, _ = os.waitpid(pid, os.WNOHANG) + return done == pid + except ChildProcessError: + return False + + +# --- a node over the real ssh line ------------------------------------------------ + + +@pytest.mark.slow +async def test_one_node_answers_every_call_to_its_machine_as_the_tool_would_here(box, pool): + box.node() + work = box.home / "work" + work.mkdir() + (work / "notes.txt").write_text("".join(f"note {i}\n" for i in range(1, 41)), encoding="utf-8") + params = {"path": str(work / "notes.txt"), "offset": 10, "limit": 3, "roots": [str(work)]} + + first = await pool.call("box", protocol.READ, params) + second = await pool.call("box", protocol.READ, params) + + local = ReadFileTool(workspace=work, allowed_dirs=(work,), follow_binding=False) + assert first == second + assert first["text"] == await local.execute(str(work / "notes.txt"), offset=10, limit=3) + assert first["sha256"] == hashlib.sha256((work / "notes.txt").read_bytes()).hexdigest() + assert len(box.launches()) == 1, "one ssh serves every call" + assert pool.live() == ["box"] + hello = await pool.hello("box") + assert hello["digest"] == bundle.digest() and hello["home"] == str(box.home) + assert hello["protocol"] == protocol.PROTOCOL and hello["limits"]["agents"] == protocol.MAX_AGENTS + + +@pytest.mark.slow +async def test_the_node_is_started_from_where_the_registry_keeps_nodes_isolated_and_without_a_login_shell( + box, pool, tmp_path +): + kept = tmp_path / "nodes on a big disk" + box.rows([row(node_dir=f"{kept}/")]) + box.node(kept) + + hello = await pool.hello("box") + + assert hello["digest"] == bundle.digest() + (launch,) = box.launches() + command = launch[launch.index("--") + 1] + assert str(kept / bundle.install_name() / "bin" / "python") in command + # Isolated: the login shell's PYTHONPATH or PYTHONHOME must not reach the + # node, where a shadow package would answer with another digest. + assert " -I -m raven.node --attach --dir " in command + # One plain line any login shell reads, from the directory the login + # lands in: no profile, no directory made, nothing read from stdin ahead + # of the protocol. The node captures the login environment itself. + assert command.startswith("exec /bin/sh -c 'echo; ") + assert "mkdir" not in command and "-lic" not in command and "read -r" not in command + assert "-T" in launch and "-E" in launch and "ServerAliveInterval=15" in launch + + +@pytest.mark.slow +async def test_a_node_that_ended_is_started_again_on_the_next_call(box, pool): + box.node() + await pool.hello("box") + await pool._nodes["box"].client.close() + assert pool.live() == [] + + await pool.hello("box") + + assert len(box.launches()) == 2 + assert pool.live() == ["box"] + + +@pytest.mark.slow +async def test_a_machine_without_the_node_says_how_to_install_it(box, pool): + with pytest.raises(NodeError) as caught: + await pool.hello("box") + + assert str(caught.value) == ( + f"Raven node {bundle.install_name()} is not installed on machine 'Lab box' (box); " + "install it with `raven ops connection install-node box`" + ) + assert pool.live() == [] + assert pool.status("box").state == "error" and "is not installed" in pool.status("box").detail + + +@pytest.mark.slow +async def test_an_unreachable_machine_is_named_without_its_address(box, pool): + box.node() + box.mode("unreachable") + + with pytest.raises(NodeError) as caught: + await pool.hello("box") + + assert str(caught.value) == "machine 'Lab box' (box) did not answer within the connection timeout" + _no_address(str(caught.value)) + + +@pytest.mark.slow +@pytest.mark.parametrize("stub", ["other-code", "other-protocol", "not-an-object"]) +async def test_a_node_of_other_code_is_refused_with_the_command_that_updates_it(box, pool, stub): + box.node(stub=stub) + + with pytest.raises(NodeError) as caught: + await pool.hello("box") + + assert str(caught.value) == ( + "the Raven node on machine 'Lab box' (box) runs different code from this Raven; " + "update it with `raven ops connection install-node box`" + ) + assert pool.live() == [] + + +@pytest.mark.slow +async def test_a_node_that_refuses_the_handshake_says_why(box, pool): + box.node(stub="refuse") + + with pytest.raises(NodeError) as caught: + await pool.hello("box") + + assert str(caught.value) == ( + "the Raven node on machine 'Lab box' (box) did not complete its handshake: this node speaks protocol 1, not 2" + ) + + +@pytest.mark.slow +async def test_a_node_that_cannot_start_gives_its_own_last_word(box, pool): + box.node(stub="crash") + + with pytest.raises(NodeError) as caught: + await pool.hello("box") + + assert str(caught.value) == ( + "the Raven node on machine 'Lab box' (box) stopped: ModuleNotFoundError: No module named 'loguru'" + ) + + +@pytest.mark.slow +async def test_a_node_that_never_answers_the_handshake_is_given_up_on(box, pool, monkeypatch): + monkeypatch.setattr(node_client, "_HELLO_TIMEOUT_S", 0.5) + box.node(stub="silent") + + with pytest.raises(NodeError, match=r"^the Raven node on machine 'Lab box' \(box\) did not complete its handshake"): + await pool.hello("box") + + assert pool.live() == [] + + +@pytest.mark.slow +async def test_a_call_that_runs_out_of_time_closes_the_node(box, pool): + box.node(stub="hello-only") + + with pytest.raises(NodeError) as caught: + await pool.call("box", protocol.READ, {"path": "x", "roots": ["/"]}, timeout=1.0) + + assert str(caught.value) == "the Raven node on machine 'Lab box' (box) did not answer fs/read within 1s" + assert pool.live() == [] and "box" not in pool._nodes + assert pool.status("box").state == "error" + + +async def test_with_the_lane_off_no_node_is_started(box, pool, monkeypatch): + box.node() + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + + with pytest.raises( + NodeError, match="the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1" + ): + await pool.hello("box") + + assert box.launches() == [] and pool.live() == [] + + +async def test_a_machine_that_is_not_registered_is_refused_before_any_ssh(box, pool): + with pytest.raises(NodeError, match="machine 'elsewhere' is not registered"): + await pool.hello("elsewhere") + + assert box.launches() == [] + + +async def test_an_ssh_line_that_cannot_be_made_is_one_sentence(box, pool, monkeypatch): + def refuse(*args: Any, **kwargs: Any) -> str: + raise ValueError("a command for a registered machine must be one line") + + monkeypatch.setattr(remote, "ssh_line", refuse) + + with pytest.raises(NodeError, match=r"^machine 'Lab box' \(box\): a command .* must be one line$") as caught: + await pool.hello("box") + + _no_address(str(caught.value)) + assert box.launches() == [] + + +async def test_no_ssh_on_this_computer_is_one_sentence(box, pool, monkeypatch, tmp_path): + empty = tmp_path / "empty-bin" + empty.mkdir() + monkeypatch.setattr(backend_env, "_LOGIN_ENV", {**os.environ, "PATH": str(empty)}) + + with pytest.raises(NodeError, match=r"^could not start ssh to machine 'Lab box' \(box\): ") as caught: + await pool.hello("box") + + _no_address(str(caught.value)) + + +# --- what the link carries beside file calls ----------------------------------------- + + +@pytest.mark.slow +async def test_the_session_directory_is_made_on_the_machine_and_comes_back_absolute(box, pool): + box.node() + + made, since = await pool.session_dir("box", "~/raven work/a b-1234") + + assert made == str((box.home / "raven work" / "a b-1234").resolve()) + assert Path(made).is_dir() + assert abs(since - time.time()) < 60, "the machine's clock, as the directory was made" + assert len(box.launches()) == 1, "on the link, not an ssh of its own" + + +@pytest.mark.slow +async def test_a_session_directory_the_machine_cannot_make_is_refused_with_its_reason(box, pool): + box.node() + (box.home / "blocker").write_text("a file, not a directory\n", encoding="utf-8") + + with pytest.raises(NodeError, match=r"refused session/dir: could not make .*blocker/t1") as caught: + await pool.session_dir("box", "~/blocker/t1") + + _no_address(str(caught.value)) + assert pool.live() == ["box"], "a refusal keeps the link" + + +@pytest.mark.slow +async def test_the_machine_and_an_agent_s_command_are_probed_over_the_link(box, pool): + box.node() + + machine = await pool.probe_machine("box") + found = await pool.probe_agent("box", box.agent()) + missing = await pool.probe_agent("box", "no-such-agent-for-raven-tests --acp") + + assert machine["python"].count(".") == 2 and str(box.home) in machine["disk_free_kb"] + assert found["found"] == sys.executable and missing["found"] is None + assert len(box.launches()) == 1 + + +# --- an agent on the machine, through the node ----------------------------------------- + + +@pytest.mark.slow +async def test_an_agent_started_through_the_node_is_driven_as_a_local_one_is(box, pool): + box.node() + work = box.home / "raven-work" + + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~/raven-work", env={}) + client = AcpClient.over(endpoint, name="echo@box") + try: + assert await client.request("ping", {"x": 1}, timeout=10) == {"echo": {"x": 1}} + assert await client.request("ping", {"x": 2}, timeout=10) == {"echo": {"x": 2}} + await _until(lambda: "agent: started" in client.stderr_tail()) + assert work.is_dir(), "the agent's root is made for it" + assert isinstance(endpoint.pid, int) and endpoint.describe().startswith("task echo@box-") + assert "on machine 'Lab box' (box), pid " in endpoint.describe() + assert pool.status("box").state == "connected" and pool.status("box").said() == "connected, 1 agent running" + assert len(box.launches()) == 1, "the agent rides the node's ssh" + finally: + await client.close() + await _until(lambda: pool.status("box").state == "idle") + assert pool.status("box").said() == "connected, idle" + + +@pytest.mark.slow +async def test_the_entry_s_env_reaches_the_agent_over_the_link_and_never_an_ssh_argument(box, pool): + box.node() + probe = ( + "import json, os, sys; " + "print(json.dumps({'jsonrpc': '2.0', 'id': 1, 'result': {'key': os.environ.get('OPENAI_API_KEY'), " + "'role': os.environ.get('RAVEN_SUBAGENT')}}), flush=True); sys.stdin.read()" + ) + env = {"OPENAI_API_KEY": "sk-raven-test-not-real", "RAVEN_SUBAGENT": "1"} + + endpoint = await pool.open_channel( + "box", name="probe@box", command=f"{sys.executable} -c {json.dumps(probe)}", cwd="~", env=env + ) + client = AcpClient.over(endpoint, name="probe@box") + try: + assert await client.request("whatever", {}, timeout=10) == {"key": "sk-raven-test-not-real", "role": "1"} + finally: + await client.close() + for argv in box.launches(): + assert "sk-raven-test-not-real" not in " ".join(argv) + + +@pytest.mark.slow +async def test_an_agent_that_exits_ends_its_connection_with_its_exit_code(box, pool): + box.node() + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + client = AcpClient.over(endpoint, name="echo@box") + + with pytest.raises(AcpConnectionError, match=r"connection ended \(exit 7\)"): + await client.request("quit", {"rc": 7}, timeout=10) + + await _until(lambda: not client.alive) + assert endpoint.returncode == 7 and endpoint.ended() is None + assert "agent: started" in client.stderr_tail() + assert pool.status("box").state == "idle", "the channel is gone, the link stays" + await client.close() + + +@pytest.mark.slow +async def test_closing_the_connection_ends_the_agent_and_what_it_started(box, pool): + box.node() + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + client = AcpClient.over(endpoint, name="echo@box") + child = (await client.request("spawn", {}, timeout=10))["pid"] + agent = endpoint.pid + assert agent is not None and not _gone(agent) and not _gone(child) + + await client.close() + + await _until(lambda: _gone(agent)) + await _until(lambda: _gone(child)) + assert endpoint.returncode is not None and not client.alive + + +@pytest.mark.slow +async def test_an_agent_whose_command_is_not_on_the_machine_says_so(box, pool): + box.node() + + with pytest.raises(NodeError) as caught: + await pool.open_channel("box", name="x@box", command="no-such-agent-for-raven-tests --acp", cwd="~", env={}) + + assert str(caught.value) == ( + "the agent's command was not found on machine 'Lab box' (box); " + "install it there, for the user the machine is registered with" + ) + assert pool.live() == ["box"] and pool.status("box").state == "idle" + + +@pytest.mark.slow +async def test_an_agent_whose_root_cannot_be_made_says_so(box, pool): + box.node() + (box.home / "blocker").write_text("a file\n", encoding="utf-8") + + with pytest.raises(NodeError, match=r"could not start 'x@box': could not make .*blocker/agents") as caught: + await pool.open_channel("box", name="x@box", command=box.agent(), cwd="~/blocker/agents", env={}) + + _no_address(str(caught.value)) + + +async def test_a_machine_full_of_agents_refuses_the_next_in_one_sentence(tmp_path): + # The node's own refusal (tested with the node); here the sentence the + # host makes of its code, with the limit the handshake reported. + pool = NodePool() + busy = AcpRemoteError(protocol.AGENT_LAUNCH, protocol.BUSY, "8 agents are already running on this machine") + _live(pool, tmp_path, raises=busy) + + with pytest.raises(NodeError, match=r"^machine 'Lab box' \(box\) already runs 8 agents; wait for one to finish$"): + await pool.open_channel("box", name="b@box", command="agent", cwd="~", env={}) + + assert pool.live() == ["box"] and pool._nodes["box"].channels == {} + + +@pytest.mark.slow +async def test_a_link_that_goes_leaves_the_agent_running_and_the_turn_goes_on_once_it_is_back(box, pool): + """The ssh is the way to the machine, not the agent's life: a request made while it is down is answered after.""" + box.node() + pool._heartbeat_s = 0.1 + pool._lost_after_s = 1.0 + pool._backoff_s = (0.2,) + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + client = AcpClient.over(endpoint, name="echo@box") + assert await client.request("ping", {"x": 1}, timeout=10) == {"echo": {"x": 1}} + relay = pool._nodes["box"].client._proc.pid + daemon = pool._nodes["box"].hello["pid"] + assert relay != daemon, "the ssh's far end is the relay; the daemon is another process" + agent = endpoint.pid + + os.kill(relay, signal.SIGKILL) + await _until(lambda: endpoint.detached is not None) + + # Asked while the link is down: held here, sent once the task is attached again. + assert await client.request("ping", {"x": 2}, timeout=20) == {"echo": {"x": 2}} + assert endpoint.detached is None and endpoint.pid == agent and not _gone(agent) + assert len(box.launches()) == 2, "one ssh per link" + assert pool.status("box").state == "connected" and pool._nodes["box"].hello["pid"] == daemon + await client.close() + await _until(lambda: _gone(agent)) + + +@pytest.mark.slow +async def test_a_daemon_that_stops_takes_its_agents_and_the_turn_says_so(box, pool): + box.node() + pool._heartbeat_s = 0.1 + pool._lost_after_s = 1.0 + pool._backoff_s = (0.2,) + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + client = AcpClient.over(endpoint, name="echo@box") + assert await client.request("ping", {"x": 1}, timeout=10) == {"echo": {"x": 1}} + daemon = pool._nodes["box"].hello["pid"] + agent = endpoint.pid + + os.kill(daemon, signal.SIGTERM) + + await _until(lambda: _gone(agent)), "a daemon told to stop takes its agents with it" + with pytest.raises(AcpConnectionError) as caught: + await client.request("ping", {"x": 2}, timeout=20) + said = str(caught.value) + assert "echo@box" in said and "connection ended" in said + _no_address(said) + assert endpoint.returncode is not None and endpoint.returncode != 0 + # The machine is reachable again: the next relay started a daemon anew. + assert pool.status("box").state in ("idle", "connected") and pool._nodes["box"].hello["pid"] != daemon + await client.close() + + +@pytest.mark.slow +async def test_closing_the_link_on_purpose_leaves_the_agent_on_the_machine_for_the_next_link_to_find(box, pool): + box.node() + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + client = AcpClient.over(endpoint, name="echo@box") + assert await client.request("ping", {}, timeout=10) == {"echo": {}} + agent = endpoint.pid + + await pool.close("box") + + assert endpoint.ended() == "the link to machine 'Lab box' (box) was closed" + assert not _gone(agent), "closing the link is not cancelling the agent" + listed = await pool.tasks("box") + assert [(row["task"], row["state"], row["attached"]) for row in listed] == [(endpoint.task, "running", False)] + await pool.call("box", protocol.AGENT_ATTACH, {"channel": 9, "task": endpoint.task, "after": 1}) + await pool.call("box", protocol.AGENT_CANCEL, {"channel": 9}) + await _until(lambda: _gone(agent)) + await client.close() + + +# --- a live node's calls, with the connection stood in for ------------------------- + + +class _Client: + """A connection that answers, or fails, the way it is told to.""" + + def __init__(self, *, result: Any = None, raises: BaseException | None = None) -> None: + self.result = {"text": "ok"} if result is None else result + self.raises = raises + self.alive = True + self.asked: list[tuple[str, dict[str, Any], float]] = [] + self.said: list[tuple[str, dict[str, Any]]] = [] + + async def request(self, method: str, params: dict[str, Any], *, timeout: float) -> Any: + self.asked.append((method, params, timeout)) + if self.raises is not None and method != protocol.SHUTDOWN: + raise self.raises + return self.result + + async def notify(self, method: str, params: dict[str, Any]) -> None: + self.said.append((method, params)) + + async def close(self) -> None: + self.alive = False + + +def _box(machine_id: str = "box") -> remote.Machine: + return remote.Machine(id=machine_id, display_name="Lab box", _row=row(id=machine_id)) + + +def _live(pool: NodePool, tmp_path: Path, machine_id: str = "box", **kw: Any) -> _Client: + client = _Client(**kw) + log = tmp_path / f"{machine_id}.log" + log.write_text(f"ssh: connect to host {HOST} port {PORT}: Connection refused\n", encoding="utf-8") + hello = {"protocol": protocol.PROTOCOL, "home": "/home/worker", "version": "0.9", "limits": {"agents": 8}} + pool._nodes[machine_id] = _Node(client, _box(machine_id), log, hello, {}) # type: ignore[arg-type] + return client + + +async def test_a_call_is_sent_with_its_budget_and_its_answer_returned(tmp_path): + pool = NodePool() + client = _live(pool, tmp_path, result={"text": "line 1"}) + before = pool._nodes["box"].used + + assert await pool.call("box", protocol.READ, {"path": "a"}, timeout=7.0) == {"text": "line 1"} + + assert client.asked == [(protocol.READ, {"path": "a"}, 7.0)] + assert pool._nodes["box"].used >= before + assert await pool.home("box") == "/home/worker" + + +async def test_an_answer_that_is_not_an_object_reads_as_empty(tmp_path): + pool = NodePool() + _live(pool, tmp_path, result=["not", "an", "object"]) + + assert await pool.call("box", protocol.READ, {}) == {} + + +async def test_a_refused_call_keeps_the_node(tmp_path): + pool = NodePool() + client = _live(pool, tmp_path, raises=AcpRemoteError(protocol.READ, -32602, "roots must be a non-empty list")) + + with pytest.raises(NodeError) as caught: + await pool.call("box", protocol.READ, {}) + + assert str(caught.value) == ( + "the Raven node on machine 'Lab box' (box) refused fs/read: roots must be a non-empty list" + ) + assert caught.value.reason is None and client.alive and pool.live() == ["box"] + client.raises = AcpRemoteError(protocol.WRITE, protocol.CHANGED, "changed") + with pytest.raises(NodeError) as changed: + await pool.call("box", protocol.WRITE, {}) + assert changed.value.reason == "changed", "the node's own refusal, by name, for the file tools" + + +async def test_a_call_that_timed_out_drops_the_node(tmp_path): + pool = NodePool(backoff_s=()) + client = _live(pool, tmp_path, raises=AcpTimeoutError("no answer", method=protocol.READ)) + + with pytest.raises(NodeError, match=r"did not answer fs/read within 120s$"): + await pool.call("box", protocol.READ, {}) + + assert not client.alive and "box" not in pool._nodes + await pool.close_all() + + +@pytest.mark.parametrize( + ("returncode", "stderr", "said"), + [ + (255, "", "machine 'Lab box' (box) could not be reached"), + ( + 127, + "", + "this Raven's node is not installed on machine 'Lab box' (box); " + "install it with `raven ops connection install-node box`", + ), + ( + 126, + "sh: exec: /root/.raven-node/x/bin/python: cannot execute: No such file or directory", + "this Raven's node is not installed on machine 'Lab box' (box); " + "install it with `raven ops connection install-node box`", + ), + ( + 1, + "bash: no job control in this shell\nTraceback (most recent call last):\nMemoryError: out of memory\n", + "the Raven node on machine 'Lab box' (box) stopped: MemoryError: out of memory", + ), + (None, None, "the Raven node on machine 'Lab box' (box) stopped"), + ], +) +async def test_a_connection_that_ended_mid_call_is_one_sentence(tmp_path, returncode, stderr, said): + pool = NodePool(backoff_s=()) + client = _live(pool, tmp_path, raises=AcpConnectionError("gone", stderr=stderr, returncode=returncode)) + + with pytest.raises(NodeError) as caught: + await pool.call("box", protocol.READ, {}) + + assert str(caught.value) == said + _no_address(str(caught.value)) + assert not client.alive and "box" not in pool._nodes + await pool.close_all() + + +async def test_a_node_left_idle_is_closed_when_another_machine_is_asked_for_unless_an_agent_runs_on_it(tmp_path): + pool = NodePool(idle_close_s=60.0) + old = _live(pool, tmp_path, "old") + busy = _live(pool, tmp_path, "busy") + recent = _live(pool, tmp_path, "recent") + asked = _live(pool, tmp_path, "asked") + for machine_id in ("old", "busy", "asked"): + pool._nodes[machine_id].used = time.monotonic() - 61.0 + pool._nodes["busy"].channels[1] = ChannelEndpoint(busy, 1, target=_box("busy"), agent="a") # type: ignore[arg-type] + + await pool.call("asked", protocol.READ, {}) + + assert not old.alive and busy.alive and recent.alive and asked.alive + assert sorted(pool.live()) == ["asked", "busy", "recent"] + assert old.asked == [], "a link closed by choice is just closed; its agents stay with the machine" + + +async def test_live_lists_only_the_nodes_still_running_and_close_ends_them(tmp_path): + pool = NodePool() + one = _live(pool, tmp_path, "one") + two = _live(pool, tmp_path, "two") + two.alive = False + assert pool.live() == ["one"] + + await pool.close("one") + assert not one.alive and pool.live() == [] + + three = _live(pool, tmp_path, "three") + await pool.close_all() + assert not three.alive and pool._nodes == {} + await pool.close("never-started") + + +async def test_one_pool_serves_the_process_until_it_is_closed(tmp_path, monkeypatch): + monkeypatch.setattr(node_client, "_POOL", None) + pool = node_client.get_node_pool() + assert node_client.get_node_pool() is pool + client = _live(pool, tmp_path) + + await node_client.close_node_pool() + + assert not client.alive + assert node_client.get_node_pool() is not pool + await node_client.close_node_pool() + await node_client.close_node_pool() + + +# --- the heartbeat, with the connection stood in for ------------------------------------ + + +class _Deaf(_Client): + """A connection that answers everything but a ping.""" + + async def request(self, method: str, params: dict[str, Any], *, timeout: float) -> Any: + self.asked.append((method, params, timeout)) + if method == protocol.PING: + raise AcpTimeoutError("no answer", method=method) + return self.result + + +async def test_a_heartbeat_unanswered_loses_the_link_tells_its_agents_and_reconnects(tmp_path, monkeypatch): + pool = NodePool(heartbeat_s=0.01, lost_after_s=0.5, backoff_s=(0.01, 0.01)) + client = _Deaf() + log = tmp_path / "box.log" + log.write_text("", encoding="utf-8") + node = _Node(client, _box(), log, {"home": "/h"}, {}) # type: ignore[arg-type] + pool._nodes["box"] = node + endpoint = ChannelEndpoint(client, 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + node.channels[1] = endpoint + tried: list[str] = [] + + async def launch(machine_id: str) -> _Node: + tried.append(machine_id) + if len(tried) == 1: + raise NodeError("machine 'Lab box' (box) could not be reached") + return _Node(_Client(), _box(), log, {"home": "/h"}, {}) # type: ignore[arg-type] + + monkeypatch.setattr(pool, "_launch", launch) + node.heartbeat = asyncio.create_task(pool._heartbeat("box", node)) + + await _until(lambda: not client.alive) + assert endpoint.returncode is None and endpoint.ended() is None, "the agent is not ended: its link is" + assert endpoint.detached == "machine 'Lab box' (box) did not answer a heartbeat within 0s" + status = pool.status("box") + assert status.state == "lost" and status.agents == 1 + assert status.said() == ( + "link lost: machine 'Lab box' (box) did not answer a heartbeat within 0s; " + "1 task continues on the machine; reconnecting" + ) + await _until(lambda: len(tried) == 2 and pool.live() == ["box"] and endpoint.detached is None) + again = pool._nodes["box"].client + assert again.asked[-1][:2] == (protocol.AGENT_ATTACH, {"channel": 1, "task": endpoint.task, "after": 0}) + assert pool.status("box").state == "connected" and pool._nodes["box"].channels[1] is endpoint + await pool.close_all() + + +async def test_a_lost_link_that_does_not_come_back_keeps_its_last_sentence(tmp_path, monkeypatch): + pool = NodePool(heartbeat_s=0.01, lost_after_s=0.5, backoff_s=(0.01,)) + client = _Deaf() + node = _Node(client, _box(), tmp_path / "box.log", {}, {}) # type: ignore[arg-type] + pool._nodes["box"] = node + + async def launch(machine_id: str) -> _Node: + raise NodeError("machine 'Lab box' (box) refused the login; check the key it is registered with") + + monkeypatch.setattr(pool, "_launch", launch) + node.heartbeat = asyncio.create_task(pool._heartbeat("box", node)) + + await _until(lambda: not client.alive and "box" not in pool._reconnects) + assert pool.status("box") == node_client.MachineStatus( + "error", "machine 'Lab box' (box) refused the login; check the key it is registered with" + ) + assert pool.status("elsewhere") == node_client.MachineStatus("off") and pool.status("elsewhere").said() == "" + + +async def test_a_channel_endpoint_carries_frames_both_ways_and_ends_with_the_agent(tmp_path): + link = _Client() + endpoint = ChannelEndpoint(link, 3, target=_box(), agent="echo@box") # type: ignore[arg-type] + + await endpoint.write(b'{"jsonrpc":"2.0","id":1,"method":"x"}\n') + endpoint.feed_frame('{"jsonrpc":"2.0","id":1,"result":{}}') + endpoint.feed_stderr("agent: hello") + endpoint.exited(3) + endpoint.feed_frame("too late") + + assert link.said == [(protocol.AGENT_FRAME, {"channel": 3, "line": '{"jsonrpc":"2.0","id":1,"method":"x"}'})] + assert await endpoint.readline() == b'{"jsonrpc":"2.0","id":1,"result":{}}' + assert await endpoint.readline() == b"" + assert await endpoint.readerr() == b"agent: hello" and await endpoint.readerr() == b"" + assert endpoint.returncode == 3 and await endpoint.wait() == 3 and endpoint.ended() is None + with pytest.raises(BrokenPipeError): + await endpoint.write(b"{}\n") + await endpoint.end() + assert all(method != protocol.AGENT_CANCEL for method, *_ in link.asked), "nothing to cancel once it has exited" + + +async def test_ending_a_running_channel_is_a_cancel_on_the_link(tmp_path): + link = _Client() + endpoint = ChannelEndpoint(link, 2, target=_box(), agent="echo@box") # type: ignore[arg-type] + + await endpoint.end() + + assert link.asked == [(protocol.AGENT_CANCEL, {"channel": 2}, node_client._CANCEL_TIMEOUT_S)] + assert endpoint.returncode is None, "the exit is the node's to report" + + +# --- where a node lives ------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("kept", "want"), + [ + (None, "~/.raven-node"), + ("", "~/.raven-node"), + (" /srv/raven-node/ ", "/srv/raven-node"), + ("~/elsewhere", "~/elsewhere"), + ], +) +def test_the_node_dir_is_the_row_s_or_the_default(kept, want): + extra = {} if kept is None else {"node_dir": kept} + target = remote.Machine(id="box", display_name="Lab box", _row=row(**extra)) + + assert node_client.node_dir(target) == want + assert node_client.node_python(target, "0.1-abc") == f"{want}/0.1-abc/bin/python" + assert node_client.install_hint(target) == "raven ops connection install-node box" + + +# --- files both ways, through the node ------------------------------------------------- + + +@pytest.mark.slow +async def test_a_file_is_fetched_from_the_machine_whole_and_checked(box, pool, tmp_path): + box.node() + session = box.home / "raven-work" / "t1" + session.mkdir(parents=True) + roots = [f"{box.home}/raven-work"] + payload = os.urandom(protocol.CHUNK * 2 + 123) + (session / "report.bin").write_bytes(payload) + + info = await pool.stat("box", f"{session}/report.bin", roots=roots, hash=True) + back = tmp_path / "back" / "report.bin" + got = await pool.pull_file("box", f"{session}/report.bin", back, roots=roots) + + assert info["kind"] == "file" and info["size"] == len(payload) + assert info["sha256"] == hashlib.sha256(payload).hexdigest() + assert got == len(payload) and back.read_bytes() == payload + assert not back.with_name("report.bin.raven-part").exists() + with pytest.raises(NodeError, match="outside allowed directories"): + await pool.pull_file("box", f"{box.home}/elsewhere.bin", back, roots=roots) + assert len(box.launches()) == 1, "all of it on the one link" + + +async def test_frames_held_for_a_link_that_is_down_are_capped(tmp_path): + endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + endpoint.detach("the link went") + for n in range(node_client._PENDING_FRAMES): + await endpoint.write(f"{n}\n".encode()) + with pytest.raises(BrokenPipeError, match="down too long: 1000 frames wait"): + await endpoint.write(b"one more\n") + assert endpoint.detached == "the link went" and endpoint.returncode is None + + +async def test_a_task_the_machine_no_longer_knows_ends_its_endpoint_when_the_link_is_back(tmp_path): + pool = NodePool() + gone = AcpRemoteError(protocol.AGENT_ATTACH, protocol.UNKNOWN_TASK, "no task 'echo@box-1' is running or recorded") + client = _live(pool, tmp_path, raises=gone) + endpoint = ChannelEndpoint(client, 1, target=_box(), agent="echo@box", task="echo@box-1") # type: ignore[arg-type] + endpoint.detach("the link went") + pool._parked["box"] = [endpoint] + + await pool._reattach("box") + + assert endpoint.returncode == node_client.LOST_RC + assert endpoint.ended() == ( + "machine 'Lab box' (box) no longer runs 'echo@box' (task echo@box-1): no task 'echo@box-1' is running or recorded" + ) + assert pool._parked.get("box", []) == [] and pool._nodes["box"].channels == {} + assert client.asked[-1][1] == {"channel": 1, "task": "echo@box-1", "after": 0} + await pool.close_all() + + +async def test_an_attach_the_link_cannot_carry_parks_every_waiting_endpoint_again(tmp_path): + pool = NodePool(backoff_s=()) + client = _live(pool, tmp_path, raises=AcpTimeoutError("no answer", method=protocol.AGENT_ATTACH)) + first = ChannelEndpoint(client, 1, target=_box(), agent="a@box") # type: ignore[arg-type] + second = ChannelEndpoint(client, 2, target=_box(), agent="b@box") # type: ignore[arg-type] + for endpoint in (first, second): + endpoint.detach("the link went") + pool._parked["box"] = [first, second] + + await pool._reattach("box") + + assert first.returncode is None and second.returncode is None + assert pool._parked["box"] == [first, second], "the one tried and the one not yet reached both wait" + assert not client.alive and "box" not in pool._nodes + assert len(client.asked) == 1, "the second was not tried on a link that had just failed" + assert pool.status("box").state == "lost" and pool.status("box").agents == 2 + await pool.close_all() diff --git a/tests/test_node_install.py b/tests/test_node_install.py new file mode 100644 index 000000000..c6aa3237c --- /dev/null +++ b/tests/test_node_install.py @@ -0,0 +1,611 @@ +"""raven.node.install: putting this Raven's node on a registered machine. + +The steps are shell text run on the machine, so the parts that decide +something -- the probe, the prune -- run here under a real ``sh`` against a +stand-in tree, and one install runs for real on this computer: a virtualenv of +this interpreter, this Raven's code unpacked into it, and the node that makes +reporting this Raven's digest. Only the package index is stood in for; no test +reaches the network. + +Every failure is one sentence that names the machine, never its address, and +a step that fails leaves no half-made node behind. +""" + +from __future__ import annotations + +import dataclasses +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from raven.acp_client import remote +from raven.node import bundle, install, protocol +from raven.node.install import InstallError, Plan + +HOST = "203.0.113.7" +PORT = 58717 + +ROW = {"id": "box", "display_name": "Lab box", "host": HOST, "port": PORT, "user": "worker", "key": "/keys/id_test"} + + +@pytest.fixture +def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + path = tmp_path / "connections.json" + + def write(**extra: Any) -> None: + path.write_text(json.dumps({"connections": [{**ROW, **extra}]}), encoding="utf-8") + + write() + monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + return write + + +def _target(**extra: Any) -> remote.Machine: + return remote.Machine(id="box", display_name="Lab box", _row={**ROW, **extra}) + + +def _plan(**fields: Any) -> Plan: + base: dict[str, Any] = { + "target": _target(), + "node_dir": "~/.raven-node", + "install_name": bundle.install_name(), + "present": False, + "python": "/usr/bin/python3.12", + "uv": "", + "free_kb": 10 * 1024 * 1024, + } + return Plan(**{**base, **fields}) + + +class Machine: + """Answers each step by name, records what it was sent, and fails where told.""" + + def __init__(self, **answers: tuple[int, str]) -> None: + self.answers = answers + self.sent: list[str] = [] + self.pushed: list[bytes] = [] + self.check = json.dumps({"protocol": protocol.PROTOCOL, "digest": bundle.digest()}) + + def _answer(self, command: str) -> tuple[int, str]: + for name, cmd in install.steps(_plan()) + install.steps(_plan(python="", uv="/usr/bin/uv")): + if command == cmd and name in self.answers: + return self.answers[name] + if command.endswith("-I -m raven.node --version"): + return 0, self.check + if " du -sk " in f" {command}": + return self.answers.get("du", (0, "51234\n")) + if "echo RG=1 || echo RG=0" in command: + return 0, "RG=1\n" + return 0, "" + + def run(self, target: remote.Machine, command: str) -> tuple[int, str]: + assert target.id == "box" + self.sent.append(command) + return self._answer(command) + + def push(self, target: remote.Machine, command: str, data: bytes) -> tuple[int, str]: + self.sent.append(command) + self.pushed.append(data) + return self.answers.get("code", (0, "")) + + +@pytest.fixture +def small_tarball(monkeypatch: pytest.MonkeyPatch) -> bytes: + monkeypatch.setattr(bundle, "tarball", lambda root=None: b"TAR") + return b"TAR" + + +def test_each_step_runs_through_the_registry_s_ssh_with_the_step_budget(monkeypatch): + seen: list[tuple] = [] + monkeypatch.setattr( + remote, "run", lambda target, command, *, timeout: seen.append(("run", command, timeout)) or (0, "") + ) + monkeypatch.setattr( + remote, + "push", + lambda target, command, data, *, timeout: seen.append(("push", command, data, timeout)) or (0, ""), + ) + + assert install._run(_target(), "true") == (0, "") + assert install._push(_target(), "cat", b"x") == (0, "") + + assert seen == [("run", "true", 900.0), ("push", "cat", b"x", 900.0)] + + +# --- one look at the machine ------------------------------------------------------ + + +def test_the_probe_s_answers_become_the_plan(registry): + sent: list[str] = [] + + def run(target: remote.Machine, command: str) -> tuple[int, str]: + sent.append(command) + return ( + 0, + "bash: no job control\nPRESENT=1\nPYTHON=/usr/bin/python3.12\nUV=/root/.local/bin/uv\nFREE_KB=2097152\n", + ) + + p = install.plan("box", run=run) + + assert (p.node_dir, p.install_name, p.present, p.python, p.uv, p.free_kb) == ( + "~/.raven-node", + bundle.install_name(), + True, + "/usr/bin/python3.12", + "/root/.local/bin/uv", + 2097152, + ) + assert p.target.label == "'Lab box' (box)" + (script,) = sent + assert bundle.digest() in script and HOST not in script + + +def test_a_machine_with_nothing_found_plans_nothing_present(registry): + p = install.plan("box", run=lambda target, command: (0, "FREE_KB=\n")) + + assert (p.present, p.python, p.uv, p.free_kb, p.how) == (False, "", "", None, "") + + +@pytest.mark.parametrize( + ("registered", "asked", "want"), + [ + (None, None, "~/.raven-node"), + ("/data/raven-node/", None, "/data/raven-node"), + ("/data/raven-node", " /tmp/nodes/ ", "/tmp/nodes"), + ], +) +def test_the_node_dir_is_the_one_asked_for_else_the_registry_s(registry, registered, asked, want): + if registered is not None: + registry(node_dir=registered) + + p = install.plan("box", node_dir=asked, run=lambda target, command: (0, "")) + + assert p.node_dir == want + + +def test_a_machine_that_cannot_be_looked_at_is_named_without_its_address(registry): + def run(target: remote.Machine, command: str) -> tuple[int, str]: + return 255, f"ssh: connect to host {HOST} port {PORT}: Connection refused" + + with pytest.raises(InstallError) as caught: + install.plan("box", run=run) + + assert str(caught.value) == "machine 'Lab box' (box) could not be reached" + + +def test_a_look_that_failed_past_ssh_says_so(registry): + with pytest.raises(InstallError, match=r"^could not look at machine 'Lab box' \(box\)$"): + install.plan("box", run=lambda target, command: (2, "df: weird")) + + +def test_with_the_lane_off_a_plan_is_refused_before_any_ssh(registry, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + + with pytest.raises( + InstallError, match="the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1" + ): + install.plan("box", run=lambda target, command: pytest.fail("no ssh with the lane off")) + + +def test_a_machine_that_is_not_registered_is_refused_before_any_ssh(registry): + with pytest.raises(InstallError, match="machine 'elsewhere' is not registered"): + install.plan("elsewhere", run=lambda target, command: pytest.fail("no ssh for an unknown machine")) + + +# --- what an install would do ----------------------------------------------------- + + +def test_an_install_with_the_machine_s_own_python_is_described_before_it_runs(): + p = _plan(free_kb=2 * 1024 * 1024) + + said = p.describe() + + assert p.how == "python" and p.needed_kb == 60 * 1024 + assert said.startswith( + f"Install the Raven node {bundle.install_name()} on machine 'Lab box' (box) into ~/.raven-node/{bundle.install_name()}: " + "a virtualenv of the machine's own /usr/bin/python3.12, " + ) + for pin in bundle.pinned((*protocol.REQUIREMENTS, protocol.SEARCH_REQUIREMENT)): + assert pin in said + assert "About 60 MB; 2 GB free there. Nothing else on the machine changes." in said + assert HOST not in said + + +def test_an_install_through_uv_needs_room_for_a_python_too(): + p = _plan(python="", uv="/usr/bin/uv", free_kb=None) + + assert p.how == "uv" and p.needed_kb == 140 * 1024 + assert "a Python 3.12 made by the machine's uv, kept beside it" in p.describe() + assert p.describe().endswith("About 140 MB. Nothing else on the machine changes.") + + +def test_the_machine_s_own_python_wins_over_uv(): + assert _plan(python="/usr/bin/python3.12", uv="/usr/bin/uv").how == "python" + + +@pytest.mark.parametrize( + ("fields", "said"), + [ + ( + {"python": "", "uv": ""}, + "machine 'Lab box' (box) has neither a Python 3.12 (with venv) nor uv; " + "install one of them there, then run this again", + ), + ( + {"free_kb": 10 * 1024}, + "~/.raven-node on machine 'Lab box' (box) has 10 MB free and a node needs about 60 MB; " + "free some space or choose another directory with --dir", + ), + ( + {"python": "", "uv": "/usr/bin/uv", "free_kb": 100 * 1024}, + "~/.raven-node on machine 'Lab box' (box) has 100 MB free and a node needs about 140 MB; " + "free some space or choose another directory with --dir", + ), + ({"free_kb": 60 * 1024}, None), + ({"free_kb": None}, None), + ], +) +def test_an_install_that_cannot_go_ahead_says_why(fields, said): + assert install.refusal(_plan(**fields)) == said + + +def test_the_steps_build_a_virtualenv_then_fill_it(): + steps = install.steps(_plan(python="/opt/py 3.12/bin/python3")) + + assert [name for name, _ in steps] == ["virtualenv", "packages", "search", "code", "path", "check"] + head = "dir=~/.raven-node; id=" + bundle.install_name() + '; node="$dir/$id"; ' + assert all(command.startswith(head) for _, command in steps) + commands = dict(steps) + assert commands["virtualenv"] == head + 'mkdir -p "$dir" && \'/opt/py 3.12/bin/python3\' -m venv "$node"' + for pin in bundle.pinned(protocol.REQUIREMENTS): + assert pin in commands["packages"] + assert "--no-cache-dir" in commands["packages"] + (rg,) = bundle.pinned((protocol.SEARCH_REQUIREMENT,)) + assert commands["search"].count(rg) == 2 + assert f"--extra-index-url {protocol.PUBLIC_INDEX}" in commands["search"] + assert commands["search"].endswith(">/dev/null 2>&1 && echo RG=1 || echo RG=0") + assert commands["code"] == head + 'mkdir -p "$node/lib" && tar -xzf - -C "$node/lib"' + # ``-I``: the machine user's PYTHONPATH or PYTHONHOME reaches neither the + # check nor, with the same flag on the launch line, the node. + assert commands["check"] == head + '"$node/bin/python" -I -m raven.node --version' + + +def test_uv_makes_its_python_inside_the_node_dir(): + commands = dict(install.steps(_plan(python="", uv="/root/.local/bin/uv"))) + + assert commands["virtualenv"].endswith( + 'mkdir -p "$dir" && UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python" ' + '/root/.local/bin/uv venv --quiet --seed --python 3.12 "$node"' + ) + + +# --- carrying it out -------------------------------------------------------------- + + +def test_an_install_runs_every_step_and_measures_what_it_made(small_tarball): + machine = Machine() + + done = install.install(_plan(), run=machine.run, push=machine.push) + + assert done == install.Installed("~/.raven-node", bundle.install_name(), 51234, rg=True) + assert machine.sent[:-1] == [command for _, command in install.steps(_plan())] + assert machine.sent[-1].endswith('du -sk "$node" 2>/dev/null | cut -f1') + assert machine.pushed == [small_tarball] + assert not any('rm -rf "$node"' in command for command in machine.sent) + + +def test_an_install_without_rg_still_finishes(small_tarball): + machine = Machine(search=(0, "RG=0\n")) + + assert install.install(_plan(), run=machine.run, push=machine.push).rg is False + + +@pytest.mark.parametrize("du", [(1, ""), (0, ""), (0, "du: cannot access\n")]) +def test_a_size_that_cannot_be_read_is_left_unsaid(small_tarball, du): + machine = Machine(du=du) + + assert install.install(_plan(), run=machine.run, push=machine.push).size_kb is None + + +@pytest.mark.parametrize( + ("step", "answer", "said"), + [ + ( + "packages", + (1, "Collecting pillow\nERROR: No matching distribution found for pillow==99.0\n"), + "installing the Raven node on machine 'Lab box' (box) failed at the packages step: " + "ERROR: No matching distribution found for pillow==99.0", + ), + ( + "virtualenv", + (255, f"worker@{HOST}: Permission denied (publickey)."), + "installing the Raven node: machine 'Lab box' (box) refused the login; check the key it is registered with", + ), + ( + "virtualenv", + (127, "sh: 1: /usr/bin/python3.12: not found"), + "installing the Raven node on machine 'Lab box' (box) failed at the virtualenv step: " + "sh: 1: /usr/bin/python3.12: not found", + ), + ( + "code", + (2, "tar: Unexpected EOF in archive\n"), + "installing the Raven node on machine 'Lab box' (box) failed at the code step: " + "tar: Unexpected EOF in archive", + ), + ( + "path", + (1, ""), + "installing the Raven node on machine 'Lab box' (box) failed at the path step: exit 1", + ), + ("check", (124, ""), "installing the Raven node: machine 'Lab box' (box) did not answer in time"), + ], +) +def test_a_step_that_fails_stops_the_install_and_removes_the_half_made_node(small_tarball, step, answer, said): + machine = Machine(**{step: answer}) + + with pytest.raises(InstallError) as caught: + install.install(_plan(), run=machine.run, push=machine.push) + + assert str(caught.value) == said + assert HOST not in str(caught.value) + names = [name for name, _ in install.steps(_plan())] + ran = [name for name, command in install.steps(_plan()) if command in machine.sent] + assert ran == names[: names.index(step) + 1], "nothing runs after the step that failed" + assert machine.sent[-1].endswith('rm -rf "$node"') + + +@pytest.mark.parametrize( + ("check", "reported"), + [ + (json.dumps({"protocol": protocol.PROTOCOL, "digest": "000000000000"}), "000000000000"), + (json.dumps({"protocol": protocol.PROTOCOL + 1, "digest": "x"}), "x"), + ("Traceback (most recent call last):\n{not json", "nothing"), + ("", "nothing"), + ], +) +def test_a_node_that_does_not_report_this_raven_s_code_is_removed(small_tarball, check, reported): + machine = Machine() + machine.check = check + + with pytest.raises(InstallError) as caught: + install.install(_plan(), run=machine.run, push=machine.push) + + assert str(caught.value) == ( + "the Raven node installed on machine 'Lab box' (box) does not report this Raven's code " + f"({reported} instead of {bundle.digest()})" + ) + assert machine.sent[-1].endswith('rm -rf "$node"') + + +def test_the_last_json_line_is_the_one_checked(small_tarball): + machine = Machine() + good = json.dumps({"protocol": protocol.PROTOCOL, "digest": bundle.digest()}) + machine.check = f'{{"stale": true}}\nsome warning\n{good}\n' + + assert install.install(_plan(), run=machine.run, push=machine.push).install_name == bundle.install_name() + + +def test_an_install_that_cannot_go_ahead_sends_nothing(): + machine = Machine() + + with pytest.raises(InstallError, match="has neither a Python 3.12"): + install.install(_plan(python="", uv=""), run=machine.run, push=machine.push) + + assert machine.sent == [] + + +def test_an_install_interrupted_by_an_error_still_cleans_up(small_tarball): + machine = Machine() + + def run(target: remote.Machine, command: str) -> tuple[int, str]: + machine.sent.append(command) + if "-m venv" in command: + raise remote.RemoteMachineError("machine 'Lab box' (box): no key") + return 0, "" + + with pytest.raises(remote.RemoteMachineError): + install.install(_plan(), run=run, push=machine.push) + + assert machine.sent[-1].endswith('rm -rf "$node"') + + +def test_prune_reports_the_nodes_it_removed(): + sent: list[str] = [] + + def run(target: remote.Machine, command: str) -> tuple[int, str]: + sent.append(command) + return 0, "PRUNED=0.2.3-aaaaaaaaaaaa\n PRUNED=0.2.4-bbbbbbbbbbbb\nnoise\n" + + assert install.prune(_plan(), run=run) == ("0.2.3-aaaaaaaaaaaa", "0.2.4-bbbbbbbbbbbb") + assert sent == [install.prune_script("~/.raven-node", bundle.install_name())] + + +def test_a_prune_that_failed_says_where(): + with pytest.raises(InstallError, match=r"failed at the prune step: rm: cannot remove 'x': Busy$"): + install.prune(_plan(), run=lambda target, command: (1, "rm: cannot remove 'x': Busy")) + + +# --- the scripts, under a real shell ---------------------------------------------- + + +def _sh(command: str, home: Path, path: str) -> str: + done = subprocess.run( + ["/bin/sh", "-c", command], + capture_output=True, + text=True, + env={"HOME": str(home), "PATH": path}, + cwd=home, + timeout=60, + ) + assert done.returncode == 0, done.stderr + return done.stdout + + +def _script(path: Path, body: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("#!/bin/sh\n" + body, encoding="utf-8") + path.chmod(0o755) + return path + + +@pytest.fixture +def shell(tmp_path: Path): + """A home and a PATH holding only what the scripts call, plus stand-in Pythons.""" + if not Path("/bin/sh").exists(): + pytest.skip("the scripts run under /bin/sh") + home = tmp_path / "home" + home.mkdir() + tools = tmp_path / "bin" + tools.mkdir() + for name in ("dirname", "df", "awk", "grep"): + found = shutil.which(name) + assert found, name + (tools / name).symlink_to(found) + return home, tools + + +def _lines(out: str) -> dict[str, str]: + return dict(line.split("=", 1) for line in out.splitlines() if "=" in line) + + +@pytest.mark.slow +def test_the_probe_finds_the_node_a_new_enough_python_uv_and_the_room(shell): + home, tools = shell + _script(tools / "python3.13", "exit 1\n") # too old, or without venv + _script(tools / "python3.12", "exit 0\n") + _script(tools / "uv", "exit 0\n") + digest = bundle.digest() + _script( + home / ".raven-node" / "0.1-x" / "bin" / "python", + f"echo '{json.dumps({'protocol': protocol.PROTOCOL, 'digest': digest})}'\n", + ) + + seen = _lines(_sh(install.probe_script("~/.raven-node", "0.1-x", digest), home, str(tools))) + + assert seen["PRESENT"] == "1" + assert seen["PYTHON"] == str(tools / "python3.12") + assert seen["UV"] == str(tools / "uv") + assert seen["FREE_KB"].isdigit() + + +@pytest.mark.slow +def test_the_probe_does_not_take_a_node_of_other_code_for_this_one(shell): + home, tools = shell + _script( + home / ".raven-node" / "0.1-x" / "bin" / "python", + f"echo '{json.dumps({'protocol': protocol.PROTOCOL, 'digest': '000000000000'})}'\n", + ) + + seen = _lines(_sh(install.probe_script("~/.raven-node", "0.1-x", bundle.digest()), home, str(tools))) + + assert "PRESENT" not in seen and "PYTHON" not in seen and "UV" not in seen + assert seen["FREE_KB"].isdigit(), "room is measured on the nearest directory that exists" + + +@pytest.mark.slow +def test_the_probe_finds_a_uv_the_non_login_path_misses(shell): + home, tools = shell + _script(home / ".local" / "bin" / "uv", "exit 0\n") + + seen = _lines(_sh(install.probe_script("/no/such/dir/nodes", "0.1-x", "d"), home, str(tools))) + + assert seen["UV"] == str(home / ".local" / "bin" / "uv") + assert seen["FREE_KB"].isdigit() + + +@pytest.mark.slow +@pytest.mark.parametrize(("pip_rc", "said", "tries"), [(0, "RG=1", 1), (1, "RG=0", 2)]) +def test_the_search_package_never_fails_the_install(shell, tmp_path, pip_rc, said, tries): + home, tools = shell + calls = tmp_path / "pip-calls" + nodes = tmp_path / "nodes" + _script(nodes / "0.1-x" / "bin" / "python", f'echo "$@" >> "{calls}"; exit {pip_rc}\n') + + search = dict(install.steps(_plan(node_dir=str(nodes), install_name="0.1-x")))["search"] + + assert _sh(search, home, str(tools)).strip() == said + asked = calls.read_text(encoding="utf-8").splitlines() + assert len(asked) == tries + assert "--extra-index-url" not in asked[0], "the machine's own index first" + if tries == 2: + assert f"--extra-index-url {protocol.PUBLIC_INDEX}" in asked[1] + + +@pytest.mark.slow +def test_prune_removes_only_other_nodes(shell, tmp_path): + home, tools = shell + nodes = tmp_path / "nodes dir" + for name in ("0.2.4-current", "0.2.3-old", "0.2.4-older"): + _script(nodes / name / "bin" / "python", "exit 0\n") + (nodes / name / "lib" / "raven").mkdir(parents=True) + _script(nodes / "notes" / "bin" / "python", "exit 0\n") # no lib/raven: not a node + (nodes / ".uv-python" / "cpython-3.12").mkdir(parents=True) + (nodes / "README").write_text("the owner's own file\n", encoding="utf-8") + + out = _sh(install.prune_script(str(nodes), "0.2.4-current"), home, str(tools) + ":/bin:/usr/bin") + + assert sorted(out.splitlines()) == ["PRUNED=0.2.3-old", "PRUNED=0.2.4-older"] + assert sorted(p.name for p in nodes.iterdir()) == [".uv-python", "0.2.4-current", "README", "notes"] + + +@pytest.mark.slow +def test_prune_of_a_node_dir_that_does_not_exist_removes_nothing(shell, tmp_path): + home, tools = shell + + assert _sh(install.prune_script(str(tmp_path / "absent"), "0.1-x"), home, str(tools) + ":/bin:/usr/bin") == "" + + +# --- one real install, on this computer ------------------------------------------- + + +@pytest.mark.slow +def test_a_real_install_makes_a_node_that_reports_this_raven_s_code(registry, tmp_path): + if not Path("/bin/sh").exists(): + pytest.skip("the install steps run under /bin/sh") + home = tmp_path / "home" + home.mkdir() + env = {"HOME": str(home), "PATH": "/usr/bin:/bin"} + + def run(target: remote.Machine, command: str) -> tuple[int, str]: + if "pip install" in command: + # The package index is the one thing not reached from a test. + return 0, "RG=0\n" if "RG=1" in command else "" + done = subprocess.run(["/bin/sh", "-c", command], capture_output=True, text=True, env=env, cwd=home) + return done.returncode, done.stdout + (f"\n{done.stderr}" if done.returncode else "") + + def push(target: remote.Machine, command: str, data: bytes) -> tuple[int, str]: + done = subprocess.run(["/bin/sh", "-c", command], input=data, capture_output=True, env=env, cwd=home) + return done.returncode, done.stdout.decode() + (f"\n{done.stderr.decode()}" if done.returncode else "") + + nodes = tmp_path / "nodes" + _script(nodes / "0.0.1-old" / "bin" / "python", "exit 0\n") + (nodes / "0.0.1-old" / "lib" / "raven").mkdir(parents=True) + + first = install.plan("box", node_dir=str(nodes), run=run) + assert not first.present + p = dataclasses.replace(first, python=sys.executable, uv="") + done = install.install(p, run=run, push=push) + + assert done.install_name == bundle.install_name() and done.rg is False and (done.size_kb or 0) > 1000 + node = nodes / bundle.install_name() + reported = subprocess.run( + [str(node / "bin" / "python"), "-m", "raven.node", "--version"], + capture_output=True, + text=True, + env=env, + cwd=home, + check=True, + ).stdout + assert json.loads(reported) == {"protocol": protocol.PROTOCOL, "digest": bundle.digest()} + assert bundle.tree_digest(node / "lib" / "raven") == bundle.digest() + assert install.plan("box", node_dir=str(nodes), run=run).present, "the next look finds it" + + assert install.prune(p, run=run) == ("0.0.1-old",) + assert sorted(path.name for path in nodes.iterdir()) == [bundle.install_name()] diff --git a/tests/test_node_relay.py b/tests/test_node_relay.py new file mode 100644 index 000000000..a2a65de6f --- /dev/null +++ b/tests/test_node_relay.py @@ -0,0 +1,94 @@ +"""The relay: the host's ssh runs it on the machine, and it bridges the session to the daemon, starting one when none answers.""" + +from __future__ import annotations + +import asyncio +import contextlib +import io +import json +import os +import signal +import time +from pathlib import Path + +import pytest + +from raven.node import __main__ as node_main +from raven.node import protocol, relay, server + + +async def _until(check, *, timeout: float = 15.0) -> None: + deadline = time.monotonic() + timeout + while not check(): + assert time.monotonic() < deadline, "did not happen in time" + await asyncio.sleep(0.05) + + +def _alive(pid: int) -> bool: + """Whether the daemon still runs: reaped here when it was this process's child, so a zombie reads as gone.""" + try: + done, _ = os.waitpid(pid, os.WNOHANG) + if done == pid: + return False + except ChildProcessError: + pass + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + +@pytest.mark.slow +async def test_the_relay_starts_a_daemon_when_none_answers_and_bridges_bytes_both_ways(tmp_path, monkeypatch): + monkeypatch.setenv("RAVEN_NODE_IDLE_S", "1") + root = tmp_path / "n" + + reader, writer = await relay.connect(root, timeout_s=60) + + pid = int(server.pid_path(root).read_text(encoding="utf-8")) + try: + stdin = asyncio.StreamReader() + out = io.BytesIO() + hello = {"jsonrpc": "2.0", "id": 1, "method": protocol.HELLO, "params": {"protocol": protocol.PROTOCOL}} + stdin.feed_data(json.dumps(hello).encode() + b"\n") + bridged = asyncio.create_task(relay.bridge(reader, writer, stdin, out)) + await _until(lambda: b"\n" in out.getvalue()) + stdin.feed_eof() + await asyncio.wait_for(bridged, timeout=10) + answer = json.loads(out.getvalue().splitlines()[0]) + assert answer["result"]["mode"] == "daemon" and answer["result"]["pid"] == pid + assert (root / server.LOG_DIR / "serve.log").exists() + + started: list[int] = [] + monkeypatch.setattr(relay, "start_daemon", lambda *a, **k: started.append(1)) + again_reader, again_writer = await relay.connect(root) + again_writer.close() + with contextlib.suppress(OSError): + await again_writer.wait_closed() + assert started == [], "a daemon that answers is not started again" + await _until(lambda: not _alive(pid), timeout=30) + finally: + if _alive(pid): + os.kill(pid, signal.SIGTERM) + + +async def test_the_relay_gives_up_when_no_daemon_comes_up(tmp_path, monkeypatch): + monkeypatch.setattr(relay, "start_daemon", lambda *a, **k: None) + + with pytest.raises(ConnectionError, match="did not answer within"): + await relay.connect(tmp_path / "n", timeout_s=0.5) + + +def test_attach_and_serve_are_run_with_the_directory_named(tmp_path, monkeypatch): + seen: list[tuple[str, Path, float | None]] = [] + monkeypatch.setattr("raven.node.relay.run_attach", lambda root: seen.append(("attach", root, None)) or 7) + monkeypatch.setattr("raven.node.server.run_daemon", lambda root, idle_s: seen.append(("serve", root, idle_s)) or 0) + + assert node_main.main(["--attach", "--dir", str(tmp_path / "n")]) == 7 + assert node_main.main(["--serve", "--dir", str(tmp_path / "n"), "--idle", "3"]) == 0 + assert node_main.main(["--serve", "--dir", "~/elsewhere"]) == 0 + + assert seen[0] == ("attach", tmp_path / "n", None) + assert seen[1] == ("serve", tmp_path / "n", 3.0) + assert seen[2] == ("serve", Path("~/elsewhere").expanduser(), server.IDLE_EXIT_S) diff --git a/tests/test_node_server.py b/tests/test_node_server.py new file mode 100644 index 000000000..cabe357f1 --- /dev/null +++ b/tests/test_node_server.py @@ -0,0 +1,1104 @@ +"""raven.node.server: Raven's own file tools, answered over stdio. + +The property everything else rests on is parity: a file call a node answers +comes back exactly as the same tool's answer on the host's own disk, footer +and limits included. Then the fence: a call reaches only the roots it names, +and one that names none is refused rather than read as "no fence". +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import io +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from raven.agent.tools.file_search import FindTool, GrepTool +from raven.agent.tools.filesystem import ListDirTool, ReadFileTool +from raven.node import __main__ as node_main +from raven.node import bundle, protocol, server + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + root = tmp_path / "work" + (root / "sub").mkdir(parents=True) + (root / "big.txt").write_text("".join(f"line {i} of the file\n" for i in range(1, 501)), encoding="utf-8") + (root / "sub" / "a.py").write_text("needle = 1\nhay = 2\n", encoding="utf-8") + (root / "b.md").write_text("hay\n", encoding="utf-8") + (tmp_path / "secret.txt").write_text("top secret\n", encoding="utf-8") + return root + + +def _local(cls: Any, root: Path) -> Any: + """The tool as the host would run it on its own disk, fenced to ``root``.""" + return cls(workspace=Path.home(), allowed_dirs=(root,), follow_binding=False) + + +async def _ask(method: str, params: Any = None, *, request_id: Any = 1) -> dict[str, Any] | None: + frame: dict[str, Any] = {"jsonrpc": "2.0", "id": request_id, "method": method} + if params is not None: + frame["params"] = params + return await server.handle(frame) + + +async def _result(method: str, params: dict[str, Any]) -> dict[str, Any]: + answer = await _ask(method, params) + assert answer is not None and "result" in answer, answer + return answer["result"] + + +async def _refusal(method: str, params: Any) -> tuple[int, str]: + answer = await _ask(method, params) + assert answer is not None and "error" in answer, answer + return answer["error"]["code"], answer["error"]["message"] + + +# --- parity ------------------------------------------------------------------- + + +async def test_a_read_through_the_node_is_the_read_the_host_would_have_made(tree): + path = str(tree / "big.txt") + + got = await _ask(protocol.READ, {"path": path, "offset": 200, "limit": 5, "roots": [str(tree)]}) + + want = await _local(ReadFileTool, tree).execute(path, offset=200, limit=5) + digest = hashlib.sha256((tree / "big.txt").read_bytes()).hexdigest() + assert got == {"jsonrpc": "2.0", "id": 1, "result": {"text": want, "sha256": digest}} + assert "line 200 of the file" in want and "line 199 of the file" not in want + + +async def test_a_read_with_no_offset_starts_at_the_first_line(tree): + path = str(tree / "b.md") + + got = await _result(protocol.READ, {"path": path, "roots": [str(tree)]}) + + assert got == { + "text": await _local(ReadFileTool, tree).execute(path), + "sha256": hashlib.sha256(b"hay\n").hexdigest(), + } + + +async def test_list_find_and_grep_answer_as_their_tools_do(tree): + roots = [str(tree)] + + listed = await _result(protocol.LIST, {"path": str(tree), "recursive": True, "max_entries": 10, "roots": roots}) + found = await _result(protocol.FIND, {"pattern": "*.py", "path": str(tree), "limit": 5, "roots": roots}) + grepped = await _result( + protocol.GREP, + { + "pattern": "HAY", + "path": str(tree), + "glob": "*.py", + "output_mode": "content", + "case_insensitive": True, + "context": 1, + "limit": 5, + "roots": roots, + }, + ) + + assert listed == {"text": await _local(ListDirTool, tree).execute(str(tree), recursive=True, max_entries=10)} + assert found == {"text": await _local(FindTool, tree).execute("*.py", path=str(tree), limit=5)} + assert grepped == { + "text": await _local(GrepTool, tree).execute( + "HAY", path=str(tree), glob="*.py", output_mode="content", case_insensitive=True, context=1, limit=5 + ) + } + assert "a.py" in found["text"] and "hay = 2" in grepped["text"] and "b.md" not in grepped["text"] + + +async def test_an_image_comes_back_with_its_blocks(tree): + from PIL import Image + + picture = tree / "dot.png" + Image.new("RGB", (2, 2), "red").save(picture) + + got = await _result(protocol.READ, {"path": str(picture), "roots": [str(tree)]}) + + want = await _local(ReadFileTool, tree).execute(str(picture)) + assert got == { + "text": want.model_text, + "display": want.display_text, + "ok": True, + "blocks": want.blocks, + "sha256": hashlib.sha256(picture.read_bytes()).hexdigest(), + } + assert [block["type"] for block in got["blocks"]] == ["text", "image_url"] + + +async def test_a_relative_path_and_a_tilde_root_mean_the_home_directory(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + (home / "notes.txt").write_text("from home\n", encoding="utf-8") + monkeypatch.setenv("HOME", str(home)) + + got = await _result(protocol.READ, {"path": "notes.txt", "roots": ["~"]}) + + assert "from home" in got["text"] + + +# --- the fence ---------------------------------------------------------------- + + +async def test_a_file_outside_the_roots_is_not_read(tree): + outside = str(tree.parent / "secret.txt") + + got = await _result(protocol.READ, {"path": outside, "roots": [str(tree)]}) + + assert "top secret" not in got["text"] + assert got == {"text": await _local(ReadFileTool, tree).execute(outside)}, "a refusal names no digest" + + +async def test_a_link_inside_the_roots_does_not_lead_out_of_them(tree): + (tree / "link.txt").symlink_to(tree.parent / "secret.txt") + + got = await _result(protocol.READ, {"path": str(tree / "link.txt"), "roots": [str(tree)]}) + + assert "top secret" not in got["text"] + + +@pytest.mark.parametrize( + ("roots", "said"), + [ + (None, "roots must be a non-empty list of directories"), + ([], "roots must be a non-empty list of directories"), + ("/", "roots must be a non-empty list of directories"), + ([7], "roots must be a non-empty list of directories"), + ([" "], "roots must be a non-empty list of directories"), + (["relative/dir"], "root 'relative/dir' is not an absolute directory"), + ], +) +async def test_a_call_that_names_no_usable_roots_is_refused(tree, roots, said): + params: dict[str, Any] = {"path": str(tree / "b.md")} + if roots is not None: + params["roots"] = roots + + assert await _refusal(protocol.READ, params) == (-32602, said) + + +# --- what a request may carry --------------------------------------------------- + + +@pytest.mark.parametrize( + ("method", "params", "said"), + [ + (protocol.READ, {"path": ""}, "path must be a non-empty string"), + (protocol.READ, {"path": 3}, "path must be a non-empty string"), + (protocol.READ, {"path": "b.md", "offset": True}, "offset must be a whole number"), + (protocol.READ, {"path": "b.md", "limit": "5"}, "limit must be a whole number"), + (protocol.LIST, {"path": ".", "recursive": "yes"}, "recursive must be true or false"), + (protocol.FIND, {"pattern": ""}, "pattern must be a non-empty string"), + (protocol.GREP, {"pattern": "x", "glob": 3}, "glob must be a string"), + (protocol.GREP, {"pattern": "x", "output_mode": 3}, "output_mode must be a string"), + (protocol.GREP, {"pattern": "x", "context": 1.5}, "context must be a whole number"), + ], +) +async def test_a_malformed_field_is_refused_by_name(tree, method, params, said): + assert await _refusal(method, {**params, "roots": [str(tree)]}) == (-32602, said) + + +async def test_params_that_are_not_an_object_are_refused(): + assert await _refusal(protocol.READ, ["b.md"]) == (-32602, "params must be an object") + + +async def test_an_unknown_method_is_refused(): + assert await _refusal("fs/nothing", {}) == (-32601, "unknown method 'fs/nothing'") + + +async def test_a_failure_inside_a_tool_is_an_answer_and_the_node_keeps_serving(monkeypatch): + async def broken(params: dict[str, Any]) -> dict[str, Any]: + raise RuntimeError("boom") + + monkeypatch.setitem(server._HANDLERS, protocol.READ, broken) + + assert await _refusal(protocol.READ, {}) == (-32603, "RuntimeError: boom") + assert (await _result(protocol.HELLO, {}))["protocol"] == protocol.PROTOCOL + + +@pytest.mark.parametrize( + "frame", + [ + {"jsonrpc": "2.0", "method": protocol.HELLO}, + {"jsonrpc": "2.0", "id": None, "method": protocol.HELLO}, + {"jsonrpc": "2.0", "id": 1, "method": 5}, + {"jsonrpc": "2.0", "id": 1, "result": {}}, + [{"jsonrpc": "2.0", "id": 1, "method": protocol.HELLO}], + ], +) +async def test_nothing_is_owed_for_a_notification_or_a_frame_that_is_not_a_request(frame): + assert await server.handle(frame) is None + + +# --- the handshake -------------------------------------------------------------- + + +async def test_hello_says_what_the_host_checks_before_using_a_node(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + + hello = await _result(protocol.HELLO, {"protocol": protocol.PROTOCOL}) + + assert hello["protocol"] == protocol.PROTOCOL + assert hello["digest"] == bundle.digest() + assert hello["home"] == str(tmp_path) + assert hello["pid"] == os.getpid() + assert hello["python"].count(".") == 2 + assert isinstance(hello["rg"], bool) + + +async def test_hello_from_a_host_of_another_protocol_is_refused(): + assert await _refusal(protocol.HELLO, {"protocol": protocol.PROTOCOL + 1}) == ( + -32600, + f"this node speaks protocol {protocol.PROTOCOL}, not {protocol.PROTOCOL + 1}", + ) + + +async def test_hello_says_whether_grep_runs_rg_here(monkeypatch): + from raven.agent.tools import file_search + + monkeypatch.setattr(file_search, "_resolve_rg", lambda: None) + assert (await _result(protocol.HELLO, {}))["rg"] is False + monkeypatch.setattr(file_search, "_resolve_rg", lambda: "/opt/rg") + assert (await _result(protocol.HELLO, {}))["rg"] is True + + +# --- the line loop -------------------------------------------------------------- + + +async def test_serve_answers_line_by_line_and_skips_what_it_cannot_read(): + reader = asyncio.StreamReader(limit=256) + for line in ( + b"\n", + b"x" * 400 + b"\n", + b"not json\n", + json.dumps({"jsonrpc": "2.0", "method": protocol.HELLO}).encode() + b"\n", + json.dumps({"jsonrpc": "2.0", "id": 7, "method": protocol.HELLO}).encode() + b"\n", + json.dumps({"jsonrpc": "2.0", "id": 8, "method": "fs/nothing"}).encode(), + ): + reader.feed_data(line) + reader.feed_eof() + out = io.BytesIO() + + await server.serve(reader, out) + + answers = [json.loads(line) for line in out.getvalue().splitlines()] + assert [a["id"] for a in answers] == [None, 7, 8] + assert answers[0]["error"] == {"code": -32700, "message": "not a JSON object"} + assert answers[1]["result"]["protocol"] == protocol.PROTOCOL + assert answers[2]["error"]["code"] == -32601 + + +_NOISY_NODE = """ +import sys +import time +from raven.node import server + +async def noisy(params): + print("a library talking on stdout") + return {"text": "ok"} + +server._HANDLERS["x/noisy"] = noisy +sys.exit(server.run_stdio()) +""" + + +def _node(argv: list[str], requests: list[dict[str, Any]], home: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *argv], + input="".join(json.dumps(r) + "\n" for r in requests), + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "HOME": str(home)}, + cwd=home, + ) + + +@pytest.mark.slow +def test_a_node_process_answers_until_its_stdin_closes(tree): + done = _node( + ["-m", "raven.node", "--stdio"], + [ + {"jsonrpc": "2.0", "id": 1, "method": protocol.HELLO, "params": {"protocol": protocol.PROTOCOL}}, + {"jsonrpc": "2.0", "id": 2, "method": protocol.READ, "params": {"path": "b.md", "roots": [str(tree)]}}, + ], + tree, + ) + + assert done.returncode == 0, done.stderr + first, second = (json.loads(line) for line in done.stdout.splitlines()) + assert first["result"]["digest"] == bundle.digest() + assert "hay" in second["result"]["text"] + + +@pytest.mark.slow +def test_stdout_carries_only_frames_whatever_else_prints(tree): + done = _node(["-c", _NOISY_NODE], [{"jsonrpc": "2.0", "id": 1, "method": "x/noisy"}], tree) + + assert done.returncode == 0, done.stderr + assert [json.loads(line) for line in done.stdout.splitlines()] == [ + {"jsonrpc": "2.0", "id": 1, "result": {"text": "ok"}} + ] + assert "a library talking on stdout" in done.stderr + + +def test_run_stdio_serves_this_process_s_own_stdin_and_moves_print_to_stderr(monkeypatch): + read_end, write_end = os.pipe() + os.write(write_end, (json.dumps({"jsonrpc": "2.0", "id": 3, "method": protocol.HELLO}) + "\n").encode()) + os.close(write_end) + frames = io.BytesIO() + + class _Stdout: + buffer = frames + + monkeypatch.setattr(sys, "stdin", os.fdopen(read_end, "rb", buffering=0)) + monkeypatch.setattr(sys, "stdout", _Stdout()) + + assert server.run_stdio() == 0 + + (answer,) = [json.loads(line) for line in frames.getvalue().splitlines()] + assert answer["id"] == 3 and answer["result"]["protocol"] == protocol.PROTOCOL + assert sys.stdout is sys.stderr + + +# --- python -m raven.node --------------------------------------------------------- + + +def test_version_prints_the_protocol_and_the_digest(capsys): + assert node_main.main(["--version"]) == 0 + assert json.loads(capsys.readouterr().out) == {"protocol": protocol.PROTOCOL, "digest": bundle.digest()} + + +def test_stdio_serves(monkeypatch): + monkeypatch.setattr(server, "run_stdio", lambda: 0) + + assert node_main.main(["--stdio"]) == 0 + + +@pytest.mark.parametrize("argv", [[], ["--stdio", "--version"]]) +def test_a_node_is_started_for_exactly_one_purpose(argv, capsys): + with pytest.raises(SystemExit) as caught: + node_main.main(argv) + + assert caught.value.code == 2 + + +# --- the link's own methods -------------------------------------------------------- + + +async def test_a_ping_comes_back_with_what_it_carried(): + assert await _result(protocol.PING, {"t": 12.5}) == {"t": 12.5} + + +async def test_the_session_directory_is_made_and_resolved_here(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + + got = await _result(protocol.SESSION_DIR, {"path": "~/raven-work/a b"}) + + assert got["path"] == str((tmp_path / "raven-work" / "a b").resolve()) + assert Path(got["path"]).is_dir() + assert abs(got["since"] - time.time()) < 60, "this machine's clock, as the directory was made" + + +async def test_a_session_directory_that_cannot_be_made_is_refused_with_the_reason(tmp_path): + (tmp_path / "f").write_text("a file\n", encoding="utf-8") + + code, said = await _refusal(protocol.SESSION_DIR, {"path": str(tmp_path / "f" / "t1")}) + + assert code == -32603 and said.startswith(f"could not make {tmp_path / 'f' / 't1'}: ") + + +async def test_the_machine_probe_reports_room_python_shell_and_load(): + got = await _result(protocol.PROBE_MACHINE, {}) + + assert set(got) == {"disk_free_kb", "python", "shell", "uv", "load"} + assert str(Path.home()) in got["disk_free_kb"] and got["python"].count(".") == 2 + + +def test_the_node_drives_the_login_shells_the_host_does(): + from raven.agent.subagent.backends.env import _DRIVABLE_SHELLS + + assert server._DRIVABLE_SHELLS == _DRIVABLE_SHELLS + + +def test_the_login_environment_is_captured_from_a_drivable_shell_and_left_alone_otherwise(tmp_path, monkeypatch): + monkeypatch.setenv("SHELL", "/bin/csh") + assert server.capture_login_env() == dict(os.environ), "a shell it cannot drive leaves ssh's environment" + + bash = tmp_path / "bash" + bash.write_text("#!/bin/sh\nprintf 'PATH=/from/profile\\0X=1\\0garbage\\0'\n", encoding="utf-8") + bash.chmod(0o755) + monkeypatch.setenv("SHELL", str(bash)) + assert server.capture_login_env() == {"PATH": "/from/profile", "X": "1"} + + bash.write_text("#!/bin/sh\nprintf 'X=1\\0'\n", encoding="utf-8") + assert server.capture_login_env() == dict(os.environ), "a capture without a PATH is no capture" + bash.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + assert server.capture_login_env() == dict(os.environ), "a failed capture leaves ssh's environment" + + +# --- agents the node starts ----------------------------------------------------------- + +# An agent for these tests: says one thing on stderr, answers every request +# with its params (or its environment, or a child it started), and leaves when +# its stdin closes. +_AGENT = """ +import json, os, subprocess, sys +sys.stderr.write("agent: started\\n") +sys.stderr.flush() +for line in sys.stdin: + line = line.strip() + if not line: + continue + request = json.loads(line) + method = request.get("method") + if method == "env": + result = {"login": os.environ.get("FROM_LOGIN"), "entry": os.environ.get("FROM_ENTRY"), "cwd": os.getcwd()} + elif method == "spawn": + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) + result = {"pid": child.pid} + elif method == "detach": + # In a session of its own, the way Claude Code's shell tool starts a command. + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"], start_new_session=True) + result = {"pid": child.pid} + else: + result = {"echo": request.get("params")} + print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True) +""" + + +class _Host: + """The host's end of the connection: every line the node wrote, decoded.""" + + def __init__(self) -> None: + self.out = io.BytesIO() + self.node = server.Node(self.out) + + def frames(self) -> list[dict[str, Any]]: + return [json.loads(line) for line in self.out.getvalue().splitlines() if line.strip()] + + def notified(self, method: str) -> list[dict[str, Any]]: + return [frame["params"] for frame in self.frames() if frame.get("method") == method] + + async def ask(self, method: str, params: dict[str, Any], request_id: int = 1) -> dict[str, Any]: + answer = await server.handle( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}, self.node + ) + assert answer is not None + return answer + + async def tell(self, method: str, params: dict[str, Any]) -> None: + assert await server.handle({"jsonrpc": "2.0", "method": method, "params": params}, self.node) is None + + async def until(self, check: Any, *, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + while not check(): + assert time.monotonic() < deadline, "did not happen in time" + await asyncio.sleep(0.02) + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + try: + done, _ = os.waitpid(pid, os.WNOHANG) + return done == pid + except ChildProcessError: + return False + + +@pytest.fixture +def agent(tmp_path: Path) -> str: + script = tmp_path / "agent.py" + script.write_text(_AGENT, encoding="utf-8") + return f"{sys.executable} {script}" + + +def _frame(method: str, request_id: int = 1) -> str: + return json.dumps({"jsonrpc": "2.0", "id": request_id, "method": method}) + + +@pytest.mark.slow +async def test_an_agent_starts_in_its_root_in_the_login_environment_plus_the_entry_s_and_is_forwarded( + tmp_path, agent, monkeypatch +): + monkeypatch.setattr(server, "capture_login_env", lambda: {**os.environ, "FROM_LOGIN": "yes"}) + host = _Host() + root = tmp_path / "root" / "deep" + + started = await host.ask( + protocol.AGENT_LAUNCH, {"channel": 1, "command": agent, "cwd": str(root), "env": {"FROM_ENTRY": "1"}} + ) + + assert isinstance(started["result"]["pid"], int) and root.is_dir(), "the root is made for it" + await host.tell(protocol.AGENT_FRAME, {"channel": 1, "line": _frame("env", 5)}) + await host.until(lambda: host.notified(protocol.AGENT_FRAME)) + answer = json.loads(host.notified(protocol.AGENT_FRAME)[0]["line"]) + assert answer["id"] == 5 + assert answer["result"]["login"] == "yes" and answer["result"]["entry"] == "1" + assert Path(answer["result"]["cwd"]).resolve() == root.resolve() + await host.until(lambda: host.notified(protocol.AGENT_STDERR)) + assert host.notified(protocol.AGENT_STDERR) == [{"channel": 1, "text": "agent: started"}] + + closed = await host.ask(protocol.AGENT_CLOSE, {"channel": 1}, request_id=2) + + assert closed["result"] == {"rc": 0}, "stdin closed, the agent left on its own" + await host.until(lambda: host.notified(protocol.AGENT_EXIT)) + assert host.notified(protocol.AGENT_EXIT) == [{"channel": 1, "rc": 0}] + assert host.node.agents == {} + + +@pytest.mark.slow +async def test_cancel_ends_the_agent_s_whole_process_group(tmp_path, agent): + host = _Host() + started = await host.ask(protocol.AGENT_LAUNCH, {"channel": 2, "command": agent, "cwd": str(tmp_path)}) + pid = started["result"]["pid"] + await host.tell(protocol.AGENT_FRAME, {"channel": 2, "line": _frame("spawn")}) + await host.until(lambda: host.notified(protocol.AGENT_FRAME)) + child = json.loads(host.notified(protocol.AGENT_FRAME)[0]["line"])["result"]["pid"] + assert not _gone(pid) and not _gone(child) + + assert (await host.ask(protocol.AGENT_CANCEL, {"channel": 2}, request_id=3))["result"] == {} + + await host.until(lambda: host.notified(protocol.AGENT_EXIT)) + assert host.notified(protocol.AGENT_EXIT)[0]["channel"] == 2 + assert host.notified(protocol.AGENT_EXIT)[0]["rc"] < 0, "ended by a signal" + await host.until(lambda: _gone(pid) and _gone(child)) + + +@pytest.mark.slow +async def test_cancel_ends_what_the_agent_put_in_a_session_of_its_own(tmp_path, agent): + host = _Host() + started = await host.ask(protocol.AGENT_LAUNCH, {"channel": 2, "command": agent, "cwd": str(tmp_path)}) + pid = started["result"]["pid"] + await host.tell(protocol.AGENT_FRAME, {"channel": 2, "line": _frame("detach")}) + await host.until(lambda: host.notified(protocol.AGENT_FRAME)) + child = json.loads(host.notified(protocol.AGENT_FRAME)[0]["line"])["result"]["pid"] + assert os.getpgid(child) != os.getpgid(pid), "the child escaped the agent's group" + + await host.ask(protocol.AGENT_CANCEL, {"channel": 2}, request_id=3) + + await host.until(lambda: _gone(pid) and _gone(child)) + + +def _tree(*, own_sessions: bool) -> tuple[subprocess.Popen[str], int]: + """A process with a child under it, each in a session of its own or both in this one.""" + flag = "start_new_session=True" if own_sessions else "" + parent = subprocess.Popen( + [ + sys.executable, + "-c", + "import subprocess, sys, time; print(subprocess.Popen([sys.executable, '-c'," + f" 'import time; time.sleep(30)'], {flag}).pid, flush=True); time.sleep(30)", + ], + start_new_session=own_sessions, + stdout=subprocess.PIPE, + text=True, + ) + assert parent.stdout is not None + return parent, int(parent.stdout.readline()) + + +def _ends(pid: int) -> bool: + deadline = time.monotonic() + 10 + while not _gone(pid) and time.monotonic() < deadline: + time.sleep(0.05) + return _gone(pid) + + +def _end_quietly(parent: subprocess.Popen[str], child: int) -> None: + for pid in (parent.pid, child): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + parent.wait(timeout=10) + + +def test_a_tree_is_read_off_ps_and_ended_with_what_left_its_group(): + # A leader of its own session, as the node starts an agent, with a child in + # a session of its own under it, as Claude Code's shell tool starts a command. + parent, child = _tree(own_sessions=True) + try: + assert os.getpgid(child) != os.getpgid(parent.pid) + assert server._descendants(parent.pid) == [child] + assert server._descendants(2**22 - 1) == [] + + server._end_tree(parent.pid) + + assert parent.wait(timeout=10) != 0 + assert _ends(child), "the child's own session was ended with the tree" + finally: + _end_quietly(parent, child) + + +def test_a_tree_in_the_node_s_own_group_is_ended_one_by_one_and_the_group_stays(): + parent, child = _tree(own_sessions=False) + try: + assert os.getpgid(parent.pid) == os.getpgid(child) == os.getpgrp() + + server._end_tree(parent.pid) + + assert parent.wait(timeout=10) != 0 + assert _ends(child) + finally: + _end_quietly(parent, child) + + +@pytest.mark.slow +async def test_a_command_not_on_the_machine_and_a_full_node_are_refused_by_code(tmp_path, agent, monkeypatch): + host = _Host() + + refused = await host.ask( + protocol.AGENT_LAUNCH, {"channel": 1, "command": "no-such-agent-for-raven-tests --acp", "cwd": str(tmp_path)} + ) + assert refused["error"] == { + "code": protocol.NOT_FOUND, + "message": "'no-such-agent-for-raven-tests' is not on this machine's PATH", + } + + monkeypatch.setattr(protocol, "MAX_AGENTS", 0) + full = await host.ask(protocol.AGENT_LAUNCH, {"channel": 1, "command": agent, "cwd": str(tmp_path)}) + assert full["error"] == {"code": protocol.BUSY, "message": "0 agents are already running on this machine"} + assert host.node.agents == {} + + +async def test_a_root_the_machine_cannot_make_refuses_the_launch_with_the_reason(tmp_path, agent): + host = _Host() + (tmp_path / "f").write_text("a file\n", encoding="utf-8") + + refused = await host.ask(protocol.AGENT_LAUNCH, {"channel": 1, "command": agent, "cwd": str(tmp_path / "f" / "r")}) + + assert refused["error"]["code"] == -32603 + assert refused["error"]["message"].startswith(f"could not make {tmp_path / 'f' / 'r'}: ") + + +@pytest.mark.parametrize( + ("method", "params", "said"), + [ + (protocol.AGENT_CANCEL, {"channel": 9}, "no agent on channel 9"), + (protocol.AGENT_CLOSE, {"channel": "x"}, "channel must be a whole number"), + (protocol.AGENT_LAUNCH, {"command": "x", "cwd": "/"}, "channel must be a whole number"), + (protocol.AGENT_LAUNCH, {"channel": 1, "command": "", "cwd": "/"}, "command must be a non-empty string"), + ( + protocol.AGENT_LAUNCH, + {"channel": 1, "command": "x 'y", "cwd": "/"}, + "command cannot be parsed: No closing quotation", + ), + ( + protocol.AGENT_LAUNCH, + {"channel": 1, "command": "x", "cwd": "/", "env": {"A": 1}}, + "env must be an object of strings", + ), + (protocol.PROBE_AGENT, {"command": " "}, "command must be a non-empty string"), + ], +) +async def test_a_channel_call_that_names_nothing_usable_is_refused_by_name(method, params, said): + host = _Host() + + refused = await host.ask(method, params) + + assert refused["error"] == {"code": -32602, "message": said} + + +async def test_a_frame_for_a_channel_nobody_opened_is_dropped(): + host = _Host() + + await host.tell(protocol.AGENT_FRAME, {"channel": 9, "line": _frame("x")}) + await host.tell(protocol.AGENT_FRAME, {"channel": ["not", "a", "number"], "line": _frame("x")}) + + assert host.frames() == [] + + +async def test_probe_agent_answers_on_the_path_an_agent_here_would_get(tmp_path, monkeypatch): + monkeypatch.setattr(server, "capture_login_env", lambda: {"PATH": str(tmp_path)}) + host = _Host() + found = tmp_path / "fake-agent" + found.write_text("#!/bin/sh\n", encoding="utf-8") + found.chmod(0o755) + + assert (await host.ask(protocol.PROBE_AGENT, {"command": "fake-agent --acp"}))["result"] == {"found": str(found)} + assert (await host.ask(protocol.PROBE_AGENT, {"command": "nothing-here"}))["result"] == {"found": None} + overridden = await host.ask(protocol.PROBE_AGENT, {"command": "fake-agent", "env": {"PATH": "/nonexistent"}}) + assert overridden["result"] == {"found": None}, "the entry's own PATH wins, as it does at launch" + + +@pytest.mark.slow +async def test_shutdown_ends_every_agent_and_the_loop_leaves_without_an_eof(tmp_path, agent): + reader = asyncio.StreamReader() + launch = {"jsonrpc": "2.0", "id": 1, "method": protocol.AGENT_LAUNCH} + launch["params"] = {"channel": 1, "command": agent, "cwd": str(tmp_path)} + reader.feed_data(json.dumps(launch).encode() + b"\n") + reader.feed_data(json.dumps({"jsonrpc": "2.0", "id": 2, "method": protocol.SHUTDOWN}).encode() + b"\n") + out = io.BytesIO() + + await asyncio.wait_for(server.serve(reader, out), timeout=30) + + answers = { + frame["id"]: frame for frame in (json.loads(line) for line in out.getvalue().splitlines()) if "id" in frame + } + assert answers[2]["result"] == {} + assert _gone(answers[1]["result"]["pid"]) + + +async def test_requests_are_answered_as_they_finish_not_in_order(monkeypatch): + async def slow(params: dict[str, Any]) -> dict[str, Any]: + await asyncio.sleep(0.3) + return {"text": "slow"} + + monkeypatch.setitem(server._HANDLERS, "x/slow", slow) + reader = asyncio.StreamReader() + reader.feed_data(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "x/slow"}).encode() + b"\n") + reader.feed_data(json.dumps({"jsonrpc": "2.0", "id": 2, "method": protocol.PING}).encode() + b"\n") + reader.feed_eof() + out = io.BytesIO() + + await server.serve(reader, out) + + assert [frame["id"] for frame in (json.loads(line) for line in out.getvalue().splitlines())] == [2, 1] + + +# --- what the host writes, sends and collects -------------------------------------- + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +async def test_stat_says_what_is_there_and_hashes_a_file_on_request(tree): + roots = [str(tree)] + + file = await _result(protocol.STAT, {"path": str(tree / "b.md"), "roots": roots, "hash": True}) + folder = await _result(protocol.STAT, {"path": str(tree / "sub"), "roots": roots}) + missing = await _result(protocol.STAT, {"path": str(tree / "nothing"), "roots": roots}) + + assert file["exists"] and file["kind"] == "file" and file["size"] == 4 + assert file["sha256"] == hashlib.sha256(b"hay\n").hexdigest() + assert abs(file["mtime"] - (tree / "b.md").stat().st_mtime) < 1e-6 + assert folder["kind"] == "dir" and folder["exists"] and "sha256" not in folder + assert missing == {"exists": False, "kind": "missing", "size": 0, "mtime": None} + + +async def test_a_call_outside_the_roots_is_refused_whatever_it_is(tree): + outside = str(tree.parent / "secret.txt") + for method in (protocol.STAT, protocol.PULL): + code, said = await _refusal(method, {"path": outside, "roots": [str(tree)]}) + assert code == -32602 and "outside allowed directories" in said, method + + +async def test_a_write_through_the_node_is_the_write_the_host_would_have_made(tree): + path = str(tree / "new" / "note.md") + roots = [str(tree)] + + got = await _result(protocol.WRITE, {"path": path, "content": "one\n", "roots": roots}) + appended = await _result(protocol.WRITE, {"path": path, "content": "two\n", "mode": "append", "roots": roots}) + + assert got["text"] == f"Successfully wrote 4 bytes to {path}" + assert got["file_change"] == {"path": path, "after": "one\n", "before": None} + assert got["diff"].startswith(f"--- {path}") and "+one" in got["diff"] + assert appended == {"text": f"Successfully appended 4 bytes to {path}"} + assert (tree / "new" / "note.md").read_text(encoding="utf-8") == "one\ntwo\n" + + +async def test_a_write_that_expects_the_file_as_it_was_read_is_refused_once_it_changed(tree): + path = str(tree / "b.md") + roots = [str(tree)] + read = hashlib.sha256(b"hay\n").hexdigest() + + first = await _result(protocol.WRITE, {"path": path, "content": "x\n", "expect": read, "roots": roots}) + code, said = await _refusal(protocol.WRITE, {"path": path, "content": "y\n", "expect": read, "roots": roots}) + + assert first["text"].startswith("Successfully wrote") + assert (code, said) == (protocol.CHANGED, f"{path} changed after it was read; nothing was written") + assert (tree / "b.md").read_text(encoding="utf-8") == "x\n" + # A file expected absent that is there, and one expected absent that is. + assert (await _refusal(protocol.WRITE, {"path": path, "content": "z\n", "expect": "absent", "roots": roots}))[ + 0 + ] == protocol.CHANGED + fresh = str(tree / "fresh.md") + made = await _result(protocol.WRITE, {"path": fresh, "content": "z\n", "expect": "absent", "roots": roots}) + assert made["file_change"]["before"] is None and (tree / "fresh.md").read_text(encoding="utf-8") == "z\n" + + +async def test_an_edit_through_the_node_is_the_edit_the_host_would_have_made(tree): + path = str(tree / "sub" / "a.py") + roots = [str(tree)] + + got = await _result( + protocol.EDIT, {"path": path, "old_text": "needle = 1", "new_text": "needle = 2", "roots": roots} + ) + miss = await _result( + protocol.EDIT, {"path": path, "old_text": "nothing like this", "new_text": "x", "roots": roots} + ) + + assert got["text"] == f"Successfully edited {path}" + assert got["file_change"] == {"path": path, "after": "needle = 2\nhay = 2\n", "before": "needle = 1\nhay = 2\n"} + assert "-needle = 1" in got["diff"] and "+needle = 2" in got["diff"] + assert miss["text"].startswith(f"Error: old_text not found in {path}") + assert (tree / "sub" / "a.py").read_text(encoding="utf-8") == "needle = 2\nhay = 2\n" + + +async def test_a_pull_returns_the_bytes_asked_for_with_the_whole_file_s_digest_first(tree): + data = (tree / "big.txt").read_bytes() + path = str(tree / "big.txt") + roots = [str(tree)] + + first = await _result(protocol.PULL, {"path": path, "offset": 0, "length": 1000, "roots": roots}) + last = await _result(protocol.PULL, {"path": path, "offset": len(data) - 10, "length": 1000, "roots": roots}) + + assert base64.b64decode(first["data"]) == data[:1000] and first["total"] == len(data) + assert first["sha256"] == hashlib.sha256(data).hexdigest() and first["eof"] is False + assert base64.b64decode(last["data"]) == data[-10:] and last["eof"] is True and "sha256" not in last + assert (await _refusal(protocol.PULL, {"path": str(tree / "sub"), "roots": roots}))[1].endswith( + "is not a file on this machine" + ) + code, said = await _refusal(protocol.PULL, {"path": path, "length": protocol.CHUNK + 1, "roots": roots}) + assert code == -32602 and f"between 1 and {protocol.CHUNK}" in said + + +# --- tasks that outlive a connection --------------------------------------------------- + + +class _Conn(_Host): + """One connection to a daemon several connections share.""" + + def __init__(self, daemon: server.Daemon) -> None: + self.out = io.BytesIO() + self.node = server.Node(self.out, daemon) + + +def _record(daemon: server.Daemon, task: str) -> dict[str, Any]: + return json.loads((daemon.record_dir(task) / "task.json").read_text(encoding="utf-8")) + + +@pytest.mark.slow +async def test_a_task_is_recorded_as_it_runs_and_goes_on_when_its_connection_ends(tmp_path, agent): + daemon = server.Daemon(tmp_path / "node") + first = _Conn(daemon) + started = await first.ask( + protocol.AGENT_LAUNCH, {"channel": 1, "task": "t-one", "command": agent, "cwd": str(tmp_path / "w")} + ) + assert started["result"]["task"] == "t-one" + pid = started["result"]["pid"] + await first.tell(protocol.AGENT_FRAME, {"channel": 1, "line": _frame("one")}) + await first.until(lambda: len(first.notified(protocol.AGENT_FRAME)) == 1) + assert first.notified(protocol.AGENT_FRAME)[0]["n"] == 1 + assert _record(daemon, "t-one")["state"] == "running" + + await first.node.detach_all() + + assert not _gone(pid), "the agent runs on without its connection" + task = daemon.tasks["t-one"] + assert task.subscriber is None and daemon.connections == set() + await task.write(_frame("two", 2)) + await first.until(lambda: task.frames == 2) + assert (daemon.record_dir("t-one") / "frames.jsonl").read_text(encoding="utf-8").count("\n") == 2 + + second = _Conn(daemon) + attached = await second.ask(protocol.AGENT_ATTACH, {"channel": 7, "task": "t-one", "after": 1}, request_id=2) + assert attached["result"]["replayed"] == 1 and attached["result"]["rc"] is None + assert [f["n"] for f in second.notified(protocol.AGENT_FRAME)] == [2], "only what the host had not read" + assert second.node.channels[7] is task + await second.ask(protocol.AGENT_CLOSE, {"channel": 7}, request_id=3) + await second.until(lambda: second.notified(protocol.AGENT_EXIT)) + assert second.notified(protocol.AGENT_EXIT) == [{"channel": 7, "rc": 0}] + assert first.notified(protocol.AGENT_EXIT) == [], "the connection that ended hears nothing" + assert _record(daemon, "t-one")["state"] == "exited" and _record(daemon, "t-one")["rc"] == 0 + assert 7 not in second.node.channels and daemon.running() == [] + + +@pytest.mark.slow +async def test_a_finished_task_is_replayed_from_its_record_by_a_daemon_that_never_ran_it(tmp_path, agent): + root = tmp_path / "node" + first = _Conn(server.Daemon(root)) + await first.ask(protocol.AGENT_LAUNCH, {"channel": 1, "task": "t-done", "command": agent, "cwd": str(tmp_path)}) + await first.tell(protocol.AGENT_FRAME, {"channel": 1, "line": _frame("a")}) + await first.tell(protocol.AGENT_FRAME, {"channel": 1, "line": _frame("b", 2)}) + await first.until(lambda: len(first.notified(protocol.AGENT_FRAME)) == 2) + await first.ask(protocol.AGENT_CLOSE, {"channel": 1}, request_id=2) + await first.until(lambda: first.notified(protocol.AGENT_EXIT)) + + later = _Conn(server.Daemon(root)) + attached = await later.ask(protocol.AGENT_ATTACH, {"channel": 1, "task": "t-done", "after": 0}) + + assert attached["result"]["replayed"] == 2 and attached["result"]["rc"] == 0 + assert [f["n"] for f in later.notified(protocol.AGENT_FRAME)] == [1, 2] + assert later.notified(protocol.AGENT_EXIT) == [{"channel": 1, "rc": 0}] + assert 1 not in later.node.channels, "a finished task holds no channel" + + +async def test_a_record_left_running_by_a_daemon_that_died_is_read_as_lost(tmp_path): + root = tmp_path / "node" + daemon = server.Daemon(root) + record = daemon.record_dir("t-gone") + record.mkdir(parents=True) + (record / "task.json").write_text( + json.dumps({"task": "t-gone", "state": "running", "pid": 1, "started": 1.0, "frames": 1}), encoding="utf-8" + ) + (record / "frames.jsonl").write_text('{"jsonrpc":"2.0","id":1,"result":{}}\n', encoding="utf-8") + conn = _Conn(daemon) + + attached = await conn.ask(protocol.AGENT_ATTACH, {"channel": 1, "task": "t-gone"}) + + assert attached["result"]["rc"] == server.LOST_RC and attached["result"]["replayed"] == 1 + assert conn.notified(protocol.AGENT_EXIT) == [{"channel": 1, "rc": server.LOST_RC}] + assert _record(daemon, "t-gone")["state"] == "lost" + + +@pytest.mark.parametrize( + ("params", "code", "said"), + [ + ({"channel": 1, "task": "t-nobody"}, protocol.UNKNOWN_TASK, "no task 't-nobody' is running or recorded"), + ({"channel": 1, "task": "../etc"}, -32602, "task must be a short name"), + ({"channel": 1, "task": "t-x", "after": -1}, -32602, "after must be a whole number"), + ], +) +async def test_an_attach_that_names_nothing_usable_is_refused_by_code(tmp_path, params, code, said): + conn = _Conn(server.Daemon(tmp_path / "node")) + + answer = await conn.ask(protocol.AGENT_ATTACH, params) + + assert answer["error"]["code"] == code and said in answer["error"]["message"] + + +@pytest.mark.slow +async def test_a_launch_under_a_running_task_s_id_is_refused_and_the_list_names_every_task(tmp_path, agent): + daemon = server.Daemon(tmp_path / "node") + old = daemon.record_dir("t-old") + old.mkdir(parents=True) + (old / "task.json").write_text( + json.dumps({"task": "t-old", "state": "exited", "rc": 0, "started": 1.0, "ended": 2.0, "frames": 3}), + encoding="utf-8", + ) + conn = _Conn(daemon) + await conn.ask(protocol.AGENT_LAUNCH, {"channel": 1, "task": "t-run", "command": agent, "cwd": str(tmp_path)}) + + again = await conn.ask( + protocol.AGENT_LAUNCH, {"channel": 2, "task": "t-run", "command": agent, "cwd": str(tmp_path)}, request_id=2 + ) + listed = (await conn.ask(protocol.TASK_LIST, {}, request_id=3))["result"]["tasks"] + hello = (await conn.ask(protocol.HELLO, {"protocol": protocol.PROTOCOL}, request_id=4))["result"] + + assert again["error"]["code"] == -32602 and "already running; attach to it" in again["error"]["message"] + assert [(row["task"], row["state"], row["attached"]) for row in listed] == [ + ("t-old", "exited", False), + ("t-run", "running", True), + ] + assert hello["mode"] == "daemon" and hello["tasks"] == 1 + await conn.ask(protocol.AGENT_CANCEL, {"channel": 1}, request_id=5) + await conn.until(lambda: conn.notified(protocol.AGENT_EXIT)) + + +def test_sweep_drops_the_records_of_tasks_that_ended_long_ago_and_keeps_the_rest(tmp_path): + daemon = server.Daemon(tmp_path / "node") + for name, state, ended in ( + ("t-ancient", "exited", time.time() - protocol.TASK_KEEP_S - 60), + ("t-recent", "exited", time.time() - 60), + ("t-still", "running", None), + ): + record = daemon.record_dir(name) + record.mkdir(parents=True) + (record / "task.json").write_text(json.dumps({"task": name, "state": state, "ended": ended}), encoding="utf-8") + + assert daemon.sweep() == 1 + assert sorted(p.name for p in (tmp_path / "node" / server.TASKS_DIR).iterdir()) == ["t-recent", "t-still"] + + +@pytest.mark.slow +async def test_shutdown_from_a_host_leaves_a_daemon_s_tasks_running(tmp_path, agent): + daemon = server.Daemon(tmp_path / "node") + reader = asyncio.StreamReader() + launch = {"jsonrpc": "2.0", "id": 1, "method": protocol.AGENT_LAUNCH} + launch["params"] = {"channel": 1, "task": "t-keep", "command": agent, "cwd": str(tmp_path)} + reader.feed_data(json.dumps(launch).encode() + b"\n") + reader.feed_data(json.dumps({"jsonrpc": "2.0", "id": 2, "method": protocol.SHUTDOWN}).encode() + b"\n") + out = io.BytesIO() + + await asyncio.wait_for(server.serve(reader, out, daemon), timeout=30) + + answers = {f["id"]: f for f in (json.loads(line) for line in out.getvalue().splitlines()) if "id" in f} + pid = answers[1]["result"]["pid"] + try: + assert not _gone(pid) and daemon.tasks["t-keep"].subscriber is None + finally: + daemon.tasks["t-keep"].end() + await daemon.tasks["t-keep"].proc.wait() + + +@pytest.mark.slow +async def test_the_daemon_serves_on_its_socket_across_connections_and_leaves_when_idle(tmp_path, agent, monkeypatch): + monkeypatch.setattr(server, "_IDLE_CHECK_S", 0.1) + root = tmp_path / "n" + running = asyncio.create_task(server.run_daemon_async(root, idle_s=0.3)) + sock = server.socket_path(root) + deadline = time.monotonic() + 10 + while not sock.exists() and time.monotonic() < deadline: + await asyncio.sleep(0.05) + + async def ask( + writer: asyncio.StreamWriter, reader: asyncio.StreamReader, request: dict[str, Any] + ) -> dict[str, Any]: + writer.write(json.dumps(request).encode() + b"\n") + await writer.drain() + while True: + frame = json.loads(await reader.readline()) + if frame.get("id") == request["id"]: + return frame + + reader, writer = await asyncio.open_unix_connection(str(sock)) + hello = await ask(writer, reader, {"jsonrpc": "2.0", "id": 1, "method": protocol.HELLO, "params": {}}) + assert hello["result"]["mode"] == "daemon" and hello["result"]["pid"] == os.getpid() + launch = {"channel": 1, "task": "t-sock", "command": agent, "cwd": str(tmp_path)} + started = await ask(writer, reader, {"jsonrpc": "2.0", "id": 2, "method": protocol.AGENT_LAUNCH, "params": launch}) + pid = started["result"]["pid"] + writer.close() + await writer.wait_closed() + await asyncio.sleep(0.2) + assert not _gone(pid), "the agent survives its connection" + + reader, writer = await asyncio.open_unix_connection(str(sock)) + attach = {"channel": 3, "task": "t-sock", "after": 0} + answer = await ask(writer, reader, {"jsonrpc": "2.0", "id": 3, "method": protocol.AGENT_ATTACH, "params": attach}) + assert answer["result"]["rc"] is None + await ask(writer, reader, {"jsonrpc": "2.0", "id": 4, "method": protocol.AGENT_CANCEL, "params": {"channel": 3}}) + exited = json.loads(await reader.readline()) + assert exited["method"] == protocol.AGENT_EXIT and exited["params"]["channel"] == 3 + writer.close() + await writer.wait_closed() + + assert await asyncio.wait_for(running, timeout=10) == 0, "nothing to do: the daemon leaves" + assert not sock.exists() + assert _record(server.Daemon(root), "t-sock")["state"] == "exited" + + +@pytest.mark.slow +async def test_a_second_daemon_under_the_same_directory_leaves_at_once(tmp_path, monkeypatch): + monkeypatch.setattr(server, "_IDLE_CHECK_S", 0.1) + root = tmp_path / "n" + first = asyncio.create_task(server.run_daemon_async(root, idle_s=0.5)) + deadline = time.monotonic() + 10 + while not server.socket_path(root).exists() and time.monotonic() < deadline: + await asyncio.sleep(0.05) + + assert await asyncio.wait_for(server.run_daemon_async(root), timeout=5) == 0 + assert server.socket_path(root).exists(), "the first daemon's socket is untouched" + assert await asyncio.wait_for(first, timeout=10) == 0 + + +@pytest.mark.parametrize("argv", [["--serve"], ["--attach"]]) +def test_serve_and_attach_need_the_node_directory(argv, capsys): + with pytest.raises(SystemExit): + node_main.main(argv) + assert "--dir" in capsys.readouterr().err diff --git a/tests/test_ops_connection_add.py b/tests/test_ops_connection_add.py index 51830b066..b123662a2 100644 --- a/tests/test_ops_connection_add.py +++ b/tests/test_ops_connection_add.py @@ -381,3 +381,44 @@ def no_config(): monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) monkeypatch.delenv(connections.CONNECTIONS_ENV, raising=False) assert _REAL_STORE_PATH() == tmp_path / "home" / "connections.json" + + +# --- what is learned about a machine after it was added ------------------------------ + + +def _store(_home, rows) -> object: + path = _home / "connections.json" + path.write_text(rows if isinstance(rows, str) else json.dumps({"connections": rows}), encoding="utf-8") + return path + + +def test_set_fields_changes_one_machine_and_keeps_everything_else(_home): + path = _store(_home, [{"id": "a", "host": "h1", "note": "kept"}, {"id": "b", "host": "h2"}]) + + assert adder.set_fields("a", {"node_dir": "/data/raven-node"}) == path + + assert json.loads(path.read_text(encoding="utf-8")) == { + "connections": [ + {"id": "a", "host": "h1", "note": "kept", "node_dir": "/data/raven-node"}, + {"id": "b", "host": "h2"}, + ] + } + + +@pytest.mark.parametrize( + ("rows", "said"), + [ + ([{"id": "b"}], "machine 'a' is listed 0 times"), + ([{"id": "a"}, {"id": " a "}], "machine 'a' is listed 2 times"), + ([{"id": "a"}, {"host": "no id"}], "Give those entries an id"), + ("{not json", "Fix or move that file before changing it."), + ], +) +def test_set_fields_refuses_rather_than_guess_or_lose_rows(_home, rows, said): + path = _store(_home, rows) + before = path.read_text(encoding="utf-8") + + with pytest.raises(ValueError, match=said): + adder.set_fields("a", {"node_dir": "/x"}) + + assert path.read_text(encoding="utf-8") == before diff --git a/tests/test_ops_remote_path.py b/tests/test_ops_remote_path.py new file mode 100644 index 000000000..d93d454ae --- /dev/null +++ b/tests/test_ops_remote_path.py @@ -0,0 +1,318 @@ +"""raven.ops.remote_path: a file on a registered machine, written the way scp writes one. + +The rules are scp(1)'s own (OpenSSH 9.9p2): the short form has no port field, +the URI form is the only one with a port, and in it the first slash after the +host only separates. A path names exactly one registered machine -- by its id, +or by address and port -- or it is refused before any ssh. Shown, it uses the +registry's user, address and port, in a form scp reads back. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from raven.ops import remote_path +from raven.ops.remote_path import RemotePathError, absolute, looks_remote, parse, within + +ROWS = [ + {"id": "box2", "display_name": "Box 2", "host": "203.0.113.7", "port": 49273, "user": "root"}, + {"id": "box", "display_name": "Box", "host": "203.0.113.7", "port": 41941, "user": "root"}, + {"id": "std", "host": "198.51.100.9", "port": 22, "user": "alice", "paths": ["/data/shared"]}, + {"id": "v6", "host": "2001:db8::5", "port": 22, "user": "root"}, +] + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch): + monkeypatch.setattr(remote_path, "_HOMES", {}) + monkeypatch.setattr(remote_path, "_NODES", None) + + +def _where(text: str): + found = parse(text, ROWS) + assert found is not None, text + return found.machine, found.path + + +# --- the two forms ------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("text", "machine", "path"), + [ + # A16: the machine's id, the id with its user, and the URI with the address. + ("box2:/root/x", "box2", "/root/x"), + ("root@box2:/root/x", "box2", "/root/x"), + ("scp://root@203.0.113.7:49273//root/x", "box2", "/root/x"), + # C25: the first slash after host:port only separates. + ("scp://203.0.113.7:49273/root/x", "box2", "root/x"), + ("scp://box2", "box2", ""), + # C26: after the colon, a path not starting with / is under the home. + ("box2:", "box2", ""), + ("box2:raven-work/a", "box2", "raven-work/a"), + ("box2:~/raven-work/a", "box2", "~/raven-work/a"), + # C21: an address with no port is port 22; the registry decides the rest. + ("alice@198.51.100.9:/data/x", "std", "/data/x"), + ("198.51.100.9:/data/x", "std", "/data/x"), + ("sftp://alice@198.51.100.9//data", "std", "/data"), + ("root@[2001:db8::5]:/etc/hostname", "v6", "/etc/hostname"), + ("scp://root@[2001:db8::5]:22//etc/hostname", "v6", "/etc/hostname"), + # An address is matched without regard to case; a user and an id are not. + ("scp://root@203.0.113.7:49273//x", "box2", "/x"), + ], +) +def test_a_remote_path_names_one_registered_machine(text, machine, path): + assert _where(text) == (machine, path) + + +def test_a_user_and_a_machine_id_are_compared_as_written(): + with pytest.raises(RemotePathError, match="registered for another user"): + parse("scp://ROOT@203.0.113.7:49273//x", ROWS) + with pytest.raises(RemotePathError, match="no registered machine matches it"): + parse("BOX2:/x", ROWS) + + +@pytest.mark.parametrize( + ("text", "said"), + [ + # A17: no port means 22, and no registered machine is on 22 there. + ("root@203.0.113.7:/etc/hostname", "no registered machine matches it"), + # C24: the short form has no port field -- `49273:/a` is a file name on 22. + ("root@203.0.113.7:49273:/a", "no registered machine matches it"), + ("bob@std:/x", "logs in as 'bob', and machine 'std' is registered for another user"), + ("scp://box2:41941//x", "names machine 'Box 2' (box2) on a port it is not registered on"), + ("scp://203.0.113.7:99999//x", "names port '99999', which is not a port"), + ("scp://203.0.113.7:ssh//x", "names port 'ssh', which is not a port"), + ("[2001:db8::5/x", "opens an address with '[' and does not close it"), + ("root@[2001:db8::5:/x", "opens an address with '['"), + ("scp://[2001:db8::5//x", "opens an address with '['"), + ("scp://[2001:db8::5]x//x", "something after the address that is not a port"), + ("nobox:/x", "write it as ./name or as an absolute path"), + # C27: a local name holding a colon reads as remote unless written as a path. + ("a:b.txt", "write it as ./name or as an absolute path"), + ], +) +def test_a_path_that_names_no_registered_machine_is_refused_before_any_ssh(text, said): + with pytest.raises(RemotePathError, match=re.escape(said)): + parse(text, ROWS) + + +def test_two_rows_on_one_address_and_port_are_named_by_id_instead(): + rows = [*ROWS, {**ROWS[0], "id": "box2-again"}] + + with pytest.raises(RemotePathError, match=r"matches more than one registered machine .*write the machine's id"): + parse("scp://203.0.113.7:49273//x", rows) + assert parse("box2-again:/x", rows).machine == "box2-again" + + +@pytest.mark.parametrize( + "text", + ["./a:b.txt", "/abs/a:b", "C:\\x", "c:/x", "C:", "http://x.org/a", "https://x.org/a:b", "~/x", "dir/a:b", ":x", ""], +) +def test_a_local_path_is_left_to_this_computer(text): + assert not looks_remote(text) + assert parse(text, ROWS) is None + + +# --- shown the way scp writes it ------------------------------------------------------ + + +@pytest.mark.parametrize( + ("text", "shown"), + [ + # C28: not port 22 -- the URI form, absolute paths with two slashes. + ("box2:/root/raven-work/t1/report.md", "scp://root@203.0.113.7:49273//root/raven-work/t1/report.md"), + ("box2:raven-work/a", "scp://root@203.0.113.7:49273/raven-work/a"), + # Port 22 -- the short form. + ("std:/data/x", "alice@198.51.100.9:/data/x"), + ("std:notes", "alice@198.51.100.9:notes"), + ("v6:/etc/hostname", "root@[2001:db8::5]:/etc/hostname"), + ], +) +def test_a_path_is_shown_with_the_registry_s_user_address_and_port(text, shown): + assert parse(text, ROWS).display() == shown + + +@pytest.mark.parametrize( + "text", ["box2:/root/x", "box2:raven-work/a", "std:/data/x", "std:notes", "v6:/etc/hostname", "box2:"] +) +def test_what_is_shown_reads_back_as_the_same_file_on_the_same_machine(text): + first = parse(text, ROWS) + + again = parse(first.display(), ROWS) + + assert (again.machine, again.path) == (first.machine, first.path) + + +def test_a_path_is_shown_absolute_once_the_home_is_known(): + where = parse("box2:raven-work/a", ROWS) + + assert where.display(where.absolute("/root")) == "scp://root@203.0.113.7:49273//root/raven-work/a" + + +# --- placing a path on its machine ------------------------------------------------ + + +@pytest.mark.parametrize( + ("path", "placed"), + [ + ("", "/root"), + ("~", "/root"), + ("~/a", "/root/a"), + ("a/b", "/root/a/b"), + ("/x/../y", "/y"), + ("raven-work/../.ssh/id", "/root/.ssh/id"), + ("~bob/x", "~bob/x"), + ], +) +def test_a_path_is_placed_against_the_login_home(path, placed): + assert absolute(path, "/root") == placed + + +def test_without_a_home_a_relative_path_stays_as_written(): + assert absolute("a/b", "") == "a/b" + assert absolute("", "") == "" + + +@pytest.mark.parametrize( + ("path", "root", "inside"), + [ + ("/root/raven-work", "/root/raven-work", True), + ("/root/raven-work/t1/a", "/root/raven-work", True), + ("/root/raven-work/t1/a", "/root/raven-work/", True), + ("/root/raven-workx/a", "/root/raven-work", False), + ("/root/raven-work/../.ssh/id", "/root/raven-work", False), + ("raven-work/a", "/root/raven-work", False), + ], +) +def test_within_compares_whole_directories(path, root, inside): + assert within(path, root) is inside + + +def test_only_an_absolute_home_is_remembered(): + remote_path.remember_home("box2", "/root/") + remote_path.remember_home("box", "relative") + + assert remote_path.known_home("box2") == "/root" + assert remote_path.known_home("box") == "" + + +# --- what a read reaches, for its permission check ---------------------------------- + + +@pytest.fixture +def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + path = tmp_path / "connections.json" + path.write_text(json.dumps({"connections": ROWS}), encoding="utf-8") + monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + + +@pytest.fixture +def agents(monkeypatch: pytest.MonkeyPatch): + """The acp entries config holds; none unless a test adds one.""" + entries: list[SimpleNamespace] = [] + monkeypatch.setattr( + "raven.config.loader.load_config", lambda *a, **k: SimpleNamespace(subagents=SimpleNamespace(agents=entries)) + ) + return entries + + +@pytest.mark.parametrize( + ("tool", "path", "inside", "directory"), + [ + ("read_file", "box2:raven-work/t1/a.md", True, "~/raven-work/t1"), + ("read_file", "box2:~/raven-work/t1/a.md", True, "~/raven-work/t1"), + ("list_dir", "box2:raven-work", True, "~/raven-work"), + ("grep", "box2:raven-work/t1", True, "~/raven-work/t1"), + ("find", "box2:", False, "~"), + ("read_file", "box2:.ssh/id_rsa", False, "~/.ssh"), + ("read_file", "box2:raven-work/../.ssh/id_rsa", False, "~/.ssh"), + # Absolute with no home known yet: placed nowhere, so it asks. + ("read_file", "box2:/root/raven-work/a", False, "/root/raven-work"), + # The row's own paths. + ("read_file", "std:/data/shared/a/b.txt", True, "/data/shared/a"), + ("list_dir", "std:/data", False, "/data"), + ], +) +def test_a_read_reaches_one_directory_inside_the_roots_or_outside(registry, agents, tool, path, inside, directory): + reach = remote_path.reading(tool, {"path": path}) + + assert (reach.inside, reach.directory) == (inside, directory) + + +def test_once_the_home_is_known_an_absolute_path_is_placed_in_the_roots(registry, agents): + remote_path.remember_home("box2", "/root") + + inside = remote_path.reading("read_file", {"path": "scp://root@203.0.113.7:49273//root/raven-work/t1/a"}) + outside = remote_path.reading("read_file", {"path": "box2:/root/.ssh/id_rsa"}) + + assert (inside.inside, inside.directory) == (True, "/root/raven-work/t1") + assert (outside.inside, outside.directory) == (False, "/root/.ssh") + + +def test_an_agent_entry_s_remote_cwd_on_that_machine_is_a_root(registry, agents): + agents.append(SimpleNamespace(machine="box2", remote_cwd="/srv/agents")) + agents.append(SimpleNamespace(machine="box", remote_cwd="/elsewhere")) + agents.append(SimpleNamespace(machine=None, remote_cwd=None)) + + assert remote_path.reading("read_file", {"path": "box2:/srv/agents/t1/a"}).inside + assert not remote_path.reading("read_file", {"path": "box2:/elsewhere/a"}).inside + + +def test_a_config_that_cannot_load_adds_no_root(registry, monkeypatch): + def broken(*args, **kwargs): + raise ValueError("not json") + + monkeypatch.setattr("raven.config.loader.load_config", broken) + + assert remote_path.reading("read_file", {"path": "box2:raven-work/a"}).inside + assert not remote_path.reading("read_file", {"path": "box2:/srv/agents/a"}).inside + + +@pytest.mark.parametrize( + ("tool", "params"), + [ + ("write_file", {"path": "box2:raven-work/a"}), + ("read_file", {"path": "./notes.md"}), + ("read_file", {"path": "nobox:/x"}), + ("read_file", {"path": 7}), + ("read_file", None), + ("grep", {"pattern": "x"}), + ], +) +def test_a_call_that_reaches_no_machine_has_no_reading(registry, agents, tool, params): + assert remote_path.reading(tool, params) is None + + +def test_the_fence_is_the_root_a_path_is_under_else_what_was_approved(registry, agents): + where = parse("box2:raven-work/t1/a", ROWS) + + assert remote_path.fence("read_file", where, "/root/raven-work/t1/a", "/root") == "/root/raven-work" + assert remote_path.fence("read_file", where, "/root/.ssh/id_rsa", "/root") == "/root/.ssh" + assert remote_path.fence("list_dir", where, "/root/.ssh", "/root") == "/root/.ssh" + assert remote_path.fence("read_file", where, "/id", "/root") == "/" + + +def test_with_the_lane_off_no_path_names_a_machine(registry, agents, monkeypatch): + # Off, the file tools read every path as they always did: a name with a + # colon is a local file, scp:// is a URL, and no registry is consulted. + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + + assert remote_path.target("box2:/root/x") is None + assert remote_path.target("scp://root@203.0.113.7:49273//root/x") is None + assert remote_path.reading("read_file", {"path": "box2:.ssh/id_rsa"}) is None + + +def test_the_node_pool_is_whatever_assembly_handed_over(): + assert remote_path.nodes() is None + pool = object() + + remote_path.use_nodes(lambda: pool) + + assert remote_path.nodes() is pool diff --git a/tests/test_ops_transport.py b/tests/test_ops_transport.py index 012bbc725..6b4960cea 100644 --- a/tests/test_ops_transport.py +++ b/tests/test_ops_transport.py @@ -33,6 +33,9 @@ def test_the_runner_sends_what_it_always_sent(monkeypatch): def run(argv, **kwargs): sent.append(argv) + # Its own stdin: under ``raven acp`` the caller's carries ACP frames, + # and a one-shot ssh that inherited it swallowed them. + assert kwargs["stdin"] is subprocess.DEVNULL return subprocess.CompletedProcess(argv, 0, "out", "") monkeypatch.setattr(transport.subprocess, "run", run) From a33afe490a5883775d4b12509c2641aa7e89b105 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:45:51 +0800 Subject: [PATCH 02/44] feat(*): run an acp sub-agent on a registered machine through its daemon, add it from a chat An acp entry that names a `machine` (and a `remoteCwd`, `~/raven-work` by default) is started there by the machine's Raven daemon, over the link the host keeps to it: its command runs in the machine user's login environment with the entry's env on the link, never on a command line; its session works in `/`, made there for each dispatch; its ACP frames travel on a channel of the link, under a task id, so an ssh that goes mid-turn leaves it running and the turn goes on once the link is back; a cancel ends its whole process tree. The reply says where it ran, by address, and that what it wrote is there. The files that come with a task are named to the agent by address (remote.files_note); nothing is carried over. Raven's keys are not lent to such an entry, and the lane stays debug-only: with it off the entry is left off the table and every door says so in one sentence. The roster names the machine after the entry's tags; the free probe reads the link's state and the last test, never the machine; a test runs the real handshake there and marks a refusal "needs attention" without offering a command to run here. `subagents.add` takes `machine` and `remote_cwd` for an acp preset -- from the page, and from a chat through `raven_config`, which lists the registered machines while the lane is on -- names the row `@`, refuses before anything is spent when the lane is off, the id is not usable, the preset is not acp or a key would be lent, and proves the entry with the same readiness ping as a local add, run on the machine. The list rows carry `machine`; the contract and the generated clients carry the fields. Second of three changes on this design; the one after reads and writes files on a machine by address. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 33 +- raven/acp_client/acp_agent.py | 139 +++- raven/acp_client/capabilities.py | 114 +++- raven/acp_client/pool.py | 87 ++- raven/agent/subagent/backends/__init__.py | 33 +- raven/agent/subagent/manager.py | 34 +- raven/agent/subagent/probe.py | 84 ++- raven/agent/subagent/probe_state.py | 4 + raven/agent/subagent/registry.py | 13 + raven/agent/tools/raven_config.py | 72 +- raven/config/schema.py | 52 ++ raven/config/update_subagents.py | 8 + raven/rpc/methods/subagents.py | 43 +- raven/rpc/models.py | 19 + rpc-schema/openrpc.json | 20 + schemas/subagent.schema.json | 24 + tests/acp_stub_server.py | 19 + tests/test_raven_config_tool.py | 97 +++ tests/test_rpc_subagents.py | 144 ++++ tests/test_subagent_acp.py | 14 +- tests/test_subagent_acp_remote.py | 764 ++++++++++++++++++++++ tests/test_subagent_probe.py | 14 + ui-tui/src/rpc/generated.ts | 6 + ui-web/src/rpc/generated.ts | 6 + 24 files changed, 1763 insertions(+), 80 deletions(-) create mode 100644 tests/test_subagent_acp_remote.py diff --git a/CONTEXT.md b/CONTEXT.md index 7b6a32ecc..7d23f5e69 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -381,8 +381,22 @@ keeping the registry and handing it over: a launcher points the agent's `RAVEN_CONNECTIONS` at the owner's store rather than copying rows into the agent's own home, and a sub-agent inherits `RAVEN_HOME` so it resolves the same file. Which machine a job lands on, and whether it can run at all, is -settled inside the agent that runs it. Nothing on the dispatch path reads the -registry, so no graph and no spawn is ever refused over the state of it. +settled inside the agent that runs it, with one exception: an `acp` agent entry +that names a `machine` is started there by the machine's **Raven node**, reached over +the one ssh link the host keeps to the machine (`raven/node/client.py`) -- a link that +may go and come back while the agent runs on; the row's +address, port, key and user are read for that link alone (`raven/acp_client/remote.py`). +Its dispatch reads the registry once (`remote.machine`), and the run fails before +anything starts when the id is not registered, is listed twice, cannot be used, or +is this computer; the `spawn` itself has already answered that it started. No other +dispatch reads the registry, so no graph and no spawn is ever refused over the state +of it. The address stays in the ssh line, in memory; the entry names only the +machine's id, and the entry's `env` reaches the agent on the link, never on a +command line. Such an entry is added the way any preset is, with `machine` (and +`remote_cwd`) on `subagents.add` — from the page or from a chat through `raven_config`, +which lists the registered machines while the lane is on — refused before anything +is spent when the lane is off or the id is not usable, and proved by the same +readiness ping as a local add, run on that machine through its node. _Avoid_: "host" / "server" / "node" (too broad, no link to the `ops connection` registry that supplies the rows); "GPU box" (only some are GPU hosts, and the term covers any registered compute destination); @@ -424,7 +438,10 @@ The agent table rendered as the text spliced into `spawn`'s and `run_subagent_da descriptions — `name [stateful, local-files, live-progress] (description)`. The model's only account of which agents exist, so an agent absent from it cannot be chosen; the `enum` on the `subagent` parameter constrains the same set. All three capabilities render positive *or* -negative, because "no tag" and "the roster does not say" are indistinguishable otherwise. +negative, because "no tag" and "the roster does not say" are indistinguishable otherwise. An +acp row that names a **Machine** renders a fourth tag after them, `on machine `: the one +conditional tag, since a local row has nothing to say in that position, and the id only, never +the address. Also spliced into the skill gate's prompt under push discovery, where it is the account of what can be delegated and so decides which candidate skills are dropped as already covered; that copy omits the generic `builtin` row, which claims no capability bias and would read as @@ -485,7 +502,15 @@ one the dispatching turn's own tools get. Every dispatch supplies it — `spawn` read), a DAG node takes the same, and a Direct Chat resolves `session_workdir` because its branch returns before `workdir.bind` wraps the turn body. Distinct from **Agent home** (`SubagentManager.workspace`, `~/.raven/workspace`), which holds raven's memory and skills -and is only the fallback for a dispatch that supplied nothing. +and is only the fallback for a dispatch that supplied nothing. An `acp` entry that names a +**Machine** is the exception: its session works on that machine, in +`/` (`remoteCwd` defaults to `~/raven-work`), made and +resolved there by the machine's **Raven node** for each dispatch before the session +opens (`session/dir`); this computer's working directory never reaches the machine. +The files that come with the turn are named to the agent by address (`remote.files_note`); +nothing is copied to the machine, and nothing comes back unasked: what the session made +stays there, named in the reply by address, and `deliver_files` fetches one file by its +remote path on request. _Avoid_: treating the fallback as the default — a sub-agent working in Agent home inspects raven's own memory instead of the user's checkout, and says nothing about having done so. diff --git a/raven/acp_client/acp_agent.py b/raven/acp_client/acp_agent.py index 320ad8c3d..f6fea6d1e 100644 --- a/raven/acp_client/acp_agent.py +++ b/raven/acp_client/acp_agent.py @@ -29,12 +29,14 @@ import time from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any from loguru import logger +from raven.acp_client import remote from raven.acp_client.acp_dialects import AcpDialect, ToolCall, content_texts, dialect_for from raven.acp_client.ask_user import AskUserResponder, clarify_responder from raven.acp_client.capabilities import ( @@ -50,6 +52,7 @@ SESSION_MCP_CAPABILITY, STEER_METHOD, AcpBusyError, + AcpConnectionError, AcpError, AcpRemoteError, AcpTimeoutError, @@ -72,7 +75,9 @@ from raven.agent.tools.snapshot import take as take_snapshot from raven.contracts.subagent_backend import SubagentActionAbortedError from raven.contracts.tool import FileChange +from raven.home import subagent_role_env from raven.mcp.endpoint import McpEndpoints, bridge_command +from raven.ops import remote_path from raven.spine.message import Media if TYPE_CHECKING: @@ -116,6 +121,23 @@ def _uploads_root() -> Path | None: return None +@dataclass(frozen=True) +class _OnMachine: + """One dispatch's place on its machine, resolved through the machine's Raven node.""" + + target: remote.Machine + root: str + """Where the agent process starts there: the entry's ``remoteCwd`` or the default, as written.""" + session_cwd: str + """The session's directory under the root, made there and resolved to an absolute path.""" + since: float + """The machine's clock as the session directory was made: what counts as the session's output comes after.""" + link: Any + """The machine's node pool, for the files sent there and brought back.""" + launcher: Callable[[], Awaitable[Any]] + """Starts the agent through the node; what the pool runs the connection over.""" + + class AcpEmptyTurnError(RuntimeError): """The agent finished a turn without producing anything usable. @@ -241,6 +263,7 @@ def __init__( prompt: str | None = None, workspace: Path | None = None, limit: int | None = None, + track_files: bool = True, ) -> None: # The reply's cap, renewed for every message rather than spent across # the turn: the reply is one message, and narration spending a shared @@ -273,6 +296,10 @@ def __init__( # of its commands produced is only ever visible as a directory that # changed around the call, which is what the per-call listings are for. self._workspace = workspace + # Off for an agent on another machine: its paths name that machine's + # disk, so a listing here would see none of its files, and a path it + # wrote that is absent here would be recorded as a delete. + self._track_files = track_files self._blocks: dict[str, dict[str, list[dict[str, Any]]]] = {} self._listings: dict[str, Any] = {} self._settled_calls: set[str] = set() @@ -442,7 +469,8 @@ async def __call__(self, method: str, params: dict[str, Any]) -> None: } ) self._backfill_subject(str(update.get("toolCallId") or ""), update) - await self._note_files(update) + if self._track_files: + await self._note_files(update) elif kind == "plan": self._plan(update) elif kind == "usage_update": @@ -1013,10 +1041,16 @@ def __init__( allow_mcp_secrets: bool = False, session_mcp: bool = True, pool: Any = None, + machine: str | None = None, + remote_cwd: str | None = None, ) -> None: self.name = name self.command = command self.cwd = cwd + # The registered machine this agent runs on, or None for this computer; + # see ``raven.acp_client.remote`` for what changes when it is set. + self.machine = (machine or "").strip() or None + self.remote_cwd = remote_cwd self.env = env or {} self.ready_timeout_ms = ready_timeout_ms self.timeout = timeout @@ -1338,7 +1372,7 @@ def _session_mcp_refused(self) -> bool: """ from raven.agent.subagent.backends import session_mcp_delivered - return not session_mcp_delivered(session_mcp=self.session_mcp, snapshot=self._snapshot) + return not session_mcp_delivered(session_mcp=self.session_mcp, snapshot=self._snapshot, machine=self.machine) def _mcp_note(self, grant: McpGrant) -> str: """Every secret-free reason this dispatch's MCP is degraded. @@ -1352,7 +1386,12 @@ def _mcp_note(self, grant: McpGrant) -> str: notes = [] if self._session_mcp_refused and grant.granted: names = ", ".join(repr(server.name) for server in grant.granted) - if not self.session_mcp: + if self.machine: + notes.append( + f"MCP servers {names} were not lent to acp agent {self.name!r}: it runs on machine " + f"{self.machine!r}, and the host's servers are reached through a socket on this computer" + ) + elif not self.session_mcp: notes.append( f"MCP servers {names} were withheld because acp agent {self.name!r} is configured as not " f"keeping one session's servers to that session (sessionMcp: false); raven pools one " @@ -1432,7 +1471,9 @@ async def _mcp_endpoints(self, node_id: str, grant: McpGrant) -> AsyncIterator[l "acp agent {!r}: withholding {} MCP server(s); {}", self.name, len(grant.granted), - "configured as not keeping a session's servers to that session (sessionMcp: false)" + f"it runs on machine {self.machine!r}" + if self.machine + else "configured as not keeping a session's servers to that session (sessionMcp: false)" if not self.session_mcp else f"a raven build with no {SESSION_MCP_CAPABILITY} declaration", ) @@ -1510,8 +1551,22 @@ async def run( # it, so it must be the caller's workspace whatever the entry pinned. # Conflating them sent a pinned entry's own folder to `session/new`, and # the agent then edited that tree instead of the caller's. - launch_cwd = self.cwd or str(workspace) - session_cwd = str(workspace) + # An agent on another machine has neither: its machine's Raven node + # starts it in the entry's root there, and the session works in a + # directory made under that root (`_on_machine`), never in this + # computer's workspace, which does not exist there. + launched = await self._on_machine(handle) if self.machine else None + if launched is not None: + launch_cwd = launched.root + session_cwd = launched.session_cwd + else: + launch_cwd = self.cwd or str(workspace) + session_cwd = str(workspace) + # Nothing is carried to the machine: the files that come with the + # task are named to the agent by address, and it looks for them there. + sent_text = task + if launched is not None and (note := remote.files_note(media, launched.target)): + sent_text = f"{task}\n\n{note}" started = time.monotonic() grant = mcp_grant if mcp_grant is not None else await self.resolve_mcp_grant_async(mcps) async with self._mcp_endpoints(task_id, grant) as mcp_servers: @@ -1533,13 +1588,16 @@ async def run( # binding gets a connection of its own and the two coexist. binding: dict[str, str] = {} - if model: + if model and launched is None: binding["RAVEN_PARENT_MODEL"] = model parent_provider = str(getattr(provider, "provider_name", "") or "") - if parent_provider: + # Not for an agent on another machine: the binding is launch + # environment for a process of raven's own, and there it would + # start a second agent process per parent model for nothing. + if parent_provider and launched is None: binding["RAVEN_PARENT_PROVIDER"] = parent_provider parent_protocol = str(getattr(provider, "api_protocol", "") or "") - if parent_protocol: + if parent_protocol and launched is None: binding["RAVEN_PARENT_PROTOCOL"] = parent_protocol # One of raven's own with no pin of its own follows the parent. # The binding above puts a fresh worker on the parent's model at @@ -1562,6 +1620,10 @@ async def run( env=dict(self.env), binding=binding or None, ready_timeout_s=budget, + # On a machine the node starts the process (`launcher`); + # the command, root and env still key the connection. + machine=self.machine, + launcher=launched.launcher if launched is not None else None, ) client = connection.client # Marked after acquire, so a call's range covers its own traffic and @@ -1609,8 +1671,9 @@ async def run( on_delta, dialect_for(connection.initialize), prompt=task, - workspace=Path(session_cwd), + workspace=Path(session_cwd) if launched is None else None, limit=self.max_output_chars, + track_files=launched is None, ) # Built here, in the turn's context, for the reason `_TurnCollector` # documents: the read loop's ContextVars predate this run, and the @@ -1647,8 +1710,11 @@ async def run( # text, the block an editor sends for an @-mentioned # file; see raven.agent.subagent.attachments. "prompt": [ - {"type": "text", "text": task}, - *attachment_blocks(media, root=_uploads_root()), + {"type": "text", "text": sent_text}, + # Links to this computer's files, which an agent + # on another machine cannot open; the manager has + # already told the caller they were not sent. + *(attachment_blocks(media, root=_uploads_root()) if launched is None else ()), ], }, timeout=_prompt_deadline(self.timeout), @@ -1758,7 +1824,18 @@ async def run( await self._registry.commit(skey, self.name, handle, session_id, kind="acp") reply = await self._finished(collector.reply, stop_reason=stop_reason, span=span, sink=on_delta) - if note := self._mcp_note(grant): + notes = [self._mcp_note(grant)] + if launched is not None: + # Where to find what it made: the caller's workspace line names + # this computer, and what it wrote is there, by address -- + # nothing is brought back unasked. + notes.append( + f"Ran on machine {launched.target.label}, in " + f"{remote.shown(launched.target, session_cwd)}; the files it wrote are there, " + "not on this computer -- read_file, list_dir, find, grep, write_file and edit_file take " + "that path as it is, and deliver_files fetches one by it" + ) + for note in filter(None, notes): notice = f"\n\n[raven] {note}." activity.append_closing(notice) if on_delta is not None: @@ -1766,6 +1843,42 @@ async def run( reply += notice return reply + async def _on_machine(self, handle: str) -> _OnMachine: + """This dispatch's place on its machine, or a connection error that says why not. + + The session directory is made fresh for every dispatch, never cached: + one removed on the machine between two dispatches would otherwise be + named to ``session/new`` and refused there, and the round trip costs + milliseconds on the link against a turn. The launcher is handed to the + pool, which calls it only when no connection of this entry is live. + """ + try: + target = await asyncio.to_thread(remote.machine, self.machine or "") + except remote.RemoteMachineError as exc: + raise AcpConnectionError(f"acp agent {self.name!r}: {exc}") from None + link = remote_path.nodes() + if link is None: + raise AcpConnectionError( + f"acp agent {self.name!r}: this process does not reach registered machines, so it cannot run on " + f"machine {target.label}" + ) + root = remote.root_of(self.remote_cwd) + try: + session_cwd, since = await link.session_dir(target.id, remote.session_root(self.remote_cwd, handle)) + except remote_path.NodeCallError as exc: + raise AcpConnectionError(f"acp agent {self.name!r}: {exc}") from None + # The role goes on before the entry's own map, as the local launch + # puts it; the host's identity stays here, where it means something. + env = {**subagent_role_env(), **self.env} + + async def launcher() -> Any: + try: + return await link.open_channel(target.id, name=self.name, command=self.command, cwd=root, env=env) + except remote_path.NodeCallError as exc: + raise AcpConnectionError(f"acp agent {self.name!r}: {exc}") from None + + return _OnMachine(target=target, root=root, session_cwd=session_cwd, since=since, link=link, launcher=launcher) + @property def can_steer(self) -> bool: """Whether this agent takes text mid-turn (``_raven/session/steer``). diff --git a/raven/acp_client/capabilities.py b/raven/acp_client/capabilities.py index 5e0bead20..6056c77df 100644 --- a/raven/acp_client/capabilities.py +++ b/raven/acp_client/capabilities.py @@ -22,6 +22,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import json import os @@ -35,10 +36,12 @@ from loguru import logger -from raven.acp_client import protocol +from raven.acp_client import protocol, remote from raven.acp_client.client import AcpClient from raven.acp_client.permissions import auto_approver from raven.acp_client.protocol import SESSION_MCP_CAPABILITY, STEER_CAPABILITY, AcpError, AcpRemoteError, reason_of +from raven.home import subagent_role_env +from raven.ops import remote_path from raven.utils.atomic_io import atomic_update from raven.utils.commands import command_argv @@ -57,6 +60,11 @@ _LAUNCH_FIELDS = ("command", "cwd", "env", "ready_timeout_ms") +#: Where an acp agent runs, when that is a registered machine rather than here. +#: Folded into launch digests only when set, so a digest recorded before these +#: fields existed still matches the row it was taken from. +REMOTE_LAUNCH_FIELDS = ("machine", "remote_cwd") + # Substrings that mark a remote error as "the operator has to authenticate", # which is a different action from "this thing is not reachable". Matched on the # message because ACP has no typed auth-required error: measured against @@ -319,6 +327,10 @@ def snapshot_fingerprint(cfg: Any) -> str: # Only when set, so every snapshot measured before the field existed holds. if getattr(cfg, "lend_keys", None): payload["lend_keys"] = list(cfg.lend_keys) + # Same rule: a row moved to another machine is another agent install. + for name in REMOTE_LAUNCH_FIELDS: + if getattr(cfg, name, None): + payload[name] = getattr(cfg, name) raw = json.dumps(payload, sort_keys=True, default=str) return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] @@ -785,6 +797,41 @@ def npx_fetch_lead(verdict: str) -> str: return f"npx could not download it ({verdict}); check the network, the npm registry or the proxy" +class _NotThereError(Exception): + """A machine an agent cannot be verified on, in one sentence.""" + + +@dataclass(frozen=True) +class _OnMachine: + """The link to a machine, with the agent's command confirmed to be there.""" + + link: Any + target: remote.Machine + + +async def _reach(machine_id: str, command: str, env: dict[str, str]) -> _OnMachine: + """The machine's node, after it has found ``command`` there; :class:`_NotThereError` says why not. + + One round trip on the link (``probe/agent``) before the real handshake, + so "not installed there" is told apart from "installed and broken" + without starting anything. + """ + try: + target = await asyncio.to_thread(remote.machine, machine_id) + except remote.RemoteMachineError as exc: + raise _NotThereError(str(exc)) from None + link = remote_path.nodes() + if link is None: + raise _NotThereError(f"machine {target.label}: this process does not reach registered machines") + try: + probed = await link.probe_agent(machine_id, command, env) + except remote_path.NodeCallError as exc: + raise _NotThereError(str(exc)) from None + if not probed.get("found"): + raise _NotThereError(remote.agent_missing(target)) + return _OnMachine(link, target) + + async def verify_agent(cfg: Any, *, env: dict[str, str] | None = None) -> CapabilitySnapshot: """Connect once, read what the agent reports, and disconnect. Never raises. @@ -843,19 +890,42 @@ def done( elapsed_ms=int((time.monotonic() - started) * 1000), ) + agent_env = dict(env if env is not None else (getattr(cfg, "env", None) or {})) + command = getattr(cfg, "command", "") or "" + machine_id = (getattr(cfg, "machine", None) or "").strip() + there: _OnMachine | None = None + if machine_id: + # An agent on a registered machine is verified where it runs, through + # the machine's Raven node as a dispatch starts it, after the node has + # said the command is there. Every failure there is "attention", never + # "missing": the roster answers "missing" with an installer, and that + # installer would run on this computer. + try: + there = await _reach(machine_id, command, agent_env) + except _NotThereError as exc: + return done("attention", str(exc)) client: AcpClient | None = None try: - client = await AcpClient.launch( - name=name, - command=getattr(cfg, "command", "") or "", - cwd=getattr(cfg, "cwd", None), - env=dict(env if env is not None else (getattr(cfg, "env", None) or {})), - # Nothing here is prompted, so no permission request is expected. - # One that arrives anyway still has to be answered, or the agent - # waits for a reply that never comes and the handshake stalls behind - # its own timeout. - on_request=auto_approver(name), - ) + # Nothing here is prompted, so no permission request is expected. One + # that arrives anyway still has to be answered, or the agent waits for + # a reply that never comes and the handshake stalls behind its own + # timeout. + if there is not None: + try: + endpoint = await there.link.open_channel( + machine_id, + name=name, + command=command, + cwd=remote.root_of(getattr(cfg, "remote_cwd", None)), + env={**subagent_role_env(), **agent_env}, + ) + except remote_path.NodeCallError as exc: + return done("attention", str(exc)) + client = AcpClient.over(endpoint, name=name, on_request=auto_approver(name)) + else: + client = await AcpClient.launch( + name=name, command=command, cwd=getattr(cfg, "cwd", None), env=agent_env, on_request=auto_approver(name) + ) try: init = await client.request("initialize", protocol.initialize_params(), timeout=budget) except AcpRemoteError as exc: @@ -866,6 +936,10 @@ def done( except AcpError as exc: tail = client.stderr_tail(400) suffix = f"; stderr: {tail}" if tail else "" + if there is not None: + # In the far side's own words (a full disk, a crash), never + # "missing": whatever is absent is absent there. + return done("attention", f"handshake failed: {exc}{suffix}") unfetched = npx_fetch_failure(getattr(cfg, "command", "") or "", exc, client.stderr_tail() or None) if unfetched is not None: # Not `missing`: nothing is absent that an install would bring, and @@ -881,7 +955,19 @@ def done( f"agent speaks ACP v{handshake.protocol_version}, raven speaks v{protocol.PROTOCOL_VERSION}" ) - with tempfile.TemporaryDirectory(prefix="raven_acp_verify_") as tmp: + with contextlib.ExitStack() as scratch: + if there is None: + tmp = scratch.enter_context(tempfile.TemporaryDirectory(prefix="raven_acp_verify_")) + else: + # On the machine, the one directory every check reuses: a local + # temp dir does not exist where the agent runs, and it refuses one. + try: + tmp, _since = await there.link.session_dir( + machine_id, + remote.session_root(getattr(cfg, "remote_cwd", None), remote.CHECK_HANDLE), + ) + except remote_path.NodeCallError as exc: + return done("attention", str(exc), handshake) try: session = await client.request("session/new", {"cwd": tmp, "mcpServers": []}, timeout=budget) except AcpRemoteError as exc: @@ -931,6 +1017,8 @@ def done( ) return done("ready", detail, handshake) except AcpError as exc: + if there is not None: + return done("attention", str(exc)) return done("missing", str(exc)) except Exception as exc: # noqa: BLE001 - a raising verify would blank the page logger.opt(exception=True).warning("acp verify for {!r} failed unexpectedly: {}", name, exc) diff --git a/raven/acp_client/pool.py b/raven/acp_client/pool.py index 73c008c19..fb39f8f8b 100644 --- a/raven/acp_client/pool.py +++ b/raven/acp_client/pool.py @@ -78,11 +78,16 @@ # hold the two in step: a parameter added to `acquire` and not to the key would # be silently discarded for the life of the process, which is the exact bug this # key exists to prevent. -LAUNCH_PARAMS = ("command", "cwd", "env", "binding") +LAUNCH_PARAMS = ("command", "cwd", "env", "binding", "machine") def launch_key( - *, command: str, cwd: str | None, env: dict[str, str] | None, binding: dict[str, str] | None = None + *, + command: str, + cwd: str | None, + env: dict[str, str] | None, + binding: dict[str, str] | None = None, + machine: str | None = None, ) -> str: """A digest of what a connection was launched from. @@ -95,10 +100,14 @@ def launch_key( `binding` is launch environment too, so it is in the key; the pool keeps it apart from `env` because the two are answered differently (see `acquire`). + `machine` is where the process runs: a row moved to another machine, or + back to this computer, is another process. Folded in only when set, so no + key of a local agent changes. """ - raw = json.dumps( - {"command": command, "cwd": cwd, "env": env or {}, "binding": binding or {}}, sort_keys=True, default=str - ) + params: dict[str, Any] = {"command": command, "cwd": cwd, "env": env or {}, "binding": binding or {}} + if machine: + params["machine"] = machine + raw = json.dumps(params, sort_keys=True, default=str) return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] @@ -387,6 +396,8 @@ async def acquire( binding: dict[str, str] | None = None, on_request: Any = None, ready_timeout_s: float | None = None, + machine: str | None = None, + launcher: Callable[[], Awaitable[Any]] | None = None, ) -> _Connection: """The live connection for ``name`` under ``binding``, starting or restarting it if needed. @@ -407,11 +418,18 @@ async def acquire( invoking session's model/provider/protocol, and two sessions on two bindings dispatch to the same agent at once. Each binding holds its own connection under the name, and a new binding leaves the others' turns - running. A change to the config half (`command`, `cwd`, `env`) still - retires every connection of the name, as it always did. + running. A change to the config half (`command`, `cwd`, `env`, + `machine`) still retires every connection of the name, as it always did. + + ``launcher`` opens the endpoint the connection runs over when the + process is not started here: an agent on a registered machine is + started by that machine's Raven node, and the launcher is the call + that asks it to (``NodePool.open_channel``). The launch arguments + still describe the process, so the key covers them as it covers a + local one's; ``machine`` says where. """ - config = launch_key(command=command, cwd=cwd, env=env) - key = launch_key(command=command, cwd=cwd, env=env, binding=binding) + config = launch_key(command=command, cwd=cwd, env=env, machine=machine) + key = launch_key(command=command, cwd=cwd, env=env, binding=binding, machine=machine) bkey = _binding_key(binding) launch_env = {**(env or {}), **binding} if binding else env async with self._lock_for(name): @@ -440,26 +458,39 @@ async def acquire( # `initialize` exchange belongs to the connection, not to whichever # call happened to be the one that started it. journal = open_journal(name) - client = await AcpClient.launch( - name=name, - command=command, - cwd=cwd, - env=launch_env, - # Defaulted here rather than at the caller: answering a - # permission request is a property of raven-as-an-ACP-client, - # not of whichever backend happens to hold the turn, and every - # caller leaving it unset is how an unanswered request came to - # cancel turns. - on_request=on_request - if on_request is not None - else request_dispatcher(name, router.dispatch, elicitors=elicitors), - # Wrapped rather than routed straight through: an - # `ask_user_request` is a question raven has to answer with a - # request of its own, and a sink returns nothing. See - # `ask_user.notification_dispatcher` -- everything still routes. - on_notification=notification_dispatcher(name, router.dispatch, responders=responders), - journal=journal, + # Defaulted here rather than at the caller: answering a + # permission request is a property of raven-as-an-ACP-client, + # not of whichever backend happens to hold the turn, and every + # caller leaving it unset is how an unanswered request came to + # cancel turns. + answering = ( + on_request if on_request is not None else request_dispatcher(name, router.dispatch, elicitors=elicitors) ) + # Wrapped rather than routed straight through: an + # `ask_user_request` is a question raven has to answer with a + # request of its own, and a sink returns nothing. See + # `ask_user.notification_dispatcher` -- everything still routes. + notifying = notification_dispatcher(name, router.dispatch, responders=responders) + if launcher is not None: + try: + endpoint = await launcher() + except BaseException: + if journal is not None: + journal.close() + raise + client = AcpClient.over( + endpoint, name=name, on_request=answering, on_notification=notifying, journal=journal + ) + else: + client = await AcpClient.launch( + name=name, + command=command, + cwd=cwd, + env=launch_env, + on_request=answering, + on_notification=notifying, + journal=journal, + ) # The handshake belongs to establishing the connection, not to the # first caller: ACP has no usable state before `initialize`, and an # agent is entitled to reject anything sent ahead of it. Two of the diff --git a/raven/agent/subagent/backends/__init__.py b/raven/agent/subagent/backends/__init__.py index c3b43f020..03b55e63b 100644 --- a/raven/agent/subagent/backends/__init__.py +++ b/raven/agent/subagent/backends/__init__.py @@ -94,6 +94,14 @@ class AgentMeta(NamedTuple): fields up: this is a NamedTuple several callers build positionally, so a field inserted anywhere but the end moves one of theirs.""" + machine: str = "" + """The registered machine this agent runs on, by id, or ``""`` for this computer. + + Rendered as a tag so the model can send work to a named machine and knows + the files it gets back are there, not here. The id only: the registry keeps + the address out of anything a model reads. Appended last, for the reason + ``model_choices`` gives.""" + def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: """The advertised capabilities of one agent config, any kind. @@ -171,11 +179,14 @@ def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: stateful = bool(getattr(cfg, "stateful", True)) else: stateful = bool(getattr(cfg, "resume_command", None)) + machine = str(getattr(cfg, "machine", None) or "") if kind == "acp" else "" return AgentMeta( getattr(cfg, "name", "") or "", getattr(cfg, "description", "") or "", stateful, - bool(getattr(cfg, "reads_local_files", True)), + # An agent on another machine reads that machine's disk, so a path here + # means nothing to it however its kind would otherwise answer. + bool(getattr(cfg, "reads_local_files", True)) and not machine, # Only the acp transport carries a live event stream. This is a fact about # the transport, not about the agent, so it is read from `kind` rather # than from anything the agent or the operator says. @@ -184,6 +195,7 @@ def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: modes, bool(getattr(cfg, "owns_watched_work", False)), model_choices, + machine, ) @@ -192,7 +204,7 @@ def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: _STALE_SNAPSHOT_SEEN: set[tuple[str, str]] = set() -def session_mcp_delivered(*, session_mcp: bool, snapshot: Any) -> bool: +def session_mcp_delivered(*, session_mcp: bool, snapshot: Any, machine: str | None = None) -> bool: """Whether a granted server would actually reach this agent's session. The one predicate behind two questions that must not answer differently: what @@ -205,8 +217,12 @@ def session_mcp_delivered(*, session_mcp: bool, snapshot: Any) -> bool: :func:`~raven.agent.subagent.presets.session_mcp_for`; ``snapshot`` carries the transitional half. Both are spelled out in ``AcpAgentBackend._session_mcp_refused``, which is the reader for why each - exists and why only one of them is temporary. + exists and why only one of them is temporary. ``machine`` is the third + reason, and not the peer's: the host's servers are reached through a + socket on this computer, which an agent on another machine cannot open. """ + if machine: + return False if not session_mcp: return False if snapshot is None or snapshot.session_mcp: @@ -222,7 +238,9 @@ def session_mcp_effective(cfg: Any, *, snapshot: Any = None) -> bool: """ if snapshot is None: snapshot = acp_snapshot_for(cfg) - return session_mcp_delivered(session_mcp=session_mcp_for(cfg), snapshot=snapshot) + return session_mcp_delivered( + session_mcp=session_mcp_for(cfg), snapshot=snapshot, machine=getattr(cfg, "machine", None) + ) def acp_snapshot_for(cfg: Any) -> Any: @@ -281,6 +299,7 @@ def format_agent_listing(meta: Sequence[AgentMeta]) -> str: "stateful" if entry.stateful else "stateless", "local-files" if entry.reads_local_files else "no-local-files", "live-progress" if entry.live_progress else "no-progress", + *((f"on machine {entry.machine}",) if entry.machine else ()), ) ) head = f"{entry.name} [{tags}]" @@ -396,7 +415,9 @@ def build_third_party_backend( name=cfg.name, command=cfg.command, cwd=cfg.cwd, - env={**lent_key_env(cfg), **cfg.env}, + # Nothing lent to an agent on another machine: the schema drops such + # a key on load, and this keeps the two from disagreeing. + env={**({} if getattr(cfg, "machine", None) else lent_key_env(cfg)), **cfg.env}, ready_timeout_ms=cfg.ready_timeout_ms if ready_timeout_ms is None else ready_timeout_ms, timeout=cfg.timeout if timeout is None else timeout, max_output_chars=cfg.max_output_chars, @@ -406,6 +427,8 @@ def build_third_party_backend( allow_mcp_secrets=cfg.allow_mcp_secrets, session_mcp=session_mcp_for(cfg), pool=pool, + machine=getattr(cfg, "machine", None), + remote_cwd=getattr(cfg, "remote_cwd", None), ) if kind == "openai": return OpenAIApiBackend( diff --git a/raven/agent/subagent/manager.py b/raven/agent/subagent/manager.py index 6721b8eeb..67f274682 100644 --- a/raven/agent/subagent/manager.py +++ b/raven/agent/subagent/manager.py @@ -1175,10 +1175,17 @@ async def chat( self.remember_session(session_key) self._require_addressable(agent, doing=f"chat with instance {handle!r}") row = self.registry.get(agent) - if media and row is not None and not row.caps.reads_local_files: + # A row on a registered machine cannot read a path here either, but its + # attachments are sent there by the backend, which tells the agent + # where they landed in its own terms; so they are kept, and the + # bullets are left as the page wrote them rather than pointed at this + # computer's disk. + on_machine = bool(row is not None and row.caps.machine) + if media and row is not None and not row.caps.reads_local_files and not on_machine: text = with_undeliverable_note(text, media) media = () - text = retarget_note(text, media, self.workspace) + if not on_machine: + text = retarget_note(text, media, self.workspace) recorded = with_attachment_note(text, media) session_dir = self.session_dir_for(session_key) @@ -1736,6 +1743,12 @@ async def _run_subagent_inner( agent = origin.get("agent") or GENERIC_AGENT handle = origin.get("handle") or task_id effective_workspace = origin.get("workspace") or self.workspace + # A run on a registered machine works there, in a directory the + # machine makes; the record names it as written, since this computer's + # workspace is not where anything of the run is. + machine, where = self._machine_workdir(agent, handle) + if machine: + effective_workspace = f"{machine}:{where}" # Opened before dispatch so a call that never returns still leaves its # input on disk. Rooted at the session's metadata directory, not at # effective_workspace: the record has to outlive whatever the working @@ -2041,6 +2054,16 @@ def _emit_status( payload["ended_at"] = ended_at self._emit_event(origin["session_key"], {"type": "subagent.status", "payload": payload}) + def _machine_workdir(self, agent: str, handle: str) -> tuple[str, str]: + """``(machine, /)`` for an agent on a registered machine, else two empty strings.""" + row = self.registry.get(agent) + machine = row.caps.machine if row is not None else "" + if not machine: + return "", "" + from raven.acp_client import remote + + return machine, remote.session_root(getattr(row.config, "remote_cwd", None), handle) + async def _announce_result( self, task_id: str, @@ -2085,6 +2108,13 @@ async def _announce_result( # Where the run was dispatched to work, so the parent can find what it # left there without searching the disk for it. workdir_line = f"Working directory: {origin['workspace']}\n" if origin.get("workspace") else "" + machine, where = self._machine_workdir(origin.get("agent") or "", origin.get("handle") or task_id) + if machine: + # Not this computer's directory: the agent worked on its machine, + # in a directory made there when the run started. Named here on + # every path, since a failed or cancelled run's result does not + # close with it the way a finished one's does. + workdir_line = f"Working directory: on machine {machine!r}, {where} (made there when the run started)\n" record_line = f"\n\nRecord: {record_path}" if record_path else "" # How the run's calls went, which nothing on this path carried. Measured # on a real dispatch: two of three calls timed out, the run produced no diff --git a/raven/agent/subagent/probe.py b/raven/agent/subagent/probe.py index 8f6923fbf..ba33849a8 100644 --- a/raven/agent/subagent/probe.py +++ b/raven/agent/subagent/probe.py @@ -312,6 +312,19 @@ def _refusal(cfg: Any, said: str, answer: str | None = None) -> tuple[str, Remed return f"{lead}; {advice}. It said: {said}"[:_DETAIL_CAP], remedy +def _on_machine(text: str, machine: str) -> str: + """``text`` closed with where any fix is made, kept whole inside the cap however long ``text`` is. + + Except for the one fix made here: the machine's Raven node, which this + computer installs (``install-node``), and whose absence ``text`` already + answers with that command. + """ + if "install-node" in text: + return text[:_DETAIL_CAP] + note = f" (on machine {machine!r}: any fix is made there)" + return text[: _DETAIL_CAP - len(note)].rstrip() + note + + def _remedy_for(cfg: Any) -> Remedy: """What fixes a refusal already known to be about a credential, on this machine.""" if getattr(cfg, "kind", None) == "openai": @@ -588,6 +601,8 @@ def done(status: ProbeStatus, detail: str, target: str = "") -> ProbeResult: return done("unknown", f"command cannot be parsed: {exc}") if not argv: return done("unknown", "command is empty") + if machine_id := (getattr(cfg, "machine", None) or "").strip(): + return _probe_remote_acp(cfg, machine_id, done) exe = argv[0] cfg_path = (getattr(cfg, "env", None) or {}).get("PATH") resolved = shutil.which(exe, path=cfg_path or path) @@ -865,6 +880,13 @@ def _ping_bounds(cfg: Any) -> tuple[int, int, float]: return ready_ms, answer, ready_ms / 1000 + answer +def remote_check_handle() -> str: + """The handle every check of an agent on a registered machine runs under.""" + from raven.acp_client import remote + + return remote.CHECK_HANDLE + + def _ping_refusal(cfg: Any, exc: BaseException) -> tuple[str, Remedy | None]: """What a ping that raised is reported as, and the fix when one is known. @@ -914,6 +936,38 @@ class PingResult: remedy: Remedy | None = None +def _probe_remote_acp(cfg: Any, machine_id: str, done: Any) -> ProbeResult: + """The free check for an acp agent on a registered machine: still registered, and verified? + + No ssh: this runs on every listing, and one machine slow to answer would hold + the whole roster behind it. ``which`` here would answer for this computer, + which is not where the agent runs. What can be said for free is whether the + registry still names a usable machine, and what the last test recorded. + """ + from raven.acp_client import remote + from raven.ops import remote_path + + try: + target = remote.machine(machine_id) + except remote.RemoteMachineError as exc: + return done("attention", str(exc), machine_id) + where = f"runs on machine {target.label}" + # The link's state is known here without a round trip: connected and + # how many agents it carries, idle, or the sentence of its last failure. + if (link := remote_path.nodes()) is not None and (said := link.status(machine_id).said()): + where += f" ({said})" + snapshot = acp_snapshot_for(cfg) + if snapshot is None: + return done("attention", f"{where}; its ACP capabilities have not been recorded yet -- run a test", machine_id) + if snapshot.stale: + return done( + "attention", f"{where}, but its launch config changed since the last test -- run a test", machine_id + ) + if not getattr(snapshot, "model_menu_measured", True): + return done("attention", f"{where}, but its model menu has not been measured yet -- run a test", machine_id) + return done(snapshot.status, snapshot.detail, machine_id) + + async def ping_agent(cfg: Any) -> PingResult: """Send one prompt and report whether the agent answered. Never raises. @@ -945,6 +999,7 @@ async def ping_agent(cfg: Any) -> PingResult: from raven.acp_client import pool as acp_pool ready_ms, answer_s, wait_s = _ping_bounds(cfg) + machine = (getattr(cfg, "machine", None) or "").strip() pool = acp_pool.AcpConnectionPool() reply: str | None = None failure: Exception | None = None @@ -964,6 +1019,10 @@ async def ping_agent(cfg: Any) -> PingResult: # model -- the fix for a default its provider refuses -- was judged # by the model it was pinned away from. pinned = optional_keyword(backend, "session_model", getattr(cfg, "model", None) or None) + # On a registered machine the session directory is named by the + # handle; one shared by every check leaves one directory there + # rather than one per press. + checked = {"instance": remote_check_handle()} if machine else {} reply = await asyncio.wait_for( backend.run( PROBE_PROMPT, @@ -971,12 +1030,14 @@ async def ping_agent(cfg: Any) -> PingResult: workspace=Path(tmp), executor=None, **pinned, + **checked, ), timeout=wait_s, ) except asyncio.TimeoutError: said = f"it did not answer within {wait_s:.0f}s" - run = diagnose_hint_for(cfg) + # A diagnosis names a command to run here, and the agent is not here. + run = None if machine else diagnose_hint_for(cfg) return PingResult(False, _silent_detail(said, run), Remedy("silent", run)) if run else PingResult(False, said) except Exception as exc: # noqa: BLE001 - every failure is the answer, not a crash failure = exc @@ -992,6 +1053,14 @@ async def ping_agent(cfg: Any) -> PingResult: if github_copilot.applies(cfg) and (named := github_copilot.read(reply)) is not None: return PingResult(False, named[0][:_DETAIL_CAP], named[1]) return PingResult(True, "it ran and replied") + if machine: + # The local follow-ups below -- asking the agent again in print mode, + # reading this computer's node, a fix command offered to run -- all act + # on this computer, and this agent is on another one. + if failure is None: + return PingResult(False, "it started and then answered nothing") + text, _fix = _refusal(cfg, *_said(failure)) + return PingResult(False, _on_machine(text, machine)) # An agent whose ACP answer leaves the reason out is asked for it its own way, # once the pool is closed, so the process asked is not racing the one pinged. if (explained := await kimi_code.explain(cfg, failure, prompt=PROBE_PROMPT)) is not None: @@ -1042,6 +1111,14 @@ async def _test_acp(cfg: Any, *, source: Source, elapsed: Any) -> TestResult: # minute, while the menu and the statefulness it holds are launch facts. snapshot = await record_capabilities(cfg) reply = ", ".join(snapshot.available_models[:5]) or None + if not snapshot.usable and (machine := (getattr(cfg, "machine", None) or "").strip()): + # The follow-ups below -- asking the agent again in print mode, a fix + # command resolved on this computer's PATH, a crash read off this + # computer's node -- act here, and the agent is there. Recorded with no + # remedy: the one that applies is made on the machine, and a sign-in + # command the page offered to copy would fix this computer. + detail = _on_machine(snapshot.detail, machine) if snapshot.needs_auth else snapshot.detail + return TestResult(cfg.name, source, "acp", False, detail[:_DETAIL_CAP], reply, elapsed(), None) if not snapshot.usable: # The handshake already classified the refusal (`needs_auth`, from the # agent's own answer), so the remedy is read off that verdict rather than @@ -1352,6 +1429,11 @@ async def _verify_missing_snapshots(manager: Any, rows: list[Any], *, configured cfg = getattr(row, "config", None) if cfg is None: continue + if (getattr(cfg, "machine", None) or "").strip(): + # A verify of a row on a registered machine logs in there and + # starts the agent, and an unattended boot does neither: the + # listing says "run a test", and that press is the owner's own. + continue try: snapshot = acp_snapshot_for(cfg) name = getattr(row, "name", "") or "" diff --git a/raven/agent/subagent/probe_state.py b/raven/agent/subagent/probe_state.py index dc6da252f..de3d62b33 100644 --- a/raven/agent/subagent/probe_state.py +++ b/raven/agent/subagent/probe_state.py @@ -197,6 +197,10 @@ def fingerprint(cfg: Any) -> str: # set, so every verdict recorded before the field existed holds. if getattr(cfg, "lend_keys", None): payload["lend_keys"] = list(cfg.lend_keys) + # The machine an acp row runs on, by the same only-when-set rule. + for field in ("machine", "remote_cwd"): + if getattr(cfg, field, None): + payload[field] = getattr(cfg, field) raw = json.dumps(payload, sort_keys=True, default=str) return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] diff --git a/raven/agent/subagent/registry.py b/raven/agent/subagent/registry.py index 7b67ee4c7..0ccbce63d 100644 --- a/raven/agent/subagent/registry.py +++ b/raven/agent/subagent/registry.py @@ -40,6 +40,7 @@ from raven.agent.subagent.builtin_agents import LEGACY_AGENT_ALIASES, canonical_agent_name, merge_builtin_seeds from raven.agent.subagent.instances import get_registry from raven.agent.subagent.vendored_agents import discover_product_rows, merge_product_seeds +from raven.home import machine_lane_enabled if TYPE_CHECKING: from raven.agent.subagent.backends.routing import Router, TargetReady @@ -69,6 +70,9 @@ class AgentCaps: A menu rather than a capability, so unlike the three above it is not rendered as a roster tag -- see ``AgentMeta.modes``.""" + machine: str = "" + """The registered machine the agent runs on, or ``""`` for this computer + (acp only) -- see ``AgentMeta.machine``.""" @dataclass(frozen=True) @@ -152,6 +156,7 @@ def meta(self) -> AgentMeta: self.caps.modes, self.owns_watched_work, self.caps.model_choices, + self.caps.machine, ) @@ -189,6 +194,7 @@ def _row_for(cfg: Any) -> AgentRow: live_progress=meta.live_progress, modes=meta.modes, model_choices=meta.model_choices, + machine=meta.machine, ), injectable=Injectable(skills=builtin, mcps=injectable_mcps), owns=meta.owns, @@ -304,6 +310,13 @@ def apply(self, configs: Sequence[Any] | None) -> None: merged = merge_product_seeds(list(configs or []), discover_product_rows()) for cfg in merge_builtin_seeds(merged): name = getattr(cfg, "name", None) + if (getattr(cfg, "machine", None) or "").strip() and not machine_lane_enabled(): + # The machine lane is debug-only: an entry on a registered + # machine stays off the table, so the model is never offered a + # dispatch the host would then refuse. The listing still shows + # the row, and its probe says the same. + logger.info("Sub-agent {!r} stays off the table: the machine lane is debug-only", name) + continue try: row = _row_for(cfg) if not row.name: diff --git a/raven/agent/tools/raven_config.py b/raven/agent/tools/raven_config.py index 425ca06a8..2759d41ad 100644 --- a/raven/agent/tools/raven_config.py +++ b/raven/agent/tools/raven_config.py @@ -194,7 +194,7 @@ def _model_check(value: str, row: dict[str, Any]) -> str: return f"The agent does not list it ({', '.join(choices[:12])}); it may refuse it -- pick one of those." -_ADD_KEYS = ("preset", "name", "description", "model", "lend_key") +_ADD_KEYS = ("preset", "name", "description", "model", "lend_key", "machine", "remote_cwd") def _add_params(value: dict[str, Any]) -> dict[str, Any]: @@ -241,11 +241,28 @@ def _lend_line(preset: str, name: str | None, can_lend: list[str]) -> str: def _what_it_needs( - preset: str, row: dict[str, Any], *, remedy: dict[str, Any] | None, detail: str, model: str + preset: str, + row: dict[str, Any], + *, + remedy: dict[str, Any] | None, + detail: str, + model: str, + machine: str | None = None, ) -> list[str]: - """The next step for one agent that did not answer, read off its refusal's remedy.""" + """The next step for one agent that did not answer, read off its refusal's remedy. + + On a registered machine there is no next step here: the agent, its install + and its sign-in are all on that machine, which exec does not reach. + """ from raven.agent.subagent.presets import DIAGNOSE_HINTS + if machine: + lines = [ + f"- It runs on machine {machine!r}, so every fix is made there by the user, in a terminal on that " + "machine: install, sign-in or setup, as the refusal says. Nothing here reaches it (exec refuses " + "ssh to a registered machine). Tell the user what it said, then retry once they have." + ] + return lines + _menu_line(row, model) remedy = remedy or {} kind = str(remedy.get("kind") or "") fallback = { @@ -278,10 +295,40 @@ def _what_it_needs( if run else "- Run it on its own with exec in its one-shot mode (its --help names it); that prints the real error." ] + return lines + _menu_line(row, model) + + +def _menu_line(row: dict[str, Any], model: str) -> list[str]: + """The agent's own model menu, when it was measured: the retry for a model its provider refused.""" choices = [c.get("value") for c in row.get("model_choices") or [] if isinstance(c, dict) and c.get("value")] - if choices: - lines.append(f"- Models it lists: {', '.join(str(c) for c in choices[:12])}. To try another, {model}.") - return lines + if not choices: + return [] + return [f"- Models it lists: {', '.join(str(c) for c in choices[:12])}. To try another, {model}."] + + +def _machines_line() -> str | None: + """The registered machines an acp preset can be added on, while the machine lane is on; else nothing.""" + from raven.home import machine_lane_enabled + from raven.ops import connections + + if not machine_lane_enabled(): + return None + try: + rows = connections.usable(connections.load()) + except (OSError, ValueError): + return None + names = [ + f"{row['id']} ({row['display_name']})" if row.get("display_name") else str(row["id"]) + for row in rows + if row.get("id") and connections.transport_of(row) == connections.SSH + ] + if not names: + return None + return ( + f" machines an acp preset can be added on: {', '.join(names)} -- add subagents " + '{"preset": "", "machine": "", "remote_cwd"?: "~/raven-work"}: it runs there, signed in ' + "there; the name defaults to @" + ) def _fix_rules(retry: str) -> list[str]: @@ -306,7 +353,7 @@ def _fix_it_yourself( preset: str, row: dict[str, Any], *, remedy: dict[str, Any] | None, detail: str, retry: str, model: str ) -> str: """What to do about an agent that did not answer: fix what Raven can, ask only for what needs the user.""" - needs = _what_it_needs(preset, row, remedy=remedy, detail=detail, model=model) + needs = _what_it_needs(preset, row, remedy=remedy, detail=detail, model=model, machine=row.get("machine")) rules = _fix_rules(retry) return "\n".join([rules[0], *needs, *rules[1:]]) @@ -645,6 +692,8 @@ def _subagent_lines(self, rows: list[dict[str, Any]] | None) -> list[str]: view = self._subagent_view(row) state = "not added" if view.get("added") is False else ("on" if view.get("enabled") else "off") bits = [state, str(view.get("kind") or "")] + if view.get("machine"): + bits.append(f"on machine {view['machine']}") if view.get("model"): bits.append(f"model {view['model']}") if view.get("status") not in (None, "ready", "installed"): @@ -659,6 +708,8 @@ def _subagent_lines(self, rows: list[dict[str, Any]] | None) -> list[str]: "not listed here cannot be connected from here; several agents go in one add as a list; " "describe subagents. for one agent's settings and health" ) + if machines := _machines_line(): + lines.append(machines) return lines async def _search_reply(self, query: str, raw: dict[str, Any]) -> str: @@ -752,6 +803,8 @@ async def _describe_subagent(self, name: str) -> dict[str, Any]: def _subagent_view(row: dict[str, Any]) -> dict[str, Any]: """One roster row with what explains a failure: added or not, health, the last test.""" out: dict[str, Any] = {k: row.get(k) for k in ("name", "kind", "enabled", "description", "model")} + if row.get("machine"): + out["machine"] = row["machine"] if not (row.get("configured") or row.get("builtin") or row.get("vendored")): preset = row.get("preset") or row.get("name") out["added"] = False @@ -1197,8 +1250,8 @@ async def _add(self, path: str, value: Any) -> str: specs = value if isinstance(value, list) else [value] if not specs or not all(isinstance(v, dict) and isinstance(v.get("preset"), str) for v in specs): raise ValueError( - 'add takes {"preset": "", "model"?: ...} for an agent in the describe list -- or a list ' - "of those to connect several under one confirmation" + 'add takes {"preset": "", "model"?: ..., "machine"?: ""} for an agent ' + "in the describe list -- or a list of those to connect several under one confirmation" ) params_list = [_add_params(v) for v in specs] rows: list[dict[str, Any]] | None = None @@ -1230,6 +1283,7 @@ async def _add(self, path: str, value: Any) -> str: remedy=remedy if isinstance(remedy, dict) else None, detail=text, model=f"add with {json.dumps(again, ensure_ascii=False)}", + machine=params.get("machine"), ) failed.append("\n".join([f"Not added: {text}", *needs])) continue diff --git a/raven/config/schema.py b/raven/config/schema.py index ce54bf9d7..9829d9299 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -11,6 +11,7 @@ from raven.config.agent_names import THIRD_PARTY_PRESET_NAMES from raven.contracts.path_policy import WORKSPACE_DEFAULT_SENTINEL +from raven.home import MACHINE_LANE_ENV, machine_lane_enabled from raven.sandbox.config import SandboxConfig #: The loop's outer LLM-error retry ladder when the config names none: one wait @@ -2029,6 +2030,26 @@ class ThirdPartyAcpSubagentConfig(Base): the field on the write path.""" cwd: str | None = None env: dict[str, str] = Field(default_factory=dict) + machine: str | None = None + """The registered machine this agent runs on, by its ``connections.json`` id, or + ``None`` for this computer. + + Set, ``command`` is what runs on that machine, started there by the + machine's Raven node in its registered user's login environment, with + this entry's ``env`` over it; raven reaches the machine over one ssh link + with the address, port and key the registry holds, and none of those is + written here. Each session works in ``/`` + there. MCP servers are not lent to it, files are not handed to it, and + what it writes stays on that machine. See :mod:`raven.node.client` and + :mod:`raven.acp_client.remote`. + + Debug-only until the lane matures: unless raven was started with + ``RAVEN_DEBUG_MACHINES=1`` (:func:`raven.home.machine_lane_enabled`), an + entry that names a machine is left off the table and no machine is + reached for it.""" + remote_cwd: str | None = None + """Where on ``machine`` this agent works, ``~/raven-work`` when unset; ignored + without ``machine``. ``~`` is the registered user's home there.""" lend_keys: list[str] = Field(default_factory=list) """Raven providers whose key this agent is started with, by name (``openrouter``). @@ -2094,6 +2115,37 @@ def _warn_on_declared_cli_fields(cls, data: Any) -> Any: ) return data + @model_validator(mode="after") + def _remote_rows_borrow_no_keys(self) -> "ThirdPartyAcpSubagentConfig": + """A row on another machine is started without Raven's keys; drop and warn. + + Lending lets an agent here use a provider key Raven holds without + holding it itself; on another machine it would mean copying Raven's key + to that machine, which is that machine's own sign-in to provide. + Warn-and-drop on load for the startup reason + ``_warn_on_declared_cli_fields`` gives; the write path refuses it + (``update_subagents.reject_unsupported_acp_fields``). + """ + self.machine = (self.machine or "").strip() or None + if self.machine and not machine_lane_enabled(): + logger.info( + "sub-agent {!r} names machine {!r}, and the machine lane is debug-only: the entry stays " + "off the table until raven is started with {}=1", + self.name, + self.machine, + MACHINE_LANE_ENV, + ) + if self.machine and self.lend_keys: + logger.warning( + "sub-agent {!r} runs on machine {!r}, so Raven's keys {} are not lent to it; " + "sign it in on that machine instead", + self.name, + self.machine, + sorted(self.lend_keys), + ) + self.lend_keys = [] + return self + @model_validator(mode="after") def _warn_on_prompt_placeholders(self) -> "ThirdPartyAcpSubagentConfig": """Warn that a task placeholder in ``command`` cannot work here. diff --git a/raven/config/update_subagents.py b/raven/config/update_subagents.py index cb27218ec..a74ea8585 100644 --- a/raven/config/update_subagents.py +++ b/raven/config/update_subagents.py @@ -132,6 +132,14 @@ def reject_unsupported_acp_fields(entries: list[dict]) -> None: "server, not one task -- the task is delivered as a session/prompt request, so the " "placeholder would be passed through literally and the handshake would fail" ) + machine = entry.get("machine") + lent = entry.get("lendKeys") or entry.get("lend_keys") + if isinstance(machine, str) and machine.strip() and lent: + raise ValueError( + f"acp sub-agent {name!r} runs on machine {machine.strip()!r}, so it cannot be started with " + f"Raven's keys {sorted(lent)}: they would have to be copied to that machine; sign it in there " + "and remove lendKeys" + ) def _raw_config(path: Path) -> dict[str, Any]: diff --git a/raven/rpc/methods/subagents.py b/raven/rpc/methods/subagents.py index 8fff3383a..024697528 100644 --- a/raven/rpc/methods/subagents.py +++ b/raven/rpc/methods/subagents.py @@ -44,6 +44,7 @@ from raven.config.schema import SubagentsConfig from raven.config.update_subagents import ( get_agents, + reject_unsupported_acp_fields, reject_unsupported_openai_fields, remove_agent, set_agents, @@ -373,6 +374,7 @@ async def _rows(*, probe: bool = True) -> list[dict]: # The row's editing rule, not ownership: what `subagents.update` # accepts for `model` on this row. `own` is the ownership mark. "model_source": _model_rule(cfg, snapshot, meta), + "machine": (getattr(cfg, "machine", None) or "").strip() or None, } ) return rows @@ -456,9 +458,19 @@ async def subagents_add(params: dict, *, agent_loop_factory: "AgentLoopFactory | data={"preset": preset_name, "known": sorted(THIRD_PARTY_SUBAGENT_PRESETS)}, ) entry = third_party_subagent_preset(preset_name) + machine = _clean_name(params.get("machine"), field="machine") + if machine: + _refuse_unless_it_can_run_there(entry, machine) + entry["machine"] = machine + if isinstance(params.get("remote_cwd"), str) and params["remote_cwd"].strip(): + entry["remoteCwd"] = params["remote_cwd"].strip() name = _clean_name(params.get("name"), field="name") if name: entry["name"] = name + elif machine: + # One preset is added once per machine it runs on; the shipped name + # alone would be every one of them, so the machine is in the default. + entry["name"] = f"{preset_name}@{machine}" if params.get("description"): entry["description"] = params["description"] if (params.get("lend_key") or "").strip(): @@ -489,6 +501,7 @@ async def subagents_add(params: dict, *, agent_loop_factory: "AgentLoopFactory | # the first of two, so it would prove a row this call is not writing. try: reject_unsupported_openai_fields([entry]) + reject_unsupported_acp_fields([entry]) entries = [*_read_agents(), entry] validate_agents(entries, config_path=get_config_path()) except (ValueError, ValidationError) as exc: @@ -510,6 +523,28 @@ async def subagents_add(params: dict, *, agent_loop_factory: "AgentLoopFactory | return {"added": True, "name": entry["name"]} +def _refuse_unless_it_can_run_there(entry: dict, machine: str) -> None: + """Refuse an add on a registered machine before anything is spent on it. + + Only an acp preset runs on a machine: its command is started there by the + machine's Raven node. The machine is judged the way a dispatch judges it + (:func:`raven.acp_client.remote.machine`): the lane on, the row registered + and usable -- each refusal one sentence, with no address in it. + """ + from raven.acp_client import remote + + if entry.get("kind") != "acp": + raise ConfigValidationError( + f"preset {str(entry.get('preset') or entry.get('name'))!r} is not an acp agent; only an acp agent " + "runs on a registered machine", + data={"field": "machine", "machine": machine}, + ) + try: + remote.machine(machine) + except remote.RemoteMachineError as exc: + raise ConfigValidationError(str(exc), data={"field": "machine", "machine": machine}) from exc + + def _lend_keys(preset: Any, providers: list[Any]) -> list[str]: """``providers`` as a row's ``lendKeys``, refused unless the preset reads each and Raven holds its key.""" from raven.agent.subagent.presets import lendable_keys @@ -838,6 +873,7 @@ async def subagents_update(params: dict, *, agent_loop_factory: "AgentLoopFactor entries, target = _merged_over_the_ping(target, row_before, name_before, materialized=materialized_discovered) try: reject_unsupported_openai_fields([target]) + reject_unsupported_acp_fields([target]) set_agents(entries, config_path=get_config_path()) except (ValueError, ValidationError) as exc: _raise_config_error(exc) @@ -967,9 +1003,12 @@ async def _read_the_shell_again(cfg: Any) -> None: Only for the kinds that launch a process: an endpoint is called from this process, whose own environment a capture does not change. The capture runs the user's shell and can take seconds, which a press about to spend a real - prompt can afford and a listing cannot. + prompt can afford and a listing cannot. Not for an agent on a registered + machine either: its node starts it in that machine's login environment, + captured there, and this computer's shell says nothing about it. """ - if getattr(cfg, "kind", None) in ("cli", "acp"): + on_machine = bool((getattr(cfg, "machine", None) or "").strip()) + if getattr(cfg, "kind", None) in ("cli", "acp") and not on_machine: await asyncio.to_thread(refresh_login_shell_env) diff --git a/raven/rpc/models.py b/raven/rpc/models.py index 83f69d3ec..81c39443d 100644 --- a/raven/rpc/models.py +++ b/raven/rpc/models.py @@ -105,6 +105,13 @@ class SubagentRow(_Strict): description: str enabled: bool configured: bool + machine: str | None = Field( + default=None, + description=( + "The registered machine an acp row runs on, by its id in the machine registry; null for this " + "computer, and on a server that predates the field." + ), + ) vendored: bool = Field( default=False, description=( @@ -2649,6 +2656,18 @@ class SubagentsAddParams(_Strict): "each start; refused unless the preset reads a key for it and Raven holds one." ), ) + machine: str | None = Field( + default=None, + description=( + "A registered machine, by its id in the machine registry, for an acp preset: the preset's command is " + "started there by the machine's Raven node and signed in there, and the row's name defaults to " + "@. Refused unless the machine lane is on and the machine is registered and usable." + ), + ) + remote_cwd: str | None = Field( + default=None, + description="Where on `machine` the agent works (`~/raven-work` when unset); ignored without `machine`.", + ) api_key: str | None = None mcps: list[str] | None = None allow_mcp_secrets: bool | None = None diff --git a/rpc-schema/openrpc.json b/rpc-schema/openrpc.json index b666457fc..4221ac41d 100644 --- a/rpc-schema/openrpc.json +++ b/rpc-schema/openrpc.json @@ -2141,6 +2141,22 @@ "type": "string" } }, + { + "name": "machine", + "required": false, + "description": "A registered machine, by its id in the machine registry, for an acp preset: the preset's command is started there by the machine's Raven node and signed in there, and the row's name defaults to @. Refused unless the machine lane is on and the machine is registered and usable.", + "schema": { + "type": "string" + } + }, + { + "name": "remote_cwd", + "required": false, + "description": "Where on `machine` the agent works (`~/raven-work` when unset); ignored without `machine`.", + "schema": { + "type": "string" + } + }, { "name": "api_key", "required": false, @@ -12017,6 +12033,10 @@ "configured": { "type": "boolean" }, + "machine": { + "type": "string", + "description": "The registered machine an acp row runs on, by its id in the machine registry; null for this computer, and on a server that predates the field." + }, "vendored": { "type": "boolean", "description": "Discovered under the `agents/` product tree rather than written into config: one of the agent products that ship with this raven, materialized as a row on every table build. Like `builtin` it leaves `configured` false -- there is no config entry to delete, and removing it means removing its folder -- but unlike `builtin` it is a real subprocess with a command, so it is probed and it can be unready (a missing engine wheel leaves it listed and disabled). The wire name predates the tree's rename. Absent from a server that predates discovery." diff --git a/schemas/subagent.schema.json b/schemas/subagent.schema.json index 05e33e5f2..dc25c78dd 100644 --- a/schemas/subagent.schema.json +++ b/schemas/subagent.schema.json @@ -157,6 +157,18 @@ "title": "Lendkeys", "type": "array" }, + "machine": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Machine" + }, "maxOutputChars": { "default": 30000, "title": "Maxoutputchars", @@ -238,6 +250,18 @@ "title": "Readytimeoutms", "type": "integer" }, + "remoteCwd": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Remotecwd" + }, "routes": { "items": { "$ref": "#/$defs/SubagentRouteConfig" diff --git a/tests/acp_stub_server.py b/tests/acp_stub_server.py index db8dddbe4..227628827 100644 --- a/tests/acp_stub_server.py +++ b/tests/acp_stub_server.py @@ -145,6 +145,8 @@ # A real server mints a distinct id per session; the counter keeps the stub from # making concurrent sessions collide in a way no real agent would. _SESSIONS = 0 +# The cwd the last session/new named, for the mode that writes a file there. +_SESSION_CWD = "" # What each session is currently answering with, so a resumed one can report it. _SESSION_MODELS: dict[str, str] = {} @@ -316,6 +318,18 @@ def handle_prompt(request_id, params) -> None: ) ok(request_id, {"stopReason": "end_turn"}) return + if MODE == "writes_file": + # Leaves a file in the session directory, the way an agent's own tools + # would, so a test can see what the host collects afterwards. + import pathlib + + out = pathlib.Path(_SESSION_CWD or ".") / "out" + out.mkdir(parents=True, exist_ok=True) + (out / "result.txt").write_text("made by the agent\n", encoding="utf-8") + (pathlib.Path(_SESSION_CWD or ".") / "notes.md").write_text("# notes\n", encoding="utf-8") + update(session_id, {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": "wrote it"}}) + ok(request_id, {"stopReason": "end_turn"}) + return if MODE == "asks_late": _PENDING.append((request_id, session_id)) if len(_PENDING) < 2: @@ -595,6 +609,9 @@ def handle_prompt(request_id, params) -> None: # Kimi Code waiting on a provider that never answers: the prompt is # taken and never finished. return + if delay := os.environ.get("ACP_STUB_PROMPT_DELAY_S"): + # A provider that takes its time: the prompt is answered after the delay. + time.sleep(float(delay)) if MODE == "empty_turn": print("provider rejected the credential: HTTP 401", file=sys.stderr, flush=True) ok(request_id, {"stopReason": "end_turn"}) @@ -733,6 +750,8 @@ def main() -> None: elif not _INITIALIZED: err(request_id, -32603, "Internal error: no initialize was received on this connection") elif method == "session/new": + global _SESSION_CWD + _SESSION_CWD = str(params.get("cwd") or "") if MODE == "no_session": err(request_id, -32000, "no api key configured for this agent") elif MODE == "no_session_other": diff --git a/tests/test_raven_config_tool.py b/tests/test_raven_config_tool.py index 056b77754..dccc62eeb 100644 --- a/tests/test_raven_config_tool.py +++ b/tests/test_raven_config_tool.py @@ -1693,3 +1693,100 @@ async def __call__(self, method: str, params: dict[str, Any]) -> Any: batch = {"agents.defaults.temperature": 0.5, "subagents.codex.enabled": False} reply = await _run(tool, action="set", value=json.dumps(batch)) assert reply.startswith("Error") and "applied before it" in reply and "agents.defaults.temperature" in reply + + +# --- an agent on a registered machine ---------------------------------------- + + +_LAB_ROW: dict[str, Any] = { + "name": "claude_code@ravenx2", + "preset": "claude_code", + "kind": "acp", + "description": "Claude Code", + "enabled": True, + "configured": True, + "machine": "ravenx2", + "probe_status": "ready", + "probe_detail": "runs on machine 'RavenX 2' (ravenx2); it ran and replied", + "model_choices": [], +} + + +@pytest.mark.asyncio +async def test_an_add_on_a_registered_machine_passes_the_machine_through_and_says_where_it_runs(config_file): + calls = Calls( + {"subagents.add": {"added": True, "name": "claude_code@ravenx2"}, "subagents.list": {"rows": [_LAB_ROW]}} + ) + tool = RavenConfigTool() + tool.set_rpc_caller(calls) + + reply = await _run( + tool, + action="add", + path="subagents", + value='{"preset": "claude_code", "machine": "ravenx2", "remote_cwd": "~/raven-work"}', + ) + + sent = next(p for m, p in calls.calls if m == "subagents.add") + assert sent == {"preset": "claude_code", "machine": "ravenx2", "remote_cwd": "~/raven-work"} + assert "Connected sub-agent claude_code@ravenx2" in reply + assert "runs on machine 'RavenX 2' (ravenx2)" in reply + assert "on machine ravenx2" in await _run(tool, action="describe", path="subagents") + assert '"machine": "ravenx2"' in await _run(tool, action="describe", path="subagents.claude_code@ravenx2") + + +@pytest.mark.asyncio +async def test_a_refused_add_on_a_machine_sends_the_user_to_that_machine_and_runs_nothing_here(config_file): + """A refusal from a machine names no command to run here: the agent, its install and its sign-in are there.""" + refusal = _Refused( + "sub-agent 'claude_code@ravenx2' did not answer a test message, so it was not added: 'claude' is not on " + "the login shell PATH (on machine 'ravenx2': any fix is made there)", + {}, + ) + calls = Calls({"subagents.add": refusal, "subagents.list": {"rows": [_LAB_ROW | {"configured": False}]}}) + tool = RavenConfigTool() + tool.set_rpc_caller(calls) + + reply = await _run(tool, action="add", path="subagents", value='{"preset": "claude_code", "machine": "ravenx2"}') + + assert reply.startswith("Not added:") and "on machine 'ravenx2': any fix is made there" in reply + assert "It runs on machine 'ravenx2', so every fix is made there by the user" in reply + assert "exec refuses ssh to a registered machine" in reply + assert "Install it with exec" not in reply and "Run it on its own with exec" not in reply + + +@pytest.mark.asyncio +async def test_describe_names_the_registered_machines_only_while_the_lane_is_on(config_file, tmp_path, monkeypatch): + from raven.ops import connections + + registry = tmp_path / "connections.json" + registry.write_text( + json.dumps( + { + "connections": [ + { + "id": "ravenx2", + "display_name": "RavenX 2", + "host": "203.0.113.8", + "port": 22, + "user": "root", + "key": "/k", + }, + {"id": "this-one", "transport": "local"}, + ] + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(connections, "store_path", lambda: registry) + tool = RavenConfigTool() + tool.set_rpc_caller(Calls({"subagents.list": {"rows": []}})) + + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + assert "machines an acp preset" not in await _run(tool, action="describe", path="subagents") + + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + said = await _run(tool, action="describe", path="subagents") + line = next(ln for ln in said.splitlines() if ln.strip().startswith("machines an acp preset")) + assert line.split("on:", 1)[1].split("--", 1)[0].strip() == "ravenx2 (RavenX 2)", "the local row is no machine" + assert '"machine": ""' in line and "203.0.113" not in said diff --git a/tests/test_rpc_subagents.py b/tests/test_rpc_subagents.py index b24e18bde..204d9a62c 100644 --- a/tests/test_rpc_subagents.py +++ b/tests/test_rpc_subagents.py @@ -3126,3 +3126,147 @@ async def _launch(**kwargs: object) -> object: (cfg,) = _as_configs([{"name": "Pi", "kind": "acp", "preset": "pi", "command": "x", "lendKeys": ["openrouter"]}]) await probe.record_capabilities(cfg) assert launched and launched[0]["OPENROUTER_API_KEY"] == "sk-or-raven" + + +# --- an add on a registered machine ------------------------------------------ + + +_MACHINES = [ + {"id": "ravenx2", "display_name": "RavenX 2", "host": "203.0.113.8", "port": 22, "user": "root", "key": "/k"}, +] + + +@pytest.fixture +def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A machine registry with one usable row, and the machine lane on.""" + from raven.ops import connections + + path = tmp_path / "connections.json" + path.write_text(json.dumps({"connections": _MACHINES}), encoding="utf-8") + monkeypatch.setattr(connections, "store_path", lambda: path) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + return path + + +async def test_add_on_a_registered_machine_writes_the_machine_and_names_the_row_after_it( + config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """One preset is added once per machine it runs on, and the gate proves the entry there.""" + seen: list[object] = [] + + async def _capture(cfg: object) -> object: + seen.append(cfg) + return await _pings_ok(cfg) + + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _capture) + + out = await subagents_add({"preset": "claude_code", "machine": " ravenx2 ", "remote_cwd": " ~/work "}) + + assert out == {"added": True, "name": "claude_code@ravenx2"} + entry = next(e for e in _stored(config_path) if e["name"] == "claude_code@ravenx2") + assert entry["machine"] == "ravenx2" and entry["remoteCwd"] == "~/work" and entry["kind"] == "acp" + assert entry["enabled"] is True + assert [getattr(c, "machine", None) for c in seen] == ["ravenx2"], "the gate proves the entry on its machine" + assert "203.0.113" not in json.dumps(_stored(config_path)), "the row names the machine, never its address" + + +async def test_add_on_a_machine_keeps_the_name_the_caller_chose( + config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _pings_ok) + + out = await subagents_add({"preset": "claude_code", "machine": "ravenx2", "name": "lab"}) + + assert out == {"added": True, "name": "lab"} + entry = next(e for e in _stored(config_path) if e["name"] == "lab") + assert entry["machine"] == "ravenx2" and not entry.get("remoteCwd"), "unset: the node's default applies" + + +async def test_add_on_a_machine_is_refused_with_the_lane_off_and_nothing_is_stored_or_pinged( + config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + pinged: list[object] = [] + + async def _ping(cfg: object) -> object: + pinged.append(cfg) + return await _pings_ok(cfg) + + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _ping) + before = _stored(config_path) + + with pytest.raises(ConfigValidationError) as caught: + await subagents_add({"preset": "claude_code", "machine": "ravenx2"}) + + assert "machine lane is debug-only" in str(caught.value) and "RAVEN_DEBUG_MACHINES=1" in str(caught.value) + assert caught.value.data["field"] == "machine" + assert pinged == [] and _stored(config_path) == before + + +@pytest.mark.parametrize(("machine", "said"), [("nowhere", "is not registered"), (" ", "must not be blank")]) +async def test_add_on_a_machine_nobody_registered_is_refused_by_name_without_an_address( + config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch, machine: str, said: str +) -> None: + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _pings_ok) + before = _stored(config_path) + + with pytest.raises(ConfigValidationError) as caught: + await subagents_add({"preset": "claude_code", "machine": machine}) + + assert said in str(caught.value) and "203.0.113" not in str(caught.value) + assert _stored(config_path) == before + + +async def test_add_on_a_machine_takes_only_an_acp_preset( + config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _pings_ok) + + with pytest.raises(ConfigValidationError) as caught: + await subagents_add({"preset": "mirothinker", "machine": "ravenx2"}) + + assert "not an acp agent" in str(caught.value) and caught.value.data["machine"] == "ravenx2" + assert all(e.get("machine") is None for e in _stored(config_path)) + + +async def test_add_on_a_machine_lends_no_key( + config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Raven's key would have to be copied to that machine; the write path refuses, as it does for a stored row.""" + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _pings_ok) + monkeypatch.setattr("raven.rpc.methods.subagents._lend_keys", lambda preset, providers: ["openrouter"]) + + with pytest.raises(ConfigValidationError) as caught: + await subagents_add({"preset": "opencode", "machine": "ravenx2", "lend_key": "openrouter"}) + + assert "Raven's keys" in str(caught.value) and "sign it in there" in str(caught.value) + assert all(e.get("machine") is None for e in _stored(config_path)) + + +async def test_the_login_shell_is_not_read_again_for_an_agent_on_a_machine(monkeypatch: pytest.MonkeyPatch) -> None: + """Its node captures that machine's login environment; this computer's shell says nothing about it.""" + import raven.rpc.methods.subagents as subagents_mod + from raven.config.schema import ThirdPartyAcpSubagentConfig + + captured: list[int] = [] + monkeypatch.setattr(subagents_mod, "refresh_login_shell_env", lambda: captured.append(1)) + there = ThirdPartyAcpSubagentConfig(name="lab", command="x acp", machine="ravenx2") + here = ThirdPartyAcpSubagentConfig(name="local", command="x acp") + + await _read_the_shell_again(there) + assert captured == [] + await _read_the_shell_again(here) + assert captured == [1] + + +async def test_list_rows_carry_the_machine_an_acp_row_runs_on(config_path: Path, registry: Path) -> None: + raw = json.loads(config_path.read_text(encoding="utf-8")) + raw["subagents"]["agents"].append( + {"name": "lab", "kind": "acp", "command": "x acp", "machine": "ravenx2", "enabled": True} + ) + config_path.write_text(json.dumps(raw), encoding="utf-8") + + rows = {r["name"]: r for r in (await subagents_list({"probe": False}))["rows"]} + + assert rows["lab"]["machine"] == "ravenx2" + assert rows["Coder"]["machine"] is None diff --git a/tests/test_subagent_acp.py b/tests/test_subagent_acp.py index 739a691af..a453aa774 100644 --- a/tests/test_subagent_acp.py +++ b/tests/test_subagent_acp.py @@ -2518,16 +2518,24 @@ def test_the_launch_key_covers_every_launch_parameter() -> None: # handler does not make the running process the wrong one. `ready_timeout_s` # is how long this caller waits for the handshake, not what was launched: two # entries differing only in patience share the connection they both wanted. - outside = {"self", "name", "on_request", "ready_timeout_s"} + # `launcher` is how a process on a `machine` is started, derived from the + # launch arguments the key already covers. + outside = {"self", "name", "on_request", "ready_timeout_s", "launcher"} params = {p for p in inspect.signature(AcpConnectionPool.acquire).parameters} - outside assert params == set(LAUNCH_PARAMS) base = {"command": "agent acp", "cwd": "/w", "env": {"A": "1"}} - for field, other in (("command", "agent acp --url x"), ("cwd", "/other"), ("env", {"A": "2"})): + for field, other in ( + ("command", "agent acp --url x"), + ("cwd", "/other"), + ("env", {"A": "2"}), + ("machine", "box"), + ): assert launch_key(**base) != launch_key(**{**base, field: other}), field assert launch_key(**base) == launch_key(**base) - # An absent env and an empty one are the same launch. + # An absent env and an empty one are the same launch, and so is a machine that is this computer. assert launch_key(command="a", cwd=None, env=None) == launch_key(command="a", cwd=None, env={}) + assert launch_key(command="a", cwd=None, env=None) == launch_key(command="a", cwd=None, env=None, machine=None) async def test_two_bindings_of_one_agent_hold_their_own_connections(tmp_path: Path) -> None: diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py new file mode 100644 index 000000000..d50f60378 --- /dev/null +++ b/tests/test_subagent_acp_remote.py @@ -0,0 +1,764 @@ +"""An acp sub-agent whose entry names a registered machine, end to end. + +Nothing here needs a second computer. A stand-in ``ssh`` on PATH does what sshd +does with the one remote-command argument it is sent -- hands it to the user's +shell, in that user's home -- so the real ssh line, the real Raven node and the +real stub ACP server the node starts all run: only the network is missing. The +stand-in can also fail the way ssh fails, writing its own address-bearing words +where the ssh line asked (``-E``). +""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import sys +import time +from pathlib import Path +from typing import Any + +import pytest + +from raven.acp_client.capabilities import snapshot_fingerprint, verify_agent +from raven.acp_client.pool import close_pool, get_pool +from raven.acp_client.protocol import AcpConnectionError +from raven.agent.subagent import probe +from raven.agent.subagent.backends import ( + agent_meta, + build_third_party_backend, + format_agent_listing, + session_mcp_effective, +) +from raven.agent.subagent.backends import env as backend_env +from raven.agent.subagent.mcp_grant import GrantedServer, McpGrant +from raven.agent.subagent.probe_state import fingerprint +from raven.agent.subagent.registry import _row_for +from raven.config.schema import MCPServerConfig, ThirdPartyAcpSubagentConfig +from raven.config.update_subagents import reject_unsupported_acp_fields +from raven.node import bundle +from raven.node.client import NodePool +from raven.ops import remote_path + +_STUB = Path(__file__).with_name("acp_stub_server.py") +HOST = "203.0.113.7" +PORT = 58717 + +_FAKE_SSH = """#!{python} +import json, os, sys +here = os.path.dirname(os.path.abspath(__file__)) +args = sys.argv[1:] +with open(os.path.join(here, "calls"), "a") as f: + f.write(json.dumps(args) + "\\n") +mode = open(os.path.join(here, "mode")).read().strip() +home = open(os.path.join(here, "home")).read().strip() +launch = "-T" in args # a held session; the one-shot runner sends no -T +log = args[args.index("-E") + 1] if "-E" in args else None + +def fail(said): + if log: + open(log, "a").write(said + "\\n") + else: + sys.stderr.write(said + "\\n") + sys.exit(255) + +if mode == "unreachable": + fail("ssh: connect to host {host} port {port}: Operation timed out") +if mode == "launch_denied" and launch: + fail("worker@{host}: Permission denied (publickey).") +command = args[args.index("--") + 1] if "--" in args else args[-1] +os.chdir(home) +os.execve("/bin/sh", ["/bin/sh", "-c", command], {{"HOME": home, "SHELL": "/bin/sh", "PATH": os.environ["PATH"]}}) +""" + + +# Every dispatch, verify and ping here starts a real node over the stand-in +# ssh, and the agent through it: real processes, real seconds. +pytestmark = pytest.mark.slow + + +@pytest.fixture(autouse=True) +async def _no_pooled_connections(): + yield + await close_pool() + + +class Machine: + """The stand-in machine: its home, its ssh, and what was asked of it.""" + + def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + self.home = root / "box-home" + self.home.mkdir() + self.bin = root / "fakebin" + self.bin.mkdir() + ssh = self.bin / "ssh" + ssh.write_text(_FAKE_SSH.format(python=sys.executable, host=HOST, port=PORT), encoding="utf-8") + ssh.chmod(0o755) + (self.bin / "home").write_text(str(self.home), encoding="utf-8") + self.mode("ok") + registry = root / "connections.json" + registry.write_text( + json.dumps( + { + "connections": [ + { + "id": "box", + "display_name": "Lab box", + "host": HOST, + "port": PORT, + "user": "worker", + "key": "/keys/id_test", + } + ] + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("RAVEN_CONNECTIONS", str(registry)) + # The lane is debug-only; these tests are about what it does when on. + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + path = f"{self.bin}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}" + # Both ways ssh is started: the one-shot runner inherits this process's + # PATH, and an agent's launch takes the captured login-shell one. + monkeypatch.setenv("PATH", path) + monkeypatch.setattr(backend_env, "_LOGIN_ENV", {**os.environ, "PATH": path}) + monkeypatch.setattr("raven.config.paths.get_logs_dir", lambda: root / "logs") + # This Raven's node, installed where the host looks for it: its + # ``bin/python`` is this test run's own interpreter. + python = self.home / ".raven-node" / bundle.install_name() / "bin" / "python" + python.parent.mkdir(parents=True) + python.write_text(f'#!/bin/sh\nexec {sys.executable} "$@"\n', encoding="utf-8") + python.chmod(0o755) + # The link to the machine, as assembly hands it to the file tools and + # the dispatch: one pool for this test, closed with it. + self.nodes = NodePool() + monkeypatch.setattr(remote_path, "_NODES", lambda: self.nodes) + + def mode(self, mode: str) -> None: + (self.bin / "mode").write_text(mode, encoding="utf-8") + + def calls(self) -> list[list[str]]: + path = self.bin / "calls" + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +@pytest.fixture +async def box(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + if not Path("/bin/sh").exists(): + pytest.skip("the stand-in ssh hands the ssh line to a POSIX shell") + machine = Machine(tmp_path, monkeypatch) + yield machine + await machine.nodes.close_all() + + +def _agent_pid(name: str = "stub@box") -> int | None: + """The pid of the agent process on the machine, read off the pooled connection's endpoint.""" + (connection,) = get_pool().connections(name) + return connection.client._endpoint.pid + + +def remote_config(*, mode: str = "ok", command: str | None = None, **kw: Any) -> ThirdPartyAcpSubagentConfig: + return ThirdPartyAcpSubagentConfig( + name=kw.pop("name", "stub@box"), + command=command or f"{sys.executable} {_STUB}", + env={"ACP_STUB_MODE": mode, **kw.pop("env", {})}, + machine="box", + remote_cwd=kw.pop("remote_cwd", "~/raven-work"), + ready_timeout_ms=kw.pop("ready_timeout_ms", 15000), + **kw, + ) + + +# ---- a dispatch ---------------------------------------------------------------- + + +async def test_a_dispatch_runs_on_the_machine_in_a_directory_made_there_and_says_where(box, tmp_path): + backend = build_third_party_backend(remote_config()) + local = tmp_path / "local-workspace" + + reply = await backend.run("hi", task_id="t1", workspace=local, executor=None, instance="reviewer") + + there = box.home / "raven-work" / "reviewer" + assert there.is_dir(), "the session directory is made on the machine" + assert "[raven] Ran on machine 'Lab box' (box), in " in reply + # Shown the way scp writes it, which the file tools read back as it is. + assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in reply + assert "not on this computer" in reply + assert "read_file, list_dir, find, grep, write_file and edit_file take that path as it is" in reply + assert not local.exists(), "the caller's workspace is never touched" + for argv in box.calls(): + assert str(local) not in " ".join(argv), "this computer's workspace path never reaches the machine" + + +async def test_the_entry_s_env_reaches_the_agent_over_the_link_and_never_an_ssh_argument(box, tmp_path): + # env is where credentials live, and an argument is in the process list + # at both ends. The stub answers in the mode its env names, so an echo + # proves the variables arrived; no ssh argv may carry any of them. + cfg = remote_config(mode="echo_blocks") + cfg = cfg.model_copy(update={"env": {**cfg.env, "OPENAI_API_KEY": "sk-raven-test-not-real"}}) + backend = build_third_party_backend(cfg) + + reply = await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None, model="parent-model") + + assert json.loads(reply.split("\n\n[raven]")[0]) == [{"type": "text", "text": "hi"}] + (launch,) = [argv for argv in box.calls() if "-T" in argv] + for argv in box.calls(): + joined = " ".join(argv) + assert "sk-raven-test-not-real" not in joined and "ACP_STUB_MODE" not in joined + remote = launch[launch.index("--") + 1] + assert "RAVEN_PARENT_MODEL" not in remote, "the parent binding is this computer's process env" + assert "raven.node --attach --dir" in remote, "the one ssh is the relay to the machine's daemon" + assert "-E" in launch, "ssh's own messages go to their log" + + +async def test_an_edited_env_value_relaunches_the_agent_on_the_same_link(box, tmp_path): + # The values travel on the link, and the pool keys on them: an edit must + # not be answered by the agent process started before it. + first = build_third_party_backend(remote_config()) + await first.run("one", task_id="t1", workspace=tmp_path, executor=None) + before = _agent_pid() + edited = remote_config().model_copy(update={"env": {"ACP_STUB_MODE": "ok", "EXTRA": "2"}}) + await build_third_party_backend(edited).run("two", task_id="t2", workspace=tmp_path, executor=None) + + assert _agent_pid() != before, "a new agent process" + assert len([argv for argv in box.calls() if "-T" in argv]) == 1, "on the one link" + + +async def test_two_handles_on_one_machine_work_in_two_directories(box, tmp_path): + backend = build_third_party_backend(remote_config()) + + await backend.run("one", task_id="t1", workspace=tmp_path, executor=None, instance="a") + await backend.run("two", task_id="t2", workspace=tmp_path, executor=None, instance="b") + + assert (box.home / "raven-work" / "a").is_dir() + assert (box.home / "raven-work" / "b").is_dir() + + +async def test_an_unreachable_machine_fails_the_dispatch_in_one_sentence(box, tmp_path): + box.mode("unreachable") + backend = build_third_party_backend(remote_config()) + + with pytest.raises(AcpConnectionError) as caught: + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + + said = str(caught.value) + assert "machine 'Lab box' (box) did not answer within the connection timeout" in said + assert HOST not in said and str(PORT) not in said + + +async def test_a_launch_ssh_refuses_is_read_from_its_own_log_not_its_stderr(box, tmp_path): + box.mode("launch_denied") + backend = build_third_party_backend(remote_config()) + + with pytest.raises(AcpConnectionError) as caught: + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + + said = str(caught.value) + assert "refused the login" in said + assert HOST not in said + + +async def test_an_agent_not_installed_on_the_machine_says_so(box, tmp_path): + backend = build_third_party_backend(remote_config(command="no-such-agent-for-raven-tests --acp")) + + with pytest.raises(AcpConnectionError, match="command was not found on machine 'Lab box'"): + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + + snapshot = await verify_agent(remote_config(command="no-such-agent-for-raven-tests --acp")) + assert snapshot.status == "attention" and "command was not found on machine 'Lab box'" in snapshot.detail + + +async def test_an_unregistered_machine_fails_before_anything_starts(box, tmp_path): + backend = build_third_party_backend(remote_config().model_copy(update={"machine": "elsewhere"})) + + with pytest.raises(AcpConnectionError, match="machine 'elsewhere' is not registered"): + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + assert box.calls() == [] + + +async def test_two_parent_models_share_one_agent_process_rather_than_starting_one_each(box, tmp_path): + # The parent binding is launch environment for a process of raven's own; + # carried, each parent model would start an agent process on the machine. + backend = build_third_party_backend(remote_config()) + + await backend.run("one", task_id="t1", workspace=tmp_path, executor=None, model="parent-a") + first = _agent_pid() + await backend.run("two", task_id="t2", workspace=tmp_path, executor=None, model="parent-b") + + assert _agent_pid() == first + assert len([argv for argv in box.calls() if "-T" in argv]) == 1 + + +async def test_this_computer_s_files_are_not_accounted_for_a_remote_turn(box, tmp_path, monkeypatch): + from raven.acp_client import acp_agent + + built: list[dict[str, Any]] = [] + real = acp_agent._TurnCollector.__init__ + + def spy(self, *args, **kwargs): + built.append(kwargs) + real(self, *args, **kwargs) + + monkeypatch.setattr(acp_agent._TurnCollector, "__init__", spy) + backend = build_third_party_backend(remote_config()) + + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + + assert built and built[0]["track_files"] is False and built[0]["workspace"] is None + + +async def test_the_node_s_ssh_starts_from_an_empty_log(box): + from raven.acp_client import remote + + log = remote.ssh_log_path("raven-node@box") + log.write_text("worker@host: Permission denied (publickey).\n", encoding="utf-8") + + hello = await box.nodes.hello("box") + + assert hello["digest"] == bundle.digest() + assert log.read_text(encoding="utf-8") == "", "a failure is read from this launch's lines" + + +async def test_a_connection_that_ends_mid_turn_is_reported_as_it_ended(box, tmp_path): + # Past ssh: the agent itself exits during the turn, so there is no ssh + # failure to classify and the connection's own report stands. + backend = build_third_party_backend(remote_config(mode="elicits_then_dies")) + + with pytest.raises(AcpConnectionError, match="connection ended"): + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + + +async def test_granted_mcp_servers_are_withheld_on_the_wire_and_named_in_the_reply(box, tmp_path): + backend = build_third_party_backend(remote_config(mode="echo_blocks")) + grant = McpGrant(granted=(GrantedServer("pg", "stdio", MCPServerConfig(command="/bin/true")),)) + + reply = await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None, mcp_grant=grant) + + assert "MCP servers 'pg' were not lent to acp agent 'stub@box'" in reply + assert "Ran on machine 'Lab box' (box)" in reply + + +# ---- what it is not given ------------------------------------------------------ + + +def test_mcp_is_not_lent_to_an_agent_on_a_machine_and_the_reply_says_why(): + cfg = remote_config() + backend = build_third_party_backend(cfg) + grant = McpGrant(granted=(GrantedServer("pg", "stdio", MCPServerConfig(command="/bin/true")),)) + + assert backend._session_mcp_refused + note = backend._mcp_note(grant) + assert "were not lent" in note and "runs on machine 'box'" in note + assert session_mcp_effective(cfg) is False + assert _row_for(cfg).injectable.mcps is False + + +def test_the_roster_names_the_machine_and_hands_it_no_paths(): + cfg = remote_config() + meta = agent_meta(cfg) + + assert meta.machine == "box" + assert meta.reads_local_files is False + listing = format_agent_listing([meta]) + assert "on machine box" in listing and "no-local-files" in listing + assert HOST not in listing + assert _row_for(cfg).caps.machine == "box" + assert _row_for(cfg).meta().machine == "box" + + +def test_a_local_row_is_listed_as_before(): + cfg = ThirdPartyAcpSubagentConfig(name="here", command="qwen --acp") + + meta = agent_meta(cfg) + assert meta.machine == "" and meta.reads_local_files is True + assert "on machine" not in format_agent_listing([meta]) + + +def test_raven_s_keys_are_not_lent_to_a_machine_row(): + loaded = ThirdPartyAcpSubagentConfig(name="x", command="pi", machine="box", lend_keys=["openrouter"]) + assert loaded.lend_keys == [], "dropped on load, with a warning" + + with pytest.raises(ValueError, match="runs on machine 'box'"): + reject_unsupported_acp_fields( + [{"kind": "acp", "name": "x", "command": "pi", "machine": "box", "lendKeys": ["openrouter"]}] + ) + + +def test_moving_a_row_to_a_machine_is_a_new_launch_and_a_local_row_s_digest_is_untouched(): + here = ThirdPartyAcpSubagentConfig(name="a", command="qwen --acp") + there = here.model_copy(update={"machine": "box"}) + elsewhere_root = there.model_copy(update={"remote_cwd": "/srv/agents"}) + + assert len({snapshot_fingerprint(c) for c in (here, there, elsewhere_root)}) == 3 + assert len({fingerprint(c) for c in (here, there, elsewhere_root)}) == 3 + # A row that names no machine digests exactly as it did before the fields + # existed, so no recorded measurement is discarded by the upgrade. + assert snapshot_fingerprint(here) == snapshot_fingerprint(here.model_copy(update={"remote_cwd": None})) + + +# ---- checks ------------------------------------------------------------------------- + + +async def test_the_free_probe_does_not_reach_the_machine(box): + result = await probe.probe_one(remote_config(), source="config") + + assert result.status == "attention" + assert "runs on machine 'Lab box' (box)" in result.detail + assert "run a test" in result.detail + assert box.calls() == [], "a listing never opens an ssh connection" + + +async def test_the_free_probe_shows_the_link_s_state_once_a_dispatch_has_opened_it(box, tmp_path): + cfg = remote_config() + await build_third_party_backend(cfg).run("hi", task_id="t1", workspace=tmp_path, executor=None) + + result = await probe.probe_one(cfg, source="config") + + assert "runs on machine 'Lab box' (box) (connected, 1 agent running)" in result.detail + await close_pool() + assert "(connected, idle)" in (await probe.probe_one(cfg, source="config")).detail + + +async def test_a_link_that_goes_mid_turn_is_waited_out_and_the_turn_finishes_when_it_is_back(box, tmp_path): + """The ssh is the way to the machine, not the agent's life: the agent answers after the link is back.""" + box.nodes._heartbeat_s = 0.1 + box.nodes._lost_after_s = 1.0 + box.nodes._backoff_s = (0.2,) + backend = build_third_party_backend(remote_config(env={"ACP_STUB_PROMPT_DELAY_S": "4"})) + turn = asyncio.ensure_future(backend.run("hi", task_id="t1", workspace=tmp_path, executor=None)) + deadline = time.monotonic() + 20 + while box.nodes.status("box").agents != 1: + assert time.monotonic() < deadline and not turn.done(), "the agent never started" + await asyncio.sleep(0.05) + await asyncio.sleep(0.5) + daemon = box.nodes._nodes["box"].hello["pid"] + + os.kill(box.nodes._nodes["box"].client._proc.pid, signal.SIGKILL) + + reply = await asyncio.wait_for(turn, timeout=30) + assert reply.split("\n\n[raven]")[0] == "pong", "the agent's answer, through the link that came back" + assert box.nodes._nodes["box"].hello["pid"] == daemon, "the same daemon, reached by a new ssh" + assert len([argv for argv in box.calls() if "-T" in argv]) == 2 + + +async def test_a_daemon_that_stops_mid_turn_fails_the_turn_in_one_sentence(box, tmp_path): + box.nodes._heartbeat_s = 0.1 + box.nodes._lost_after_s = 1.0 + box.nodes._backoff_s = (0.2,) + backend = build_third_party_backend(remote_config(mode="prompt_hangs")) + turn = asyncio.ensure_future(backend.run("hi", task_id="t1", workspace=tmp_path, executor=None)) + deadline = time.monotonic() + 20 + while box.nodes.status("box").agents != 1: + assert time.monotonic() < deadline and not turn.done(), "the agent never started" + await asyncio.sleep(0.05) + await asyncio.sleep(0.5) + + os.kill(box.nodes._nodes["box"].hello["pid"], signal.SIGTERM) + + with pytest.raises(AcpConnectionError) as caught: + await asyncio.wait_for(turn, timeout=30) + said = str(caught.value) + assert said.startswith("acp agent 'stub@box'") and "connection ended" in said + assert HOST not in said and str(PORT) not in said + + +async def test_the_free_probe_names_a_machine_that_is_gone(box): + result = await probe.probe_one(remote_config().model_copy(update={"machine": "elsewhere"}), source="config") + + assert result.status == "attention" + assert "machine 'elsewhere' is not registered" in result.detail + + +async def test_verify_handshakes_on_the_machine_in_the_one_check_directory(box): + snapshot = await verify_agent(remote_config()) + + assert snapshot.status == "ready", snapshot.detail + assert (box.home / "raven-work" / "raven-check").is_dir() + + +async def test_verify_starts_the_agent_with_the_entry_s_env(box): + # The stub refuses session/new when its env names that mode; started without + # the variables it would default to answering, and the check would read ready. + snapshot = await verify_agent(remote_config(mode="no_session")) + + assert snapshot.status == "attention" + assert "no session could be opened" in snapshot.detail + + +async def test_verify_of_an_unreachable_machine_is_attention_never_missing(box): + box.mode("unreachable") + + snapshot = await verify_agent(remote_config()) + + assert snapshot.status == "attention", "missing would offer an installer that runs on this computer" + assert "did not answer within the connection timeout" in snapshot.detail + assert HOST not in snapshot.detail + + +async def test_verify_of_a_launch_ssh_refuses_says_so(box): + box.mode("launch_denied") + + snapshot = await verify_agent(remote_config()) + + assert snapshot.status == "attention" + assert "refused the login" in snapshot.detail + assert HOST not in snapshot.detail + + +async def test_verify_of_a_root_the_machine_cannot_make_is_attention_in_its_own_words(box): + # Measured 2026-10-10: a registered machine with a full disk, where the + # root's mkdir fails before the agent starts. The node says why. + snapshot = await verify_agent(remote_config(remote_cwd="/proc/raven-cannot-make-this")) + + assert snapshot.status == "attention", "missing would offer an installer that runs on this computer" + assert "could not make /proc/raven-cannot-make-this" in snapshot.detail + assert HOST not in snapshot.detail + + +async def test_verify_of_an_unregistered_machine_names_it(box): + snapshot = await verify_agent(remote_config().model_copy(update={"machine": "elsewhere"})) + + assert snapshot.status == "attention" + assert "machine 'elsewhere' is not registered" in snapshot.detail + assert box.calls() == [] + + +async def test_verify_that_cannot_make_its_directory_says_so(box): + (box.home / "raven-work").write_text("a file where the root should be\n", encoding="utf-8") + + snapshot = await verify_agent(remote_config()) + + assert snapshot.status == "attention" + assert "could not make" in snapshot.detail and HOST not in snapshot.detail + + +async def test_verify_with_no_ssh_here_is_attention_with_the_reason(box, monkeypatch): + # A PATH holding no ssh at all, so nothing falls through to the real one. + (box.bin / "ssh").unlink() + monkeypatch.setattr(backend_env, "_LOGIN_ENV", {**os.environ, "PATH": str(box.bin)}) + + snapshot = await verify_agent(remote_config()) + + assert snapshot.status == "attention" + assert "ssh" in snapshot.detail + + +async def test_the_free_probe_reads_the_recorded_verdict_until_the_entry_changes(box): + cfg = remote_config() + await probe.record_capabilities(cfg) + + recorded = await probe.probe_one(cfg, source="config") + moved = await probe.probe_one(cfg.model_copy(update={"remote_cwd": "/srv/elsewhere"}), source="config") + + assert recorded.status == "ready", recorded.detail + assert moved.status == "attention" and "launch config changed" in moved.detail + assert [argv for argv in box.calls() if "-T" in argv], "only the recording reached the machine" + + +async def test_a_node_whose_ssh_log_cannot_be_emptied_still_starts(box): + from raven.acp_client import remote + + remote.ssh_log_path("raven-node@box").mkdir() + + assert (await box.nodes.hello("box"))["digest"] == bundle.digest() + + +async def test_a_ping_answers_from_the_machine_and_leaves_one_directory(box): + result = await probe.ping_agent(remote_config()) + + assert result.ok, result.detail + made = sorted(p.name for p in (box.home / "raven-work").iterdir()) + assert made == ["raven-check"], "every check reuses one directory instead of one per press" + + +async def test_a_failed_ping_on_a_machine_offers_no_fix_to_run_here(box): + box.mode("launch_denied") + + result = await probe.ping_agent(remote_config()) + + assert result.ok is False + assert result.remedy is None + assert "on machine 'box'" in result.detail + assert HOST not in result.detail + + +# ---- the announce ------------------------------------------------------------------ + + +async def test_the_announce_does_not_point_at_this_computer_s_workspace(box, tmp_path): + from raven.agent.subagent.manager import SubagentManager + + class _Provider: + def get_default_model(self) -> str: + return "m" + + mgr = SubagentManager(provider=_Provider(), workspace=Path("/tmp"), agents=[remote_config()]) + submitted: list[Any] = [] + mgr.set_submit(submitted.append) + + origin = {"channel": "web", "chat_id": "d", "session_key": "web:s1", "agent": "stub@box", "workspace": "/tmp"} + await mgr._announce_result("t1", "Look", "look", "done", origin, "ok") + + text = submitted[0].text + # Named on every path, as written: a failed or cancelled run's result does + # not close with the resolved directory the way a finished one's does. + assert "Working directory: on machine 'box', ~/raven-work/t1 (made there when the run started)" in text + assert "Working directory: /tmp" not in text + assert "Collected outputs:" not in text, "nothing came back from this run" + + record = tmp_path / "nodes" / "t1.out.md" + await mgr._announce_result("t1", "Look", "look", "done", origin, "ok", record_path=str(record)) + assert "Collected outputs:" not in submitted[1].text, "nothing is brought back from a machine unasked" + + +async def test_the_announce_and_the_record_name_the_machine_s_directory_for_any_handle(box): + from raven.agent.subagent.manager import SubagentManager + + class _Provider: + def get_default_model(self) -> str: + return "m" + + mgr = SubagentManager( + provider=_Provider(), + workspace=Path("/tmp"), + agents=[remote_config(), remote_config(name="deep@box", remote_cwd="/srv/agents/")], + ) + + assert mgr._machine_workdir("stub@box", "reviewer") == ("box", "~/raven-work/reviewer") + assert mgr._machine_workdir("deep@box", "a b") == ( + "box", + "/srv/agents/a_b-" + mgr._machine_workdir("deep@box", "a b")[1][-8:], + ) + assert mgr._machine_workdir("nobody", "t1") == ("", "") + + +# ---- the lane is debug-only -------------------------------------------------------------- + + +def _lane_off(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + + +def test_with_the_lane_off_a_machine_row_stays_off_the_table(box, monkeypatch): + from raven.agent.subagent.registry import AgentRegistry + + local = ThirdPartyAcpSubagentConfig(name="here", command=f"{sys.executable} {_STUB}") + registry = AgentRegistry(build_builtin=lambda row, narrowed: ("backend", row.name)) + + _lane_off(monkeypatch) + registry.apply([remote_config(), local]) + assert "stub@box" not in registry.names() and "here" in registry.names() + + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + registry.apply([remote_config(), local]) + assert "stub@box" in registry.names() + + +async def test_with_the_lane_off_nothing_reaches_the_machine_and_each_door_says_so(box, monkeypatch, tmp_path): + _lane_off(monkeypatch) + said = "the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1" + + with pytest.raises(AcpConnectionError, match=said): + await build_third_party_backend(remote_config()).run("hi", task_id="t1", workspace=tmp_path, executor=None) + probed = await probe.probe_one(remote_config(), source="config") + assert probed.status == "attention" and said in probed.detail + verified = await verify_agent(remote_config()) + assert verified.status == "attention" and said in verified.detail + tested = await probe.run_test(remote_config(), source="config") + assert tested.ok is False and said in tested.detail and tested.remedy is None + assert box.calls() == [], "off is off: no ssh for a dispatch, a probe, a verify or a test" + + +async def test_the_boot_backfill_never_logs_in_to_a_machine(box): + from raven.agent.subagent.backends import acp_snapshot_for + + cfg = remote_config() + await probe._verify_missing_snapshots(object(), [_row_for(cfg)]) + + assert box.calls() == [], "an unattended boot opens no ssh session" + assert acp_snapshot_for(cfg) is None, "the first listing says 'run a test', and that press is the owner's" + + +# ---- what a test on a machine row reports -------------------------------------------------- + + +async def test_a_test_of_a_machine_row_that_needs_a_sign_in_offers_no_fix_for_this_computer(box, monkeypatch): + async def never(*args, **kwargs): + raise AssertionError("asked the agent again in print mode, on this computer") + + monkeypatch.setattr(probe.kimi_code, "explain_refusal", never) + monkeypatch.setattr(probe, "_remedy_for", lambda cfg: pytest.fail("a fix resolved on this computer's PATH")) + + result = await probe.run_test(remote_config(mode="no_session"), source="config") + + assert result.ok is False + assert result.remedy is None, "a sign-in command the page offered to copy would fix this computer" + assert result.detail.endswith("(on machine 'box': any fix is made there)") + assert HOST not in result.detail + + +async def test_the_machine_note_survives_a_refusal_longer_than_the_detail_cap(box): + long_reason = "Authentication required: " + "x" * 2500 + cfg = remote_config(mode="no_session").model_copy( + update={"env": {"ACP_STUB_MODE": "no_session", "ACP_STUB_MESSAGE": long_reason}} + ) + + pinged = await probe.ping_agent(cfg) + tested = await probe.run_test(cfg, source="config") + + for detail in (pinged.detail, tested.detail): + assert len(detail) <= probe._DETAIL_CAP + assert detail.endswith("(on machine 'box': any fix is made there)") + + +def test_the_machine_note_is_kept_whole_inside_the_cap(): + assert probe._on_machine("x" * 5000, "box").endswith(" (on machine 'box': any fix is made there)") + assert len(probe._on_machine("x" * 5000, "box")) == probe._DETAIL_CAP + assert probe._on_machine("short", "box") == "short (on machine 'box': any fix is made there)" + + +# ---- one owner for "no session MCP on a machine" ------------------------------------------ + + +def test_the_one_mcp_predicate_answers_for_a_machine_row_too(): + from raven.agent.subagent.backends import session_mcp_delivered + + assert session_mcp_delivered(session_mcp=True, snapshot=None, machine="box") is False + assert session_mcp_delivered(session_mcp=True, snapshot=None, machine="") is True + assert session_mcp_delivered(session_mcp=True, snapshot=None) is True + + +# ---- the write doors ------------------------------------------------------------------------ + + +@pytest.fixture +def config_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + path = tmp_path / "config.json" + row = {"name": "pi-box", "kind": "acp", "preset": "pi", "command": "pi --acp", "machine": "box", "enabled": False} + config = {"providers": {"openrouter": {"apiKey": "sk-raven-test-not-real"}}, "subagents": {"agents": [row]}} + path.write_text(json.dumps(config), encoding="utf-8") + for module in ("raven.config.loader", "raven.config.self_surface", "raven.rpc.methods.subagents"): + monkeypatch.setattr(f"{module}.get_config_path", lambda: path) + return path + + +def _stored(path: Path) -> dict: + (row,) = json.loads(path.read_text(encoding="utf-8"))["subagents"]["agents"] + return row + + +async def test_the_update_door_refuses_lending_keys_to_a_machine_row(box, config_file): + from raven.rpc.methods.subagents import ConfigValidationError, subagents_update + + # The schema would drop the value on load and the handler would answer + # "updated", so the one path that can say no is the write door. + with pytest.raises(ConfigValidationError, match="cannot be started with Raven's keys"): + await subagents_update({"name": "pi-box", "lend_keys": ["openrouter"]}) + + assert "lendKeys" not in _stored(config_file) and "lend_keys" not in _stored(config_file) + + +# ---- two machines on one disk --------------------------------------------------------------- diff --git a/tests/test_subagent_probe.py b/tests/test_subagent_probe.py index 494747b9a..fa1ff9003 100644 --- a/tests/test_subagent_probe.py +++ b/tests/test_subagent_probe.py @@ -1661,3 +1661,17 @@ def _fake_executable(tmp_path: Path, name: str) -> Path: exe.write_text("#!/bin/sh\nexit 0\n") exe.chmod(0o755) return exe + + +def test_a_machine_failure_says_where_the_fix_is_made_unless_the_fix_is_the_node_installed_from_here(): + from raven.agent.subagent import probe + + there = probe._on_machine("'claude' is not on the login shell PATH", "ravenx2") + assert there.endswith("(on machine 'ravenx2': any fix is made there)") + + here = probe._on_machine( + "Raven node 0.2.4-abc is not installed on machine 'RavenX 2' (ravenx2); install it with " + "`raven ops connection install-node ravenx2`", + "ravenx2", + ) + assert "any fix is made there" not in here and here.endswith("install-node ravenx2`") diff --git a/ui-tui/src/rpc/generated.ts b/ui-tui/src/rpc/generated.ts index 1d0498272..48aaadd40 100644 --- a/ui-tui/src/rpc/generated.ts +++ b/ui-tui/src/rpc/generated.ts @@ -1013,6 +1013,10 @@ export interface SubagentRow { description: string; enabled: boolean; configured: boolean; + /** + * The registered machine an acp row runs on, by its id in the machine registry; null for this computer, and on a server that predates the field. + */ + machine?: string; /** * Discovered under the `agents/` product tree rather than written into config: one of the agent products that ship with this raven, materialized as a row on every table build. Like `builtin` it leaves `configured` false -- there is no config entry to delete, and removing it means removing its folder -- but unlike `builtin` it is a real subprocess with a command, so it is probed and it can be unready (a missing engine wheel leaves it listed and disabled). The wire name predates the tree's rename. Absent from a server that predates discovery. */ @@ -3724,6 +3728,8 @@ export interface SubagentsAddParams { description?: string; model?: string; lend_key?: string; + machine?: string; + remote_cwd?: string; api_key?: string; mcps?: string[]; allow_mcp_secrets?: boolean; diff --git a/ui-web/src/rpc/generated.ts b/ui-web/src/rpc/generated.ts index 058836129..d8fbccca8 100644 --- a/ui-web/src/rpc/generated.ts +++ b/ui-web/src/rpc/generated.ts @@ -826,6 +826,10 @@ export interface SubagentRow { description: string; enabled: boolean; configured: boolean; + /** + * The registered machine an acp row runs on, by its id in the machine registry; null for this computer, and on a server that predates the field. + */ + machine?: string; /** * Discovered under the `agents/` product tree rather than written into config: one of the agent products that ship with this raven, materialized as a row on every table build. Like `builtin` it leaves `configured` false -- there is no config entry to delete, and removing it means removing its folder -- but unlike `builtin` it is a real subprocess with a command, so it is probed and it can be unready (a missing engine wheel leaves it listed and disabled). The wire name predates the tree's rename. Absent from a server that predates discovery. */ @@ -2856,6 +2860,8 @@ export interface SubagentsAddParams { description?: string; model?: string; lend_key?: string; + machine?: string; + remote_cwd?: string; api_key?: string; mcps?: string[]; allow_mcp_secrets?: boolean; From f67f806ed950469b47d72d2175ffd4be59fd0dca Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:45:52 +0800 Subject: [PATCH 03/44] feat(*): read, write and deliver files on a registered machine by address The file tools take a path on a registered machine as it is written for scp -- `user@host:/path` or `scp://user@host:port//path` -- and answer it through the machine's daemon: read_file, list_dir, find and grep run Raven's own tools there and bring back the answer; write_file and edit_file write there with the digest of the last read as the expected state, so a file changed since is refused and nothing is written; deliver_files fetches one file by its address, whole and checked, under a size cap. A path is read inside the directories Raven's agents work in on that machine without asking, elsewhere once per machine and directory for the session, writes asked apart from reads. What is shown back is the address form, never a bare address; and the shell refuses ssh, scp, sftp and rsync to a registered machine, naming the file tools. Third of three changes on this design. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 12 + raven/agent/tools/deliver.py | 87 +++++- raven/agent/tools/file_search.py | 14 +- raven/agent/tools/filesystem.py | 33 +- raven/agent/tools/machine_exec.py | 185 ++++++++++- raven/agent/tools/remote_files.py | 127 ++++++++ raven/agent/tools/shell.py | 19 +- raven/permissions/builtin.py | 20 +- raven/permissions/rules.py | 8 + tests/test_deliver_files_tool.py | 99 ++++++ tests/test_permissions_remote_read.py | 168 ++++++++++ tests/test_remote_file_tools.py | 432 ++++++++++++++++++++++++++ tests/test_shell_machine_channel.py | 70 +++++ 13 files changed, 1256 insertions(+), 18 deletions(-) create mode 100644 raven/agent/tools/remote_files.py create mode 100644 tests/test_permissions_remote_read.py create mode 100644 tests/test_remote_file_tools.py diff --git a/CONTEXT.md b/CONTEXT.md index 7d23f5e69..a37b54a54 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -433,6 +433,18 @@ entry already rules the bare word out for a computer); "node id" for the install name (a **Node id** is a delegated task's address); "agent" or "server" (it answers only the Raven that reaches it, only over ssh). +**By address** (`remote.files_note`, `raven/ops/remote_path.py`): +How a file crosses machines: it does not. A file that comes with a task is named to the +agent on the **Machine** by its address on the computer that sent it, and the agent uses +it where the machine can read it (the same path on a disk both mount, a copy already +there) or says it could not be found and goes on; what a session makes stays on the +machine, named in the reply by address (the scp form), where the file tools read it by +that address and `deliver_files` fetches one file by it on request. Nothing is copied +ahead of a prompt, nothing is brought back unasked, and no two machines are ever told to +share a disk: whether they do shows in whether a path reads there. +_Avoid_: "transfer", "sync", "collect" (nothing moves on its own); "shared disk" as a +registry fact (it is not recorded: a path either reads on a machine or does not). + **Roster** (`format_agent_listing`): The agent table rendered as the text spliced into `spawn`'s and `run_subagent_dag`'s tool descriptions — `name [stateful, local-files, live-progress] (description)`. The model's only diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index 110b1713e..61d288588 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -14,6 +14,8 @@ from raven.agent.tools.deliverables import DeliverableStore from raven.agent.tools.filesystem import _with_current_root, resolve_path from raven.contracts.tool import Tool +from raven.ops import remote_path +from raven.ops.remote_path import RemotePath _last_stamp = 0 @@ -53,6 +55,60 @@ class _DeliverCtx: session_key: str +def _on_machine(raw: str) -> RemotePath | str | None: + """The machine ``raw`` names, the sentence for a remote spelling that names none, or ``None`` for a path here.""" + try: + return remote_path.target(raw) + except remote_path.RemotePathError as exc: + return str(exc) + + +async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path | None, str]: + """``where``'s file, fetched from its machine to ``/pulled//``; or why it was not. + + Only a file this process may read without asking -- under the directories + this Raven's agents work in there, or one it has already read (and so was + allowed) -- is fetched here: a delivery is not a way around the read + check. One over :data:`PULL_FILE_BUDGET` stays, and is named with its + size. + """ + pool = remote_path.nodes() + label = remote_path._label(where._row) + if pool is None: + return None, f"files on machine {label} cannot be brought back by this process" + try: + home = await pool.home(where.machine) + place = where.absolute(home) + roots = remote_path.reading_roots(where) + inside = any(remote_path.within(place, remote_path.absolute(root, home)) for root in roots) + if not inside and remote_path.last_read(where.machine, place) is None: + return None, ( + f"{where.display(place)} is outside the directories this Raven's agents work in on machine " + f"{label}; read it with read_file first" + ) + fence = [remote_path.fence("read_file", where, place, home)] + info = await pool.stat(where.machine, place, roots=fence) + if info.get("kind") != "file": + return None, f"not found or not a regular file on machine {label}" + size = int(info.get("size") or 0) + if size > PULL_FILE_BUDGET: + return None, ( + f"{_human_size(size)} is over the {_human_size(PULL_FILE_BUDGET)} " + f"limit for a file brought back from machine {label}; read it there instead" + ) + local = (workspace or Path.cwd()) / "pulled" / where.machine / Path(place).name + await pool.pull_file(where.machine, place, local, roots=fence) + except remote_path.NodeCallError as exc: + return None, str(exc) + except OSError as exc: + return None, f"could not write the file brought back: {exc.strerror or exc}" + return local, "" + + +#: The most a file fetched from a registered machine for a delivery may weigh; a bigger one is read there. +PULL_FILE_BUDGET = 20 * 1024 * 1024 + + def _human_size(size: int) -> str: value = float(size) for unit in ("B", "KB", "MB", "GB"): @@ -167,15 +223,30 @@ async def execute( if not raw: invalid.append({"path": raw, "reason": "empty path"}) continue - try: - bound = workdir.current() - resolved = resolve_path(raw, bound or self._workspace, _with_current_root(self._allowed_dirs, bound)) - except PermissionError as exc: - invalid.append({"path": raw, "reason": str(exc)}) - continue - except (OSError, ValueError, RuntimeError) as exc: - invalid.append({"path": raw, "reason": f"cannot resolve path: {exc}"}) + elsewhere = _on_machine(raw) + if isinstance(elsewhere, str): + invalid.append({"path": raw, "reason": elsewhere}) continue + if elsewhere is not None: + # A file on a registered machine is brought back first, within + # the budget, and delivered from where it landed. + brought, reason = await _brought_back(elsewhere, self._workspace) + if brought is None: + invalid.append({"path": raw, "reason": reason}) + continue + resolved = brought + else: + try: + bound = workdir.current() + resolved = resolve_path( + raw, bound or self._workspace, _with_current_root(self._allowed_dirs, bound) + ) + except PermissionError as exc: + invalid.append({"path": raw, "reason": str(exc)}) + continue + except (OSError, ValueError, RuntimeError) as exc: + invalid.append({"path": raw, "reason": f"cannot resolve path: {exc}"}) + continue key = str(resolved) if key in seen: diff --git a/raven/agent/tools/file_search.py b/raven/agent/tools/file_search.py index e49509725..18aec5690 100644 --- a/raven/agent/tools/file_search.py +++ b/raven/agent/tools/file_search.py @@ -24,7 +24,7 @@ from pathlib import Path from typing import Any -from raven.agent.tools import tree_walk +from raven.agent.tools import remote_files, tree_walk from raven.agent.tools.filesystem import _FsTool # Pseudo / system filesystem roots that must never be tree-walked. A model that @@ -133,6 +133,16 @@ async def execute( limit: int | None = None, **kwargs: Any, ) -> str: + params = { + "pattern": pattern, + "glob": glob, + "output_mode": output_mode, + "case_insensitive": case_insensitive, + "context": context, + "limit": limit, + } + if (remote := await remote_files.answer("grep", path, params)) is not None: + return str(remote) cap = limit or self._DEFAULT_LIMIT try: re.compile(pattern) @@ -523,6 +533,8 @@ async def execute( limit: int | None = None, **kwargs: Any, ) -> str: + if (remote := await remote_files.answer("find", path, {"pattern": pattern, "limit": limit})) is not None: + return str(remote) cap = limit or self._DEFAULT_LIMIT try: base = self._resolve(path) diff --git a/raven/agent/tools/filesystem.py b/raven/agent/tools/filesystem.py index 96cf97e6f..df4551829 100644 --- a/raven/agent/tools/filesystem.py +++ b/raven/agent/tools/filesystem.py @@ -10,8 +10,9 @@ from typing import Any from raven.agent import workdir -from raven.agent.tools import tree_walk +from raven.agent.tools import remote_files, tree_walk from raven.contracts.tool import FileChange, Tool, ToolResult +from raven.ops import remote_path from raven.utils.images import detect_image_mime _DIFF_MAX_LINES = 400 @@ -31,6 +32,15 @@ def _write_text_bytes(content: str) -> bytes: return content.replace("\n", os.linesep).encode("utf-8") +def _machine_of(path: str) -> str | None: + """The registered machine ``path`` is written for, or ``None`` for a path here (or one that names no machine).""" + try: + where = remote_path.target(path) + except remote_path.RemotePathError: + return None + return where.machine if where is not None else None + + def _unified(before: str, after: str, name: str) -> str | None: """Unified diff of one write, or None when there is nothing useful to show. @@ -228,6 +238,9 @@ def _read_image(self, fp: Path, mime: str) -> ToolResult: ) async def execute(self, path: str, offset: int = 1, limit: int | None = None, **kwargs: Any) -> str | ToolResult: + # A file on a registered machine is read there, by its Raven node. + if (remote := await remote_files.answer("read_file", path, {"offset": offset, "limit": limit})) is not None: + return remote try: fp = self._resolve(path) if not fp.exists(): @@ -315,6 +328,10 @@ def approval_evidence(self, params: dict[str, Any]) -> dict[str, Any]: """The write as a diff against the file as it stands, before it happens.""" path = str(params.get("path") or "") content = str(params.get("content") or "") + if (where := _machine_of(path)) is not None: + # The file is on another machine, so there is no file here to diff + # against; the card names the machine and shows the path as written. + return {"path": path, "machine": where} try: fp = self._resolve(path) except PermissionError: @@ -382,6 +399,9 @@ async def execute(self, path: str, content: str, mode: str = "overwrite", **kwar # it must never silently become is an overwrite. if mode == "append" and not content: return "Error: write_file with mode=append needs content; refusing to append nothing." + # A file on a registered machine is written there, by its Raven node. + if (remote := await remote_files.answer("write_file", path, {"content": content, "mode": mode})) is not None: + return remote try: fp = self._resolve(path) # Read before writing: a whole-file write carries no record of what @@ -503,11 +523,13 @@ def approval_evidence(self, params: dict[str, Any]) -> dict[str, Any]: The snippets, not the file: the file is read only if the edit runs, and what the reader is asked to allow is exactly this substitution.""" path = str(params.get("path") or "") + old, new = str(params.get("old_text") or ""), str(params.get("new_text") or "") + if (where := _machine_of(path)) is not None: + return {"path": path, "machine": where, "created": False, "diff": _unified(old, new, path)} try: shown = str(self._resolve(path)) except PermissionError: shown = path - old, new = str(params.get("old_text") or ""), str(params.get("new_text") or "") return {"path": shown, "created": False, "diff": _unified(old, new, shown)} @property @@ -542,6 +564,10 @@ async def execute( replace_all: bool = False, **kwargs: Any, ) -> str | ToolResult: + # A file on a registered machine is edited there, by its Raven node. + sent = {"old_text": old_text, "new_text": new_text, "replace_all": replace_all} + if (remote := await remote_files.answer("edit_file", path, sent)) is not None: + return remote try: fp = self._resolve(path) if not fp.exists(): @@ -658,6 +684,9 @@ async def execute( max_entries: int | None = None, **kwargs: Any, ) -> str: + params = {"recursive": recursive, "max_entries": max_entries} + if (remote := await remote_files.answer("list_dir", path, params)) is not None: + return str(remote) try: dp = self._resolve(path) if not dp.exists(): diff --git a/raven/agent/tools/machine_exec.py b/raven/agent/tools/machine_exec.py index d9a2a9529..5efe27d29 100644 --- a/raven/agent/tools/machine_exec.py +++ b/raven/agent/tools/machine_exec.py @@ -272,6 +272,181 @@ def _ssh_destinations(command: str) -> list[tuple[str, int]]: return found +#: The programs that copy files to or from a machine over ssh, and so carry a +#: destination written ``[user@]host:path``, ``host::module`` or as a URI. +_COPIERS = frozenset({"scp", "sftp", "rsync"}) +# The short options of each that take a value, so the value is never read as +# an operand: ``scp -P 58717 a.txt host:b`` names a port, not a file. rsync's +# ``-P`` takes none (it is ``--partial --progress``). +_COPY_VALUE_FLAGS = { + "scp": frozenset("iFclSJDoP"), + "sftp": frozenset("iFclSJDoPbsBRX"), + "rsync": frozenset("eBTMf"), +} +# rsync's long options that take a value the next word may carry. +_RSYNC_VALUE_OPTIONS = frozenset( + { + "--rsh", + "--port", + "--files-from", + "--exclude", + "--include", + "--exclude-from", + "--include-from", + "--log-file", + "--timeout", + "--contimeout", + "--bwlimit", + "--chmod", + "--chown", + "--rsync-path", + "--password-file", + "--out-format", + "--temp-dir", + "--compare-dest", + "--copy-dest", + "--link-dest", + "--backup-dir", + "--suffix", + "--max-size", + "--min-size", + "--filter", + "--partial-dir", + "--iconv", + "--block-size", + "--max-delete", + "--modify-window", + "--address", + "--sockopts", + "--checksum-choice", + } +) +_URI_DESTINATION = re.compile( + r"^(?:scp|sftp|rsync)://(?:[^/@]+@)?(?P\[[^\]]+\]|[^/:]+)(?::(?P\d+))?(?:/|$)", re.IGNORECASE +) +# The port an rsync ``-e``/``--rsh`` value hands its ssh: ``ssh -p 58717``, ``ssh -o Port=58717``. +_RSH_PORT = re.compile(r"(?:^|\s)-p\s*(\d+)|(?:^|\s)-o\s*[\"']?Port[=\s](\d+)", re.IGNORECASE) + + +def _copy_host(word: str) -> tuple[str, int | None] | None: + """The host (and the port a URI carries) a scp, sftp or rsync operand names, or ``None`` for a local path. + + scp's own rule: a colon before any slash makes a remote spelling, so + ``./a:b`` and ``/tmp/a:b`` are files here. ``host::module`` and + ``rsync://`` are rsync's daemon forms; the host still counts. + """ + if match := _URI_DESTINATION.match(word): + port = match.group("port") + return match.group("host").strip("[]").lower(), (int(port) if port else None) + if "::" in word: + head = word.split("::", 1)[0] + elif ":" in word: + head = word.split(":", 1)[0] + else: + return None + if not head or "/" in head: + return None + host = head.rsplit("@", 1)[-1].strip("[]").lower() + return (host, None) if host else None + + +def _copy_destinations(command: str) -> list[tuple[str, int]]: + """Every host and port a ``scp``, ``sftp`` or ``rsync`` word in a shell command would reach. + + Read the way :func:`_ssh_destinations` reads ssh, program by program: the + port from scp's and sftp's ``-P`` or ``-o Port``, from rsync's ``--port`` + or the ssh its ``-e``/``--rsh`` names, else 22; every operand written as a + remote path names the host, and all of one command's share its port. + """ + try: + lexer = shlex.shlex(command, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + tokens = list(lexer) + except ValueError: + return [] + found: list[tuple[str, int]] = [] + index = 0 + while index < len(tokens): + program = PurePosixPath(tokens[index].lstrip("\\")).name + if program not in _COPIERS: + index += 1 + continue + index += 1 + port: int | None = None + operands: list[str] = [] + options_ended = False + while index < len(tokens): + word = tokens[index] + if word in _COMMAND_SEPARATORS or PurePosixPath(word.lstrip("\\")).name in _COPIERS | {"ssh"}: + break + following = tokens[index + 1] if index + 1 < len(tokens) else None + if word in _REDIRECTIONS: + index += 1 + if following is not None and following not in _COMMAND_SEPARATORS and following not in _REDIRECTIONS: + index += 1 + continue + if word.isdigit() and following in _REDIRECTIONS: + index += 1 + continue + index += 1 + if word == "--" and not options_ended: + options_ended = True + continue + if options_ended or not word.startswith("-") or len(word) == 1: + operands.append(word) + continue + if word.startswith("--"): + name, _, value = word.partition("=") + if not _ and program == "rsync" and name in _RSYNC_VALUE_OPTIONS and index < len(tokens): + value = tokens[index] + index += 1 + if name == "--port" and value.isdigit(): + port = int(value) + elif name == "--rsh" and (match := _RSH_PORT.search(value)): + port = int(match.group(1) or match.group(2)) + continue + for offset, letter in enumerate(word[1:]): + if letter not in _COPY_VALUE_FLAGS[program]: + continue + value = word[offset + 2 :] + if not value and index < len(tokens): + value = tokens[index] + index += 1 + if letter == "P" and value.isdigit(): + port = int(value) + elif letter == "o": + pair = _OPTION_SPLIT.split(value.strip(), maxsplit=1) + if len(pair) == 2 and pair[0].lower() == "port" and pair[1].strip().isdigit(): + port = int(pair[1].strip()) + elif letter == "e" and (match := _RSH_PORT.search(value)): + port = int(match.group(1) or match.group(2)) + break + for word in operands: + named = _copy_host(word) + if named is None and program == "sftp": + # sftp's one operand is the machine, with or without a path: + # ``sftp root@h`` opens a session there. + head = word.split(":", 1)[0] + host = head.rsplit("@", 1)[-1].strip("[]").lower() + named = (host, None) if host and "/" not in head else None + if named is not None: + host, explicit = named + found.append((host, explicit or port or 22)) + return found + + +def raw_copy_target(command: str) -> dict[str, Any] | None: + """The registered machine a plain-shell ``scp``, ``sftp`` or ``rsync`` reaches, or ``None``. + + The sibling of :func:`raw_ssh_target` for the copying clients. Refused by + ``exec`` not because a copy starts anything there -- it does not -- but + because the address it carries would then stand in the model's text where + a path shown by the file tools belongs, and the copy would go around the + budgets and the read check those tools keep. + """ + return _registered(_copy_destinations(command)) + + def raw_ssh_target(command: str) -> dict[str, Any] | None: """The registered machine a plain-shell command reaches over its own ssh. @@ -288,10 +463,14 @@ def raw_ssh_target(command: str) -> dict[str, Any] | None: typed ``ssh -p root@ '... &'`` from the local shell 58 times to start GPU work: the look here is capped at 60 s and on-call's job runner was not theirs to call, so the address was the path of least resistance, - and the ledger never saw the runs. Only ssh is matched -- ``scp`` and - ``rsync`` move files and start nothing on the far side. + and the ledger never saw the runs. ``scp``, ``sftp`` and ``rsync`` are + :func:`raw_copy_target`'s, for a reason of their own. """ - destinations = _ssh_destinations(command) + return _registered(_ssh_destinations(command)) + + +def _registered(destinations: list[tuple[str, int]]) -> dict[str, Any] | None: + """The registry row at the first of ``destinations`` that is registered, host and port both.""" if not destinations: return None try: diff --git a/raven/agent/tools/remote_files.py b/raven/agent/tools/remote_files.py new file mode 100644 index 000000000..b82da45c7 --- /dev/null +++ b/raven/agent/tools/remote_files.py @@ -0,0 +1,127 @@ +"""A file call whose path is on a registered machine, answered by that machine's Raven node. + +``read_file``, ``list_dir``, ``find``, ``grep``, ``write_file`` and +``edit_file`` take a remote path (:mod:`raven.ops.remote_path`) as readily as +a local one. A remote path never touches this computer's disk: it is placed +on its machine -- absolute, against the login home the node reported -- +fenced to the directory the permission check allowed, and sent to the +machine's node (:mod:`raven.node.client`), which runs the very tool the call +named there. Only the answer comes back, and every path in it is shown the +way scp writes it, with the registry's user, address and port (C28). + +A read remembers the file's digest; a write that replaces the whole file +then tells the node what it expects to replace, and a file changed in +between is left alone, with a sentence that says to read it again. + +Every failure is one sentence the tool returns: a path that names no +registered machine, a machine without this Raven's node, one that does not +answer. +""" + +from __future__ import annotations + +import re +from typing import Any + +from raven.contracts.tool import FileChange, ToolResult +from raven.ops import remote_path +from raven.ops.remote_path import RemotePath, RemotePathError + +#: Which node method answers which tool (``raven.node.protocol``; spelled out +#: here because the node imports these tools, and they must not import it back). +_METHODS = { + "read_file": "fs/read", + "list_dir": "fs/list", + "find": "fs/find", + "grep": "fs/grep", + "write_file": "fs/write", + "edit_file": "fs/edit", +} + + +async def answer(tool: str, path: Any, params: dict[str, Any]) -> str | ToolResult | None: + """``tool``'s answer for ``path`` on its machine, or ``None`` when ``path`` is local. + + ``params`` are the call's other arguments, passed to the node as they are. + """ + if not isinstance(path, str): + return None + try: + where = remote_path.target(path) + except RemotePathError as exc: + return f"Error: {exc}" + if where is None: + return None + pool = remote_path.nodes() + if pool is None: + return f"Error: {path} is on another machine, and this process does not read registered machines' files" + try: + home = await pool.home(where.machine) + place = where.absolute(home) + roots = [remote_path.fence(tool, where, place, home)] + sent = {**params, "path": place, "roots": roots} + if ( + tool == "write_file" + and params.get("mode") != "append" + and (read := remote_path.last_read(where.machine, place)) + ): + # What this process last read there, so a file changed since is + # not written over unseen (``fs/write``'s ``expect``). + sent["expect"] = read + got = await pool.call(where.machine, _METHODS[tool], sent) + except remote_path.NodeCallError as exc: + if exc.reason == "changed": + remote_path.forget_read(where.machine, place) + return ( + f"Error: {where.display(place)} on machine {remote_path._label(where._row)} changed after it was " + "read; nothing was written. Read it again before writing." + ) + return f"Error: {exc}" + if tool == "read_file" and isinstance(got.get("sha256"), str): + remote_path.remember_read(where.machine, place, got["sha256"]) + elif tool in ("write_file", "edit_file"): + remote_path.forget_read(where.machine, place) + return _shown(got, where, [place, *roots]) + + +def _shown(got: dict[str, Any], where: RemotePath, paths: list[str]) -> str | ToolResult: + """The node's answer with each path it was given shown as scp writes it. + + One pass, longest path first: a shown path contains the plain one, and a + second pass over it would show it twice. + """ + swaps = sorted({p for p in paths if p.startswith("/")}, key=len, reverse=True) + pattern = re.compile("|".join(re.escape(p) for p in swaps)) if swaps else None + + def show(text: Any) -> str: + out = str(text or "") + return pattern.sub(lambda m: where.display(m.group(0)), out) if pattern else out + + text = show(got.get("text")) + if "blocks" not in got: + return text + blocks = [] + for block in got.get("blocks") or []: + if isinstance(block, dict) and block.get("type") == "text": + block = {**block, "text": show(block.get("text"))} + blocks.append(block) + display = got.get("display") + change = got.get("file_change") + return ToolResult( + model_text=text, + display_text=show(display) if display is not None else None, + ok=bool(got.get("ok", True)), + blocks=blocks, + # A write's account of what it changed, for a surface with a diff + # view of its own: the file's whole content both ways, under the + # path as scp writes it. + diff=show(got["diff"]) if isinstance(got.get("diff"), str) else None, + file_change=FileChange( + path=show(change.get("path")), after=str(change.get("after") or ""), before=change.get("before") + ) + if isinstance(change, dict) + else None, + ) + + +__all__ = ["answer"] diff --git a/raven/agent/tools/shell.py b/raven/agent/tools/shell.py index dff0dbd83..d7a5f3022 100644 --- a/raven/agent/tools/shell.py +++ b/raven/agent/tools/shell.py @@ -310,7 +310,24 @@ async def execute( f"this computer, and that machine is registered as {conn_id!r}. Look at it with " f"exec(machine={conn_id!r}) (capped at 60s, nothing left running), and hand anything " "longer, or anything that must keep running, to the on-call agent's ops_submit " - "(budget, dedup, ledger). scp and rsync to it are still fine here. Nothing was run." + "(budget, dedup, ledger). Its files are read and written with read_file, write_file " + "and edit_file on the path the way scp writes it. Nothing was run." + ) + from raven.agent.tools.machine_exec import raw_copy_target + + if (row := raw_copy_target(command)) is not None: + # A copy over scp, sftp or rsync starts nothing there, but it + # carries the address into the model's text and goes around the + # file tools' budgets and read check. The tools take the same + # spelling, so the model learns the path, not just that this + # one closed. + conn_id = str(row.get("id") or "").strip() + return ( + f"Error: this command copies files to or from {row.get('display_name') or conn_id} with scp, " + f"sftp or rsync from this computer, and that machine is registered as {conn_id!r}. Its files " + "are reached through its Raven node instead: give read_file, list_dir, find, grep, write_file " + "and edit_file the path the way scp writes it (user@host:/path, or scp://user@host:port//path), " + "and deliver_files brings a file back. Nothing was run." ) cwd = self._cwd_for(working_dir) diff --git a/raven/permissions/builtin.py b/raven/permissions/builtin.py index d0a506cd6..08283a2c5 100644 --- a/raven/permissions/builtin.py +++ b/raven/permissions/builtin.py @@ -30,6 +30,7 @@ from loguru import logger from raven.contracts.permissions import DecisionSource, Deny, NeedsApproval +from raven.ops import remote_path from raven.permissions.shell_policy import CommandDecision, ShellCommandPolicy # Windows del/rmdir are absent here on purpose too: an ordinary del /f or @@ -113,9 +114,11 @@ def session_keys(tool_name: str, params: dict[str, Any], ask_segments: tuple[str work on that site, and a click and the typing that follows it are one such piece of work, not two. One that names no site -- the page it acts on has none yet: a tab it is about to open, or one still blank -- keys the exact - call, so its grant is never presented by a call to a real site. Every other - tool keys the exact call -- a `path` field on an unknown tool says nothing - about what its other fields do. + call, so its grant is never presented by a call to a real site. A read on a + registered machine is keyed by its machine and the directory it reaches, + placed on that machine, so the grant covers that directory however the path + is written. Every other tool keys the exact call -- a `path` field on an + unknown tool says nothing about what its other fields do. """ if tool_name == "exec" and ask_segments: working_dir = params.get("working_dir") @@ -124,6 +127,17 @@ def session_keys(tool_name: str, params: dict[str, Any], ask_segments: tuple[str return tuple( sha256(f"exec\x00{segment}\x00cwd:{cwd}\x00on:{machine}".encode()).hexdigest() for segment in ask_segments ) + if (reach := remote_path.reading(tool_name, params)) is not None: + # A read on a registered machine is granted for its machine and + # directory, placed on the machine, so the same file written another + # way presents the same grant (C30). + material = f"remote-read\x00{reach.target.machine}\x00{reach.directory}" + return (sha256(material.encode()).hexdigest(),) + if (reach := remote_path.writing(tool_name, params)) is not None: + # A write there likewise, and apart from the read: allowing a look at + # a directory is not allowing changes to it. + material = f"remote-write\x00{reach.target.machine}\x00{reach.directory}" + return (sha256(material.encode()).hexdigest(),) path = params.get("path") if tool_name in PATH_KEYED_TOOLS and isinstance(path, str) and path.strip(): material = f"{tool_name}\x00path:{os.path.normpath(path.strip())}\x00cwd:{_bound_workdir()}" diff --git a/raven/permissions/rules.py b/raven/permissions/rules.py index 40f123dca..2a00cab21 100644 --- a/raven/permissions/rules.py +++ b/raven/permissions/rules.py @@ -54,6 +54,7 @@ from raven.config.self_surface import carries_secret_value from raven.contracts.permissions import Tier +from raven.ops import remote_path from raven.permissions.shell_policy import ( _COMMAND_RUNNERS, _MAX_EMBEDDED_SHELL_DEPTH, @@ -673,6 +674,13 @@ def self_config_tier(tool_name: str, params: dict[str, Any] | None = None) -> Ti def default_tier(tool_name: str, params: dict[str, Any] | None = None) -> Tier: + if (reach := remote_path.reading(tool_name, params) or remote_path.writing(tool_name, params)) is not None: + # A registered machine's disk is not this computer's: its login is + # often root and its disks shared. The directories this Raven's agents + # work in there and the row's own paths read and write freely; + # anywhere else asks, once per machine and directory, a write apart + # from a read (C30; ``builtin.session_keys``). + return Tier.ALLOW if reach.inside else Tier.ASK if tool_name in DEFAULT_ALLOW_TOOLS: return Tier.ALLOW if (params or {}).get("action") in DEFAULT_ALLOW_ACTIONS.get(tool_name, ()): diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 4ba3a4d0a..5875c1071 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import json import time from pathlib import Path @@ -361,3 +362,101 @@ def test_a_re_delivery_refreshes_what_the_registry_says_the_file_is(tmp_path) -> rows = store.for_conversation("web:s1") assert [r.title for r in rows] == ["second cut"] + + +# --- a file on a registered machine is brought back first ---------------------------------- + + +@pytest.fixture +def machines(tmp_path, monkeypatch): + from types import SimpleNamespace + + from raven.ops import remote_path + from tests.test_node_client import row + + registry = tmp_path / "connections.json" + registry.write_text(json.dumps({"connections": [row()]}), encoding="utf-8") + monkeypatch.setenv("RAVEN_CONNECTIONS", str(registry)) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + monkeypatch.setattr(remote_path, "_HOMES", {}) + monkeypatch.setattr(remote_path, "_READ", {}) + monkeypatch.setattr( + "raven.config.loader.load_config", lambda *a, **k: SimpleNamespace(subagents=SimpleNamespace(agents=[])) + ) + pool = _RemotePool() + monkeypatch.setattr(remote_path, "_NODES", lambda: pool) + return pool + + +class _RemotePool: + """A node pool with one file of a given size, which it hands over as x's.""" + + def __init__(self) -> None: + self.size = 5 + self.kind = "file" + self.pulled: list[tuple[str, Path, list[str]]] = [] + + async def home(self, machine: str) -> str: + return "/root" + + async def stat(self, machine: str, path: str, *, roots: list[str], hash: bool = False) -> dict: + return {"exists": self.kind != "missing", "kind": self.kind, "size": self.size, "mtime": 1.0} + + async def pull_file(self, machine: str, remote_file: str, local: Path, *, roots: list[str]) -> int: + local.parent.mkdir(parents=True, exist_ok=True) + local.write_bytes(b"x" * self.size) + self.pulled.append((remote_file, local, roots)) + return self.size + + +async def test_a_file_on_a_machine_is_brought_back_and_delivered(tool, tmp_path, machines) -> None: + summary = await tool.execute(files=[{"path": "box:raven-work/t1/report.pdf", "title": "Report"}]) + + manifest = tool.take_metadata()["raven_delivery"] + assert manifest["invalid"] == [], summary + (entry,) = manifest["files"] + landed = tmp_path / "chanwork" / "pulled" / "box" / "report.pdf" + assert entry["name"] == "report.pdf" and entry["path"] == str(landed) and entry["size"] == 5 + assert landed.read_bytes() == b"xxxxx" + assert machines.pulled == [("/root/raven-work/t1/report.pdf", landed, ["/root/raven-work"])] + + +async def test_a_file_over_the_budget_stays_on_its_machine(tool, machines) -> None: + machines.size = 21 * 1024 * 1024 + + await tool.execute(files=[{"path": "box:raven-work/t1/model.bin"}]) + + assert tool.take_metadata() is None, "nothing was delivered, so no manifest" + assert machines.pulled == [] + + +async def test_what_was_not_delivered_says_why(tool, machines) -> None: + from raven.ops import remote_path + + machines.size = 21 * 1024 * 1024 + big = await tool.execute(files=[{"path": "box:raven-work/t1/model.bin"}]) + machines.size = 5 + outside = await tool.execute(files=[{"path": "box:.ssh/id_rsa"}]) + remote_path.remember_read("box", "/root/.ssh/id_rsa", "ab" * 32) + read_first = await tool.execute(files=[{"path": "box:.ssh/id_rsa"}]) + nobody = await tool.execute(files=[{"path": "nobox:/etc/hostname"}]) + machines.kind = "dir" + folder = await tool.execute(files=[{"path": "box:raven-work/t1"}]) + + assert ( + "21.0 MB is over the 20.0 MB limit for a file brought back from machine 'Lab box' (box); read it there instead" + in big + ) + assert ( + "is outside the directories this Raven's agents work in on machine 'Lab box' (box); read it with read_file first" + in outside + ) + assert "id_rsa" in read_first and "outside" not in read_first, "a file already read was allowed to be" + assert "is written as a path on another machine" in nobody + assert "not found or not a regular file on machine 'Lab box' (box)" in folder + # The machine is named by its label in every sentence; its address appears + # only inside a path shown the way scp writes it (C28). + assert ( + outside.count("203.0.113.7") == 1 and "scp://worker@203.0.113.7:58717//root/.ssh/id_rsa is outside" in outside + ) + assert "203.0.113.7" not in big + nobody + folder diff --git a/tests/test_permissions_remote_read.py b/tests/test_permissions_remote_read.py new file mode 100644 index 000000000..c6681b8fe --- /dev/null +++ b/tests/test_permissions_remote_read.py @@ -0,0 +1,168 @@ +"""The permission check for a read on a registered machine (C30). + +A registered machine's login is often root and its disks shared, so a read +there is not waved through the way a read here is: the directories this +Raven's agents work in there and the row's own paths read freely; anywhere +else asks, and a grant for the session covers that machine and directory +however the path is written. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from raven.config.schema import PermissionsConfig +from raven.contracts.permissions import Allow, DecisionSource, NeedsApproval, Tier +from raven.ops import remote_path +from raven.permissions.builtin import session_keys +from raven.permissions.rules import default_tier +from raven.permissions.session import remember_allowed +from tests.test_permissions_gate import bind, gate_for + +ROWS = [ + {"id": "box", "display_name": "Lab box", "host": "203.0.113.7", "port": 58717, "user": "root"}, + {"id": "nas", "host": "198.51.100.9", "port": 22, "user": "alice", "paths": ["/data/shared"]}, +] + + +@pytest.fixture(autouse=True) +def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + path = tmp_path / "connections.json" + path.write_text(json.dumps({"connections": ROWS}), encoding="utf-8") + monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + monkeypatch.setattr(remote_path, "_HOMES", {}) + monkeypatch.setattr( + "raven.config.loader.load_config", + lambda *a, **k: SimpleNamespace( + subagents=SimpleNamespace(agents=[SimpleNamespace(machine="box", remote_cwd="/srv/agents")]) + ), + ) + + +@pytest.mark.parametrize( + ("tool", "path", "tier"), + [ + ("read_file", "box:raven-work/t1/report.md", Tier.ALLOW), + ("list_dir", "box:~/raven-work", Tier.ALLOW), + ("grep", "box:/srv/agents/t1", Tier.ALLOW), + ("find", "nas:/data/shared/runs", Tier.ALLOW), + ("read_file", "box:.ssh/id_rsa", Tier.ASK), + ("read_file", "box:raven-work/../.ssh/id_rsa", Tier.ASK), + ("list_dir", "nas:/data", Tier.ASK), + ("grep", "box:", Tier.ASK), + # Reaches no machine: the tool refuses it, nothing to ask about. + ("read_file", "nowhere:/etc/passwd", Tier.ALLOW), + # A read here is what it always was. + ("read_file", "/etc/hosts", Tier.ALLOW), + ], +) +def test_a_remote_read_is_free_in_the_roots_and_asks_elsewhere(tool, path, tier): + assert default_tier(tool, {"path": path}) is tier + + +def test_with_the_lane_off_a_read_is_tiered_as_it_always_was(monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + + assert default_tier("read_file", {"path": "box:.ssh/id_rsa"}) is Tier.ALLOW + assert session_keys("read_file", {"path": "box:.ssh/id_rsa"}) != session_keys("read_file", {"path": "box:.ssh/x"}) + + +def test_an_absolute_path_reads_freely_once_the_machine_s_home_is_known(): + assert default_tier("read_file", {"path": "box:/root/raven-work/a"}) is Tier.ASK + + remote_path.remember_home("box", "/root") + + assert default_tier("read_file", {"path": "box:/root/raven-work/a"}) is Tier.ALLOW + assert default_tier("read_file", {"path": "box:/root/.ssh/id_rsa"}) is Tier.ASK + + +def test_a_grant_covers_one_machine_and_directory_however_the_path_is_written(): + remote_path.remember_home("box", "/root") + + one = session_keys("read_file", {"path": "box:/root/.ssh/id_rsa"}) + same = session_keys("read_file", {"path": "scp://root@203.0.113.7:58717//root/.ssh/known_hosts"}) + relative = session_keys("read_file", {"path": "box:.ssh/config"}) + other_dir = session_keys("read_file", {"path": "box:/root/.config/x"}) + listing = session_keys("list_dir", {"path": "box:/root/.ssh"}) + other_machine = session_keys("read_file", {"path": "nas:/root/.ssh/id_rsa"}) + + assert one == same == relative == listing + assert len({one, other_dir, other_machine}) == 3 + + +def test_a_local_read_keys_the_exact_call_as_before(): + assert session_keys("read_file", {"path": "/etc/hosts"}) != session_keys("read_file", {"path": "/etc/passwd"}) + + +@pytest.mark.asyncio +async def test_the_gate_asks_once_and_the_grant_holds_for_the_directory(): + gate = gate_for(PermissionsConfig(mode="ask")) + bind(None) + outside = {"path": "box:.ssh/id_rsa"} + + first = await gate.check("read_file", outside) + assert isinstance(first, NeedsApproval) + remember_allowed("conv-1", first.session_keys) + + again = await gate.check("read_file", {"path": "box:.ssh/known_hosts"}) + assert isinstance(again, Allow) and again.source is DecisionSource.SESSION + inside = await gate.check("read_file", {"path": "box:raven-work/a"}) + assert isinstance(inside, Allow) and inside.source is DecisionSource.DEFAULT + + +# --- writes ask on their own account -------------------------------------------------------- + + +@pytest.mark.parametrize( + ("tool", "path", "tier"), + [ + ("write_file", "box:raven-work/t1/out.md", Tier.ALLOW), + ("edit_file", "box:/srv/agents/t1/a.py", Tier.ALLOW), + ("edit_file", "nas:/data/shared/runs/notes.md", Tier.ALLOW), + ("write_file", "box:.bashrc", Tier.ASK), + ("write_file", "nas:/etc/hosts", Tier.ASK), + ("edit_file", "box:raven-work/../.ssh/authorized_keys", Tier.ASK), + # A write here asks as it always did. + ("write_file", "/tmp/notes.md", Tier.ASK), + ], +) +def test_a_remote_write_is_free_in_the_roots_and_asks_elsewhere(tool, path, tier): + assert default_tier(tool, {"path": path}) is tier + + +def test_a_write_grant_is_kept_apart_from_a_read_grant_and_covers_its_directory(): + remote_path.remember_home("box", "/root") + + read = session_keys("read_file", {"path": "box:/root/.ssh/id_rsa"}) + write = session_keys("write_file", {"path": "box:/root/.ssh/id_rsa"}) + same_dir = session_keys("edit_file", {"path": "scp://root@203.0.113.7:58717//root/.ssh/config"}) + other_dir = session_keys("write_file", {"path": "box:/root/.config/x"}) + + assert write == same_dir + assert len({read, write, other_dir}) == 3 + assert session_keys("write_file", {"path": "/tmp/a"}) != session_keys("write_file", {"path": "/tmp/b"}) + + +@pytest.mark.asyncio +async def test_a_read_allowed_for_the_session_does_not_allow_a_write_there(): + # Its own directory: the session store is one per process, and the read + # test above has already allowed `.ssh` for this conversation. + gate = gate_for(PermissionsConfig(mode="ask")) + bind(None) + + read = await gate.check("read_file", {"path": "box:.config/raven/state.json"}) + assert isinstance(read, NeedsApproval) + remember_allowed("conv-1", read.session_keys) + write = await gate.check("write_file", {"path": "box:.config/raven/state.json", "content": "x"}) + assert isinstance(write, NeedsApproval), "a look is not a change" + remember_allowed("conv-1", write.session_keys) + + again = await gate.check("edit_file", {"path": "box:.config/raven/other.json", "old_text": "a", "new_text": "b"}) + assert isinstance(again, Allow) and again.source is DecisionSource.SESSION + inside = await gate.check("write_file", {"path": "box:raven-work/t1/out.md", "content": "x"}) + assert isinstance(inside, Allow) and inside.source is DecisionSource.DEFAULT diff --git a/tests/test_remote_file_tools.py b/tests/test_remote_file_tools.py new file mode 100644 index 000000000..7c7fb3a43 --- /dev/null +++ b/tests/test_remote_file_tools.py @@ -0,0 +1,432 @@ +"""read_file, list_dir, find and grep on a registered machine, answered by its Raven node. + +A remote path never touches this computer's disk. The tool places it on its +machine against the login home the node reported, fences the call to the root +it is under (else to the directory the permission check asked about), sends it +to the machine's node, and shows every path in the answer the way scp writes +it. The unit tests stand in for the node pool; the last ones run the real node +behind the stand-in ssh of tests/test_node_client.py. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from raven.agent.tools.file_search import FindTool, GrepTool +from raven.agent.tools.filesystem import ListDirTool, ReadFileTool +from raven.contracts.tool import ToolResult +from raven.node.client import NodeError, NodePool +from raven.ops import remote_path +from tests.test_node_client import HOST, PORT, Machine, row + +HOME = "/home/worker" +SHOWN = f"scp://worker@{HOST}:{PORT}/" + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch): + monkeypatch.setattr(remote_path, "_HOMES", {}) + monkeypatch.setattr(remote_path, "_NODES", None) + monkeypatch.setattr( + "raven.config.loader.load_config", lambda *a, **k: SimpleNamespace(subagents=SimpleNamespace(agents=[])) + ) + + +@pytest.fixture +def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + path = tmp_path / "connections.json" + path.write_text(json.dumps({"connections": [row()]}), encoding="utf-8") + monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + return path + + +class _Pool: + """A node pool that answers each call with ``answer`` and records it.""" + + def __init__(self, answer: dict[str, Any] | None = None, *, fails: str = "") -> None: + self.answer = answer if answer is not None else {"text": "ok"} + self.fails = fails + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + + async def home(self, machine_id: str) -> str: + if self.fails: + raise NodeError(self.fails) + return HOME + + async def call(self, machine_id: str, method: str, params: dict[str, Any]) -> dict[str, Any]: + self.calls.append((machine_id, method, params)) + return self.answer + + +@pytest.fixture +def pool(monkeypatch: pytest.MonkeyPatch) -> _Pool: + nodes = _Pool() + remote_path.use_nodes(lambda: nodes) + return nodes + + +def _tools(tmp_path: Path) -> dict[str, Any]: + fenced = {"workspace": tmp_path, "allowed_dirs": (tmp_path,), "follow_binding": False} + return { + "read_file": ReadFileTool(**fenced), + "list_dir": ListDirTool(**fenced), + "find": FindTool(**fenced), + "grep": GrepTool(**fenced), + } + + +# --- what reaches the node ------------------------------------------------------------ + + +async def test_a_read_goes_to_the_machine_s_node_placed_and_fenced(registry, pool, tmp_path): + pool.answer = {"text": "2| second\n\n(Showing lines 2-3 of 9. Use offset=4 to continue.)"} + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/a.md", offset=2, limit=3) + + assert got == pool.answer["text"] + assert pool.calls == [ + ( + "box", + "fs/read", + {"offset": 2, "limit": 3, "path": f"{HOME}/raven-work/t1/a.md", "roots": [f"{HOME}/raven-work"]}, + ) + ] + + +async def test_list_find_and_grep_send_their_own_arguments(registry, pool, tmp_path): + tools = _tools(tmp_path) + + await tools["list_dir"].execute("box:raven-work", recursive=True, max_entries=5) + await tools["find"].execute("*.py", path="box:/srv/code", limit=9) + await tools["grep"].execute( + "needle", path="box:", glob="*.md", output_mode="files", case_insensitive=True, context=2, limit=4 + ) + + assert pool.calls == [ + ( + "box", + "fs/list", + {"recursive": True, "max_entries": 5, "path": f"{HOME}/raven-work", "roots": [f"{HOME}/raven-work"]}, + ), + ("box", "fs/find", {"pattern": "*.py", "limit": 9, "path": "/srv/code", "roots": ["/srv/code"]}), + ( + "box", + "fs/grep", + { + "pattern": "needle", + "glob": "*.md", + "output_mode": "files", + "case_insensitive": True, + "context": 2, + "limit": 4, + "path": HOME, + "roots": [HOME], + }, + ), + ] + + +async def test_every_path_in_the_answer_is_shown_the_way_scp_writes_it(registry, pool, tmp_path): + pool.answer = { + "text": ( + f"Error: Path {HOME}/raven-work/t1/link is outside allowed directories {HOME}/raven-work; " + f"also {HOME}/raven-work/t1/link" + ) + } + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/link") + + assert got == ( + f"Error: Path {SHOWN}{HOME}/raven-work/t1/link is outside allowed directories {SHOWN}{HOME}/raven-work; " + f"also {SHOWN}{HOME}/raven-work/t1/link" + ) + + +async def test_an_image_comes_back_as_a_result_with_its_blocks(registry, pool, tmp_path): + pool.answer = { + "text": f"Image {HOME}/raven-work/p.png (2x2)", + "display": f"{HOME}/raven-work/p.png", + "ok": True, + "blocks": [ + {"type": "text", "text": f"Image {HOME}/raven-work/p.png (2x2)"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ], + } + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/p.png") + + assert isinstance(got, ToolResult) + assert got.model_text == f"Image {SHOWN}{HOME}/raven-work/p.png (2x2)" + assert got.display_text == f"{SHOWN}{HOME}/raven-work/p.png" + assert got.blocks == [ + {"type": "text", "text": f"Image {SHOWN}{HOME}/raven-work/p.png (2x2)"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ] + assert got.ok + + +# --- what never reaches it ---------------------------------------------------------------- + + +async def test_a_path_that_names_no_machine_is_refused_and_nothing_is_sent(registry, pool, tmp_path): + got = await _tools(tmp_path)["read_file"].execute("nobox:/etc/hostname") + + assert got.startswith("Error: 'nobox:/etc/hostname' is written as a path on another machine") + assert "./name" in got + assert pool.calls == [] + + +async def test_a_process_without_nodes_says_so(registry, tmp_path): + got = await _tools(tmp_path)["list_dir"].execute("box:raven-work") + + assert ( + got == "Error: box:raven-work is on another machine, and this process does not read registered machines' files" + ) + + +async def test_a_node_that_cannot_answer_is_one_sentence(registry, monkeypatch, tmp_path): + failing = _Pool(fails="Raven node 0.1-x is not installed on machine 'Lab box' (box); install it with `x`") + remote_path.use_nodes(lambda: failing) + + got = await _tools(tmp_path)["grep"].execute("x", path="box:raven-work") + + assert got == f"Error: {failing.fails}" + assert HOST not in got + + +async def test_with_the_lane_off_a_remote_path_is_read_as_it_always_was(registry, pool, tmp_path, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES") + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/a.md") + + assert got.startswith("Error: File not found: box:raven-work/t1/a.md") + assert pool.calls == [] + + +async def test_a_local_path_is_read_here_as_before(registry, pool, tmp_path): + (tmp_path / "notes.md").write_text("here\n", encoding="utf-8") + + got = await _tools(tmp_path)["read_file"].execute(str(tmp_path / "notes.md")) + + assert "here" in got + assert pool.calls == [] + + +# --- the real node, behind a stand-in ssh ---------------------------------------------------- + + +@pytest.fixture +async def real(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + if not Path("/bin/sh").exists(): + pytest.skip("the stand-in ssh hands the launch line to a POSIX shell") + machine = Machine(tmp_path, monkeypatch) + machine.node() + work = machine.home / "raven-work" / "t1" + work.mkdir(parents=True) + (work / "notes.txt").write_text("".join(f"note {i}\n" for i in range(1, 21)), encoding="utf-8") + (machine.home / "secret.txt").write_text("top secret\n", encoding="utf-8") + (work / "link.txt").symlink_to(machine.home / "secret.txt") + nodes = NodePool() + remote_path.use_nodes(lambda: nodes) + yield machine + await nodes.close_all() + + +@pytest.mark.slow +async def test_a_remote_read_through_the_real_node_matches_a_local_one(real, tmp_path): + home = str(real.home) + local = ReadFileTool(workspace=real.home, allowed_dirs=(real.home,), follow_binding=False) + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/notes.txt", offset=5, limit=3) + + assert got == await local.execute(f"{home}/raven-work/t1/notes.txt", offset=5, limit=3) + assert remote_path.known_home("box") == home + listed = await _tools(tmp_path)["list_dir"].execute("box:raven-work/t1") + assert "notes.txt" in listed + hits = await _tools(tmp_path)["grep"].execute("note 7", path="box:raven-work") + assert "notes.txt" in hits + + +@pytest.mark.slow +async def test_a_link_out_of_the_root_is_not_followed_and_the_refusal_shows_scp_paths(real, tmp_path): + home = str(real.home) + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/link.txt") + + assert "top secret" not in got + assert f"{SHOWN}{home}/raven-work/t1/link.txt" in got + assert f" {home}/" not in got, "every path is shown the way scp writes it" + + +# --- assembly --------------------------------------------------------------------------- + + +def test_assembly_hands_the_file_tools_this_process_s_node_pool(tmp_path, monkeypatch): + from raven.config.raven import MemoryConfig, PluginsConfig, RavenConfig + from raven.config.schema import Config + from raven.contracts.llm_provider import LLMResponse + from raven.core import runtime + from raven.node.client import get_node_pool + from raven.providers.base import LLMProvider + + monkeypatch.setattr(runtime.token_wise_stack, "install_from_config", lambda *a, **k: None) + monkeypatch.setattr(runtime.token_wise_stack, "caching_probe", lambda *a, **k: False) + + class _Provider(LLMProvider): + def __init__(self) -> None: + super().__init__(api_key="test") + + async def chat(self, messages, **kwargs): + return LLMResponse(content="ok", finish_reason="stop") + + def get_default_model(self) -> str: + return "fake/default" + + config = Config() + config.agents.defaults.workspace = str(tmp_path / "ws") + ec_config = RavenConfig(memory=MemoryConfig(backend=None), plugins=PluginsConfig(), base=config) + assert remote_path.nodes() is None + + built = runtime.build_runtime(config, ec_config, provider=_Provider()) + try: + assert remote_path.nodes() is get_node_pool() + finally: + built.discard() + + +# --- writes: the same way there, with what was read as the expectation -------------------- + + +def _writers(tmp_path: Path) -> dict[str, Any]: + from raven.agent.tools.filesystem import EditFileTool, WriteFileTool + + fenced = {"workspace": tmp_path, "allowed_dirs": (tmp_path,), "follow_binding": False} + return {"write_file": WriteFileTool(**fenced), "edit_file": EditFileTool(**fenced)} + + +async def test_a_write_and_an_edit_go_to_the_node_and_come_back_as_the_tool_s_result(registry, pool, tmp_path): + place = f"{HOME}/raven-work/t1/a.md" + pool.answer = { + "text": f"Successfully wrote 4 bytes to {place}", + "display": None, + "ok": True, + "blocks": [], + "diff": f"--- {place}\n+++ {place}\n@@ -0,0 +1 @@\n+one", + "file_change": {"path": place, "after": "one\n", "before": None}, + } + tools = _writers(tmp_path) + + written = await tools["write_file"].execute("box:raven-work/t1/a.md", "one\n") + await tools["edit_file"].execute("box:raven-work/t1/a.md", "one", "two", replace_all=True) + + assert isinstance(written, ToolResult) + assert written.model_text == f"Successfully wrote 4 bytes to {SHOWN}{place}" + assert written.diff == f"--- {SHOWN}{place}\n+++ {SHOWN}{place}\n@@ -0,0 +1 @@\n+one" + assert written.file_change is not None and written.file_change.path == f"{SHOWN}{place}" + assert written.file_change.after == "one\n" and written.file_change.before is None + assert pool.calls == [ + ("box", "fs/write", {"content": "one\n", "mode": "overwrite", "path": place, "roots": [f"{HOME}/raven-work"]}), + ( + "box", + "fs/edit", + {"old_text": "one", "new_text": "two", "replace_all": True, "path": place, "roots": [f"{HOME}/raven-work"]}, + ), + ] + + +async def test_a_read_s_digest_is_what_the_next_overwrite_expects_and_a_write_forgets_it(registry, pool, tmp_path): + place = f"{HOME}/raven-work/t1/a.md" + pool.answer = {"text": "1| one", "sha256": "ab" * 32} + await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/a.md") + pool.answer = {"text": "ok"} + tools = _writers(tmp_path) + + await tools["write_file"].execute("box:raven-work/t1/a.md", "two\n") + await tools["write_file"].execute("box:raven-work/t1/a.md", "three\n") + await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/a.md") + await tools["write_file"].execute("box:raven-work/t1/a.md", "four\n", mode="append") + await tools["write_file"].execute("box:raven-work/t1/a.md", "five\n") + + sent = [params for _, method, params in pool.calls if method == "fs/write"] + assert sent[0]["expect"] == "ab" * 32, "the whole file is replaced: say what it held when it was read" + assert "expect" not in sent[1], "after a write of its own the file is as this process left it" + assert "expect" not in sent[2], "an append replaces nothing" + assert "expect" not in sent[3], "and leaves the file other than it was read, so the next overwrite expects nothing" + + +async def test_a_file_changed_since_it_was_read_is_left_alone_and_the_sentence_says_to_read_again(registry, tmp_path): + class _Changed(_Pool): + async def call(self, machine_id: str, method: str, params: dict[str, Any]) -> dict[str, Any]: + self.calls.append((machine_id, method, params)) + raise NodeError("the Raven node on machine 'Lab box' (box) refused fs/write: changed", reason="changed") + + changed = _Changed() + remote_path.use_nodes(lambda: changed) + remote_path.remember_read("box", f"{HOME}/raven-work/t1/a.md", "ab" * 32) + + got = await _writers(tmp_path)["write_file"].execute("box:raven-work/t1/a.md", "two\n") + + assert got == ( + f"Error: {SHOWN}{HOME}/raven-work/t1/a.md on machine 'Lab box' (box) changed after it was read; " + "nothing was written. Read it again before writing." + ) + assert HOST not in got.split("on machine")[1] + assert remote_path.last_read("box", f"{HOME}/raven-work/t1/a.md") is None, "the next write reads first" + + +def test_the_approval_card_for_a_write_on_a_machine_names_the_machine_and_the_path_as_written(registry, tmp_path): + tools = _writers(tmp_path) + + write = tools["write_file"].approval_evidence({"path": "box:raven-work/t1/a.md", "content": "one\n"}) + edit = tools["edit_file"].approval_evidence( + {"path": "box:raven-work/t1/a.md", "old_text": "one", "new_text": "two"} + ) + local = tools["write_file"].approval_evidence({"path": str(tmp_path / "here.md"), "content": "x"}) + + assert write == {"path": "box:raven-work/t1/a.md", "machine": "box"} + assert edit["machine"] == "box" and edit["path"] == "box:raven-work/t1/a.md" and "+two" in edit["diff"] + assert "machine" not in local and local["created"] is True + + +@pytest.mark.slow +async def test_a_remote_write_and_edit_through_the_real_node_match_local_ones(real, tmp_path): + from raven.agent.tools.filesystem import EditFileTool, WriteFileTool + + home = str(real.home) + there = real.home / "raven-work" / "t1" / "new.md" + tools = _writers(tmp_path) + + written = await tools["write_file"].execute("box:raven-work/t1/new.md", "one\ntwo\n") + edited = await tools["edit_file"].execute("box:raven-work/t1/new.md", "two", "three") + + local_dir = tmp_path / "local" + local_dir.mkdir() + local_write = WriteFileTool(workspace=local_dir, allowed_dirs=(local_dir,), follow_binding=False) + local_edit = EditFileTool(workspace=local_dir, allowed_dirs=(local_dir,), follow_binding=False) + want_written = await local_write.execute(str(local_dir / "new.md"), "one\ntwo\n") + want_edited = await local_edit.execute(str(local_dir / "new.md"), "two", "three") + swap = (str(local_dir / "new.md"), f"{SHOWN}{home}/raven-work/t1/new.md") + assert written.model_text == want_written.model_text.replace(*swap) + assert written.diff == want_written.diff.replace(*swap) + assert edited.model_text == want_edited.model_text.replace(*swap) + assert edited.file_change.after == want_edited.file_change.after == "one\nthree\n" + assert there.read_text(encoding="utf-8") == "one\nthree\n" + + # Read, then changed behind this process's back: the overwrite is refused and says so. + await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/new.md") + there.write_text("someone else\n", encoding="utf-8") + refused = await tools["write_file"].execute("box:raven-work/t1/new.md", "four\n") + assert refused.startswith(f"Error: {SHOWN}{home}/raven-work/t1/new.md on machine 'Lab box' (box) changed after") + assert there.read_text(encoding="utf-8") == "someone else\n" + # Read again, and the write goes through. + await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/new.md") + assert (await tools["write_file"].execute("box:raven-work/t1/new.md", "four\n")).model_text.startswith( + "Successfully" + ) + assert there.read_text(encoding="utf-8") == "four\n" diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index 49a86276a..61f17cb5a 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -808,3 +808,73 @@ def served() -> dict: after = served() assert "machine" in after["parameters"]["properties"], "the lane the reply names is in the served schema" assert "pass 'machine'" in after["description"] + + +# ---- a copy to a registered machine goes through the file tools, not around them ---- + + +@pytest.mark.parametrize( + "command", + [ + "scp -P 58717 a.txt root@203.0.113.7:/tmp/", + "scp -rP 58717 root@203.0.113.7:/tmp/a ./here", + "scp -o Port=58717 a.txt 203.0.113.7:a", + "scp -P58717 a.txt root@203.0.113.7:", + "sftp -P 58717 root@203.0.113.7", + "rsync -av -e 'ssh -p 58717' ./dir root@203.0.113.7:/data/", + "rsync -avP --rsh='ssh -o Port=58717' root@203.0.113.7:/data/ ./dir", + "rsync -av --port 58717 ./dir rsync://203.0.113.7/module", + "scp scp://root@203.0.113.7:58717//tmp/a .", + "true && \\scp -P 58717 a root@203.0.113.7:b", + ], +) +async def test_a_copy_to_a_registered_machine_is_refused_and_points_at_the_file_tools(registry, tmp_path, command): + tool = ExecTool(working_dir=str(tmp_path)) + marker = tmp_path / "ran" + + out = await tool.execute(command=f"{command}; touch {marker}") + + assert out.startswith("Error: this command copies files to or from GPU box") + assert "conn_gpu" in out and "read_file" in out and "deliver_files" in out and "Raven node" in out + assert "203.0.113.7" not in out.replace("user@host", "") + assert not marker.exists() + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + ("scp a.txt root@203.0.113.7:/tmp/", [("203.0.113.7", 22)]), + ("scp -P 58717 a.txt root@203.0.113.7:/tmp/ b.txt h2:x", [("203.0.113.7", 58717), ("h2", 58717)]), + ("scp ./a:b /tmp/", []), + ("scp /tmp/a:b /tmp/c", []), + ("rsync -av ./dir /tmp/dir", []), + ("rsync -avP root@h:/data/ ./dir", [("h", 22)]), + ("rsync -av --port=873 ./dir h::module", [("h", 873)]), + ("rsync -e 'ssh -p 2222' ./x h:y", [("h", 2222)]), + ("sftp -P 2222 -b batch.txt h", [("h", 2222)]), + ("sftp root@h:uploads", [("h", 22)]), + # Read wherever the word stands, as the ssh guard reads `ssh`: a false + # refusal of an echo costs a retry, a copy let through costs the address. + ("echo scp root@203.0.113.7:/tmp/a", [("203.0.113.7", 22)]), + ("scp -i key -P 58717 a root@203.0.113.7:b", [("203.0.113.7", 58717)]), + ("scp 'a b:c' root@h:d", [("a b", 22), ("h", 22)]), + ("scp a root@h:b 2>&1", [("h", 22)]), + ("scp -- -weird:name root@h:b", [("-weird", 22), ("h", 22)]), + ("scp 'unbalanced", []), + ], +) +def test_copy_destinations_are_read_the_way_the_programs_read_them(command, expected): + assert machine_exec._copy_destinations(command) == expected + + +async def test_a_copy_to_the_same_address_on_another_port_is_another_machine(registry): + assert machine_exec.raw_copy_target("scp a.txt root@203.0.113.7:/tmp/") is None + assert machine_exec.raw_copy_target("scp -P 58717 a.txt root@203.0.113.7:/tmp/") == dict(ROW) + + +async def test_typed_ssh_to_a_registered_machine_names_the_file_tools_too(registry, tmp_path): + tool = ExecTool(working_dir=str(tmp_path)) + + out = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") + + assert "write_file and edit_file on the path the way scp writes it" in out From f143bbd62c77c5dae77b6be28ddb5aeb0e261721 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:42:30 +0800 Subject: [PATCH 04/44] fix(node): put the daemon on a machine through uv alone, fetching uv when the machine has none The install ran `python -m venv` and `python -m pip install` on the machine, against the repository's rule that uv is the only Python package manager. Everything Python now goes through uv: `uv venv` of the machine's own Python 3.12 or of one uv keeps inside the node directory, `uv pip install` for the pinned packages and the optional search package. A machine without uv gets one under `/.uv` first, from uv's own installer fetched with the machine's curl or wget and told to touch nothing else; the probe looks for curl and wget, and for a uv an earlier install put there; with no uv and no way to fetch one the install stops and says so. The plan counts the room uv takes and says when it is fetched. Co-authored-by: Claude (claude-fable-5-1) --- raven/node/install.py | 113 ++++++++++++++++++++++++------------ tests/test_node_client.py | 3 +- tests/test_node_install.py | 116 ++++++++++++++++++++++++++++--------- 3 files changed, 167 insertions(+), 65 deletions(-) diff --git a/raven/node/install.py b/raven/node/install.py index cbedf2c6d..b64ad266c 100644 --- a/raven/node/install.py +++ b/raven/node/install.py @@ -7,9 +7,13 @@ Nothing else on the machine changes: no system package, no shell rc file, no environment it already had. -The Python is the machine's own when it has 3.12 or later with ``venv``; -otherwise the machine's ``uv`` makes one inside ``node_dir``. With neither, the -install stops and says so rather than fetching an installer and running it. +Everything Python goes through ``uv``, as it does in this repository: the +virtualenv is ``uv venv`` of the machine's own Python 3.12 when it has one, +else of a Python uv keeps inside ``node_dir``; the packages are ``uv pip +install``. A machine without uv gets one under ``node_dir/.uv`` first, from +uv's own installer, fetched with the machine's curl or wget and told to +touch nothing else; with neither uv nor a way to fetch it, the install stops +and says so. Each step is one ssh call through :func:`raven.acp_client.remote.run`; a step that fails removes the half-made node directory, so the next attempt starts @@ -28,10 +32,14 @@ from raven.node.client import node_dir as registered_node_dir #: Roughly what a node takes, for the free-space check: Raven's code with -#: pillow, loguru and ripgrep-bin (measured 2026-10-10), and a Python 3.12 when -#: the machine has none of its own. +#: pillow, loguru and ripgrep-bin (measured 2026-10-10), a Python 3.12 when +#: the machine has none of its own, and uv when the machine has none. _NODE_KB = 60 * 1024 _PYTHON_KB = 80 * 1024 +_UV_KB = 25 * 1024 + +#: uv's own installer, which puts one binary where ``UV_INSTALL_DIR`` says. +UV_INSTALLER = "https://astral.sh/uv/install.sh" _PROBE_TIMEOUT_S = 60.0 _STEP_TIMEOUT_S = 900.0 @@ -52,17 +60,26 @@ class Plan: python: str uv: str free_kb: int | None + fetch: str = "" + """The machine's curl or wget, with which uv can be fetched when the machine has none.""" + + @property + def uv_from(self) -> str: + """``"machine"`` (its own uv), ``"fetched"`` (one put under node_dir first), or ``""`` (no way to one).""" + if self.uv: + return "machine" + return "fetched" if self.fetch else "" @property def how(self) -> str: - """``"python"`` (the machine's own 3.12), ``"uv"`` (one made by uv), or ``""`` (neither).""" - if self.python: - return "python" - return "uv" if self.uv else "" + """``"python"`` (uv over the machine's own 3.12), ``"uv"`` (over one uv keeps), or ``""`` (no uv to be had).""" + if not self.uv_from: + return "" + return "python" if self.python else "uv" @property def needed_kb(self) -> int: - return _NODE_KB + (_PYTHON_KB if self.how == "uv" else 0) + return _NODE_KB + (_PYTHON_KB if self.how == "uv" else 0) + (_UV_KB if self.uv_from == "fetched" else 0) def describe(self) -> str: """The install as a person is asked to approve it.""" @@ -70,13 +87,18 @@ def describe(self) -> str: python = ( f"a virtualenv of the machine's own {self.python}" if self.how == "python" - else "a Python 3.12 made by the machine's uv, kept beside it" + else "a Python 3.12 made by uv, kept beside it" + ) + uv = ( + f" uv itself first, under {self.node_dir}/.uv (the machine has none), fetched with its own {self.fetch};" + if self.uv_from == "fetched" + else "" ) space = f"; {self.free_kb // (1024 * 1024)} GB free there" if self.free_kb is not None else "" return ( - f"Install the Raven node {self.install_name} on machine {self.target.label} into {where}: {python}, " + f"Install the Raven node {self.install_name} on machine {self.target.label} into {where}:{uv} {python}, " f"{' '.join(bundle.pinned((*protocol.REQUIREMENTS, protocol.SEARCH_REQUIREMENT)))} from the " - "machine's package index, and this Raven's code. " + "machine's package index, and this Raven's code, all through uv. " f"About {self.needed_kb // 1024} MB{space}. Nothing else on the machine changes." ) @@ -110,15 +132,21 @@ def _vars(node_dir: str, install_name: str) -> str: def probe_script(node_dir: str, install_name: str, digest: str) -> str: - """One look at the machine: is this node there, which Python and uv does it have, how much room.""" + """One look at the machine: is this node there, which Python, uv and fetcher it has, how much room. + + uv is looked for on the PATH, in the two places its installer puts it + that a non-login PATH misses, and under ``node_dir/.uv``, where an + earlier install of this Raven may have put it. + """ return _vars(node_dir, install_name) + ( f'if [ -x "$node/bin/python" ] && "$node/bin/python" -I -m raven.node --version 2>/dev/null ' f"| grep -q {shlex.quote(f'"digest": "{digest}"')}; then echo PRESENT=1; fi; " 'for p in python3.13 python3.12 python3; do q=$(command -v "$p" 2>/dev/null) || continue; ' - "if \"$q\" -c 'import sys, venv, ensurepip; sys.exit(0 if sys.version_info >= (3, 12) else 1)' " + "if \"$q\" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)' " '2>/dev/null; then echo "PYTHON=$q"; break; fi; done; ' - 'for u in "$(command -v uv 2>/dev/null)" "$HOME/.local/bin/uv" "$HOME/.cargo/bin/uv"; do ' + 'for u in "$(command -v uv 2>/dev/null)" "$HOME/.local/bin/uv" "$HOME/.cargo/bin/uv" "$dir/.uv/uv"; do ' 'if [ -n "$u" ] && [ -x "$u" ]; then echo "UV=$u"; break; fi; done; ' + 'for f in curl wget; do c=$(command -v "$f" 2>/dev/null) || continue; echo "FETCH=$c"; break; done; ' 'd="$dir"; while [ ! -d "$d" ]; do d=$(dirname "$d"); done; ' 'echo "FREE_KB=$(df -Pk "$d" 2>/dev/null | awk \'NR==2 {print $4}\')"' ) @@ -142,7 +170,7 @@ def plan(machine_id: str, *, node_dir: str | None = None, run: Runner = _run) -> seen: dict[str, str] = {} for line in out.splitlines(): key, sep, value = line.strip().partition("=") - if sep and key in {"PRESENT", "PYTHON", "UV", "FREE_KB"}: + if sep and key in {"PRESENT", "PYTHON", "UV", "FETCH", "FREE_KB"}: seen[key] = value.strip() free = seen.get("FREE_KB", "") return Plan( @@ -153,6 +181,7 @@ def plan(machine_id: str, *, node_dir: str | None = None, run: Runner = _run) -> python=seen.get("PYTHON", ""), uv=seen.get("UV", ""), free_kb=int(free) if free.isdigit() else None, + fetch=seen.get("FETCH", ""), ) @@ -160,8 +189,8 @@ def refusal(p: Plan) -> str | None: """Why ``p`` cannot go ahead, or ``None`` when it can.""" if p.how == "": return ( - f"machine {p.target.label} has neither a Python 3.12 (with venv) nor uv; " - "install one of them there, then run this again" + f"machine {p.target.label} has no uv, and no curl or wget to fetch one with; " + f"install uv there ({UV_INSTALLER}), then run this again" ) if p.free_kb is not None and p.free_kb < p.needed_kb: return ( @@ -173,23 +202,33 @@ def refusal(p: Plan) -> str | None: def steps(p: Plan) -> list[tuple[str, str]]: """``(name, command)`` for each ssh step after the code is sent, in order.""" - head = _vars(p.node_dir, p.install_name) - if p.how == "python": - venv = f'mkdir -p "$dir" && {shlex.quote(p.python)} -m venv "$node"' - else: - # Python, cache and all inside node_dir: the machine's home may be on a - # disk with no room (measured 2026-10-10: one registered machine's root - # filesystem was full). - venv = ( - 'mkdir -p "$dir" && UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python" ' - f'{shlex.quote(p.uv)} venv --quiet --seed --python 3.12 "$node"' + # uv's cache and the Python it keeps live inside node_dir: the machine's + # home may be on a disk with no room (measured 2026-10-10: one registered + # machine's root filesystem was full). + uv = shlex.quote(p.uv) if p.uv_from == "machine" else '"$dir/.uv/uv"' + head = _vars(p.node_dir, p.install_name) + f"uv={uv}; " + env = 'UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python"' + out: list[tuple[str, str]] = [] + if p.uv_from == "fetched": + # uv's own installer, told where to put the one binary and to leave + # the shell's rc files alone; a wget machine reads it the same way. + fetch = f"wget -qO- {UV_INSTALLER}" if p.fetch.endswith("wget") else f"curl -LsSf {UV_INSTALLER}" + out.append( + ( + "uv", + head + f'mkdir -p "$dir/.uv" && {fetch} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' + 'INSTALLER_NO_MODIFY_PATH=1 sh >/dev/null && test -x "$dir/.uv/uv"', + ) ) - pip = '"$node/bin/python" -m pip install --quiet --no-cache-dir --disable-pip-version-check --no-input' + python = shlex.quote(p.python) if p.how == "python" else "3.12" + venv = f'mkdir -p "$dir" && {env} "$uv" venv --quiet --python {python} "$node"' + pip = f'{env} "$uv" pip install --quiet --python "$node/bin/python"' requirements = " ".join(shlex.quote(r) for r in bundle.pinned(protocol.REQUIREMENTS)) deps = f"{pip} {requirements}" - # Optional, and never fails the install: the machine's own index first (its - # owner chose it, often a nearby mirror); then the public index, for a - # mirror that does not carry it; else the node searches without rg. + # Optional, and never fails the install: the index uv is configured with + # on the machine first (its owner's choice, often a nearby mirror); then + # the public index, for a mirror that does not carry it; else the node + # searches without rg. (rg,) = bundle.pinned((protocol.SEARCH_REQUIREMENT,)) search = ( f"{{ {pip} {shlex.quote(rg)} || {pip} --extra-index-url {protocol.PUBLIC_INDEX} {shlex.quote(rg)}; }} " @@ -199,7 +238,7 @@ def steps(p: Plan) -> list[tuple[str, str]]: '"$node/bin/python" -c \'import site, sys; ' 'open(site.getsitepackages()[0] + "/raven-node.pth", "w").write(sys.argv[1] + "\\n")\' "$node/lib"' ) - return [ + out += [ ("virtualenv", head + venv), ("packages", head + deps), ("search", head + search), @@ -207,6 +246,7 @@ def steps(p: Plan) -> list[tuple[str, str]]: ("path", head + link), ("check", head + '"$node/bin/python" -I -m raven.node --version'), ] + return out def prune_script(node_dir: str, install_name: str) -> str: @@ -253,8 +293,8 @@ def install(p: Plan, *, run: Runner = _run, push: Pusher = _push) -> Installed: if name == "check": _confirm_digest(p, out) except BaseException: - # Only the node's own directory: node_dir may hold other nodes, and a - # Python uv made there is reused by the next attempt. + # Only the node's own directory: node_dir may hold other nodes, and the + # uv and the Python put there are reused by the next attempt. run(p.target, head + 'rm -rf "$node"') raise rc, out = run(p.target, head + 'du -sk "$node" 2>/dev/null | cut -f1') @@ -285,6 +325,7 @@ def _confirm_digest(p: Plan, out: str) -> None: __all__ = [ + "UV_INSTALLER", "InstallError", "Installed", "Plan", diff --git a/tests/test_node_client.py b/tests/test_node_client.py index 3c04c95df..1978c7625 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -57,7 +57,8 @@ sys.exit(255) command = args[args.index("--") + 1] if "--" in args else args[-1] os.chdir(home) -os.execve("/bin/sh", ["/bin/sh", "-c", command], {{"HOME": home, "SHELL": "/bin/sh", "PATH": os.environ["PATH"]}}) +# A daemon these tests start leaves soon after they do; the relay reads the idle time from its environment. +os.execve("/bin/sh", ["/bin/sh", "-c", command], {{"HOME": home, "SHELL": "/bin/sh", "PATH": os.environ["PATH"], "RAVEN_NODE_IDLE_S": "2"}}) """ # A node that misbehaves in one named way, for the handshake's refusals. diff --git a/tests/test_node_install.py b/tests/test_node_install.py index c6aa3237c..ee61ac064 100644 --- a/tests/test_node_install.py +++ b/tests/test_node_install.py @@ -57,8 +57,9 @@ def _plan(**fields: Any) -> Plan: "install_name": bundle.install_name(), "present": False, "python": "/usr/bin/python3.12", - "uv": "", + "uv": "/usr/bin/uv", "free_kb": 10 * 1024 * 1024, + "fetch": "/usr/bin/curl", } return Plan(**{**base, **fields}) @@ -221,10 +222,22 @@ def test_an_install_through_uv_needs_room_for_a_python_too(): p = _plan(python="", uv="/usr/bin/uv", free_kb=None) assert p.how == "uv" and p.needed_kb == 140 * 1024 - assert "a Python 3.12 made by the machine's uv, kept beside it" in p.describe() + assert "a Python 3.12 made by uv, kept beside it" in p.describe() assert p.describe().endswith("About 140 MB. Nothing else on the machine changes.") +def test_a_machine_without_uv_gets_one_first_and_the_room_for_it_is_counted(): + p = _plan(uv="", free_kb=None) + + assert p.uv_from == "fetched" and p.how == "python" and p.needed_kb == 85 * 1024 + said = p.describe() + assert ( + "uv itself first, under ~/.raven-node/.uv (the machine has none), fetched with its own /usr/bin/curl;" in said + ) + assert "a virtualenv of the machine's own /usr/bin/python3.12" in said and "all through uv" in said + assert _plan(uv="", fetch="").how == "" and _plan(uv="", fetch="").uv_from == "" + + def test_the_machine_s_own_python_wins_over_uv(): assert _plan(python="/usr/bin/python3.12", uv="/usr/bin/uv").how == "python" @@ -233,9 +246,14 @@ def test_the_machine_s_own_python_wins_over_uv(): ("fields", "said"), [ ( - {"python": "", "uv": ""}, - "machine 'Lab box' (box) has neither a Python 3.12 (with venv) nor uv; " - "install one of them there, then run this again", + {"python": "", "uv": "", "fetch": ""}, + "machine 'Lab box' (box) has no uv, and no curl or wget to fetch one with; " + f"install uv there ({install.UV_INSTALLER}), then run this again", + ), + ( + {"uv": "", "fetch": ""}, + "machine 'Lab box' (box) has no uv, and no curl or wget to fetch one with; " + f"install uv there ({install.UV_INSTALLER}), then run this again", ), ( {"free_kb": 10 * 1024}, @@ -243,10 +261,15 @@ def test_the_machine_s_own_python_wins_over_uv(): "free some space or choose another directory with --dir", ), ( - {"python": "", "uv": "/usr/bin/uv", "free_kb": 100 * 1024}, + {"python": "", "free_kb": 100 * 1024}, "~/.raven-node on machine 'Lab box' (box) has 100 MB free and a node needs about 140 MB; " "free some space or choose another directory with --dir", ), + ( + {"uv": "", "free_kb": 70 * 1024}, + "~/.raven-node on machine 'Lab box' (box) has 70 MB free and a node needs about 85 MB; " + "free some space or choose another directory with --dir", + ), ({"free_kb": 60 * 1024}, None), ({"free_kb": None}, None), ], @@ -255,19 +278,24 @@ def test_an_install_that_cannot_go_ahead_says_why(fields, said): assert install.refusal(_plan(**fields)) == said -def test_the_steps_build_a_virtualenv_then_fill_it(): - steps = install.steps(_plan(python="/opt/py 3.12/bin/python3")) +_UV_ENV = 'UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python"' + + +def test_the_steps_build_a_virtualenv_with_uv_then_fill_it_with_uv(): + steps = install.steps(_plan(python="/opt/py 3.12/bin/python3", uv="/opt/uv")) assert [name for name, _ in steps] == ["virtualenv", "packages", "search", "code", "path", "check"] - head = "dir=~/.raven-node; id=" + bundle.install_name() + '; node="$dir/$id"; ' + head = "dir=~/.raven-node; id=" + bundle.install_name() + '; node="$dir/$id"; uv=/opt/uv; ' assert all(command.startswith(head) for _, command in steps) commands = dict(steps) - assert commands["virtualenv"] == head + 'mkdir -p "$dir" && \'/opt/py 3.12/bin/python3\' -m venv "$node"' - for pin in bundle.pinned(protocol.REQUIREMENTS): - assert pin in commands["packages"] - assert "--no-cache-dir" in commands["packages"] + assert commands["virtualenv"] == ( + head + f'mkdir -p "$dir" && {_UV_ENV} "$uv" venv --quiet --python \'/opt/py 3.12/bin/python3\' "$node"' + ) + pip = f'{_UV_ENV} "$uv" pip install --quiet --python "$node/bin/python"' + assert commands["packages"] == head + f"{pip} " + " ".join(bundle.pinned(protocol.REQUIREMENTS)) + assert "python -m pip" not in " ".join(commands.values()) and " -m venv " not in " ".join(commands.values()) (rg,) = bundle.pinned((protocol.SEARCH_REQUIREMENT,)) - assert commands["search"].count(rg) == 2 + assert commands["search"].count(rg) == 2 and commands["search"].count(pip) == 2 assert f"--extra-index-url {protocol.PUBLIC_INDEX}" in commands["search"] assert commands["search"].endswith(">/dev/null 2>&1 && echo RG=1 || echo RG=0") assert commands["code"] == head + 'mkdir -p "$node/lib" && tar -xzf - -C "$node/lib"' @@ -280,11 +308,27 @@ def test_uv_makes_its_python_inside_the_node_dir(): commands = dict(install.steps(_plan(python="", uv="/root/.local/bin/uv"))) assert commands["virtualenv"].endswith( - 'mkdir -p "$dir" && UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python" ' - '/root/.local/bin/uv venv --quiet --seed --python 3.12 "$node"' + f'uv=/root/.local/bin/uv; mkdir -p "$dir" && {_UV_ENV} "$uv" venv --quiet --python 3.12 "$node"' ) +@pytest.mark.parametrize( + ("fetch", "fetched_with"), + [("/usr/bin/curl", f"curl -LsSf {install.UV_INSTALLER}"), ("/usr/bin/wget", f"wget -qO- {install.UV_INSTALLER}")], +) +def test_a_machine_without_uv_fetches_it_under_the_node_dir_first(fetch, fetched_with): + steps = install.steps(_plan(uv="", fetch=fetch)) + + assert [name for name, _ in steps] == ["uv", "virtualenv", "packages", "search", "code", "path", "check"] + head = "dir=~/.raven-node; id=" + bundle.install_name() + '; node="$dir/$id"; uv="$dir/.uv/uv"; ' + assert all(command.startswith(head) for _, command in steps) + assert dict(steps)["uv"] == ( + head + f'mkdir -p "$dir/.uv" && {fetched_with} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' + 'INSTALLER_NO_MODIFY_PATH=1 sh >/dev/null && test -x "$dir/.uv/uv"' + ) + assert dict(steps)["virtualenv"].endswith('"$uv" venv --quiet --python /usr/bin/python3.12 "$node"') + + # --- carrying it out -------------------------------------------------------------- @@ -395,8 +439,8 @@ def test_the_last_json_line_is_the_one_checked(small_tarball): def test_an_install_that_cannot_go_ahead_sends_nothing(): machine = Machine() - with pytest.raises(InstallError, match="has neither a Python 3.12"): - install.install(_plan(python="", uv=""), run=machine.run, push=machine.push) + with pytest.raises(InstallError, match="has no uv, and no curl or wget"): + install.install(_plan(python="", uv="", fetch=""), run=machine.run, push=machine.push) assert machine.sent == [] @@ -406,7 +450,7 @@ def test_an_install_interrupted_by_an_error_still_cleans_up(small_tarball): def run(target: remote.Machine, command: str) -> tuple[int, str]: machine.sent.append(command) - if "-m venv" in command: + if " venv " in command: raise remote.RemoteMachineError("machine 'Lab box' (box): no key") return 0, "" @@ -478,9 +522,10 @@ def _lines(out: str) -> dict[str, str]: @pytest.mark.slow def test_the_probe_finds_the_node_a_new_enough_python_uv_and_the_room(shell): home, tools = shell - _script(tools / "python3.13", "exit 1\n") # too old, or without venv + _script(tools / "python3.13", "exit 1\n") # too old _script(tools / "python3.12", "exit 0\n") _script(tools / "uv", "exit 0\n") + _script(tools / "curl", "exit 0\n") digest = bundle.digest() _script( home / ".raven-node" / "0.1-x" / "bin" / "python", @@ -492,9 +537,20 @@ def test_the_probe_finds_the_node_a_new_enough_python_uv_and_the_room(shell): assert seen["PRESENT"] == "1" assert seen["PYTHON"] == str(tools / "python3.12") assert seen["UV"] == str(tools / "uv") + assert seen["FETCH"] == str(tools / "curl") assert seen["FREE_KB"].isdigit() +@pytest.mark.slow +def test_the_probe_finds_the_uv_an_earlier_install_put_under_the_node_dir(shell): + home, tools = shell + _script(home / "nodes" / ".uv" / "uv", "exit 0\n") + + seen = _lines(_sh(install.probe_script(str(home / "nodes"), "0.1-x", "d"), home, str(tools))) + + assert seen["UV"] == str(home / "nodes" / ".uv" / "uv") and "FETCH" not in seen + + @pytest.mark.slow def test_the_probe_does_not_take_a_node_of_other_code_for_this_one(shell): home, tools = shell @@ -521,19 +577,19 @@ def test_the_probe_finds_a_uv_the_non_login_path_misses(shell): @pytest.mark.slow -@pytest.mark.parametrize(("pip_rc", "said", "tries"), [(0, "RG=1", 1), (1, "RG=0", 2)]) -def test_the_search_package_never_fails_the_install(shell, tmp_path, pip_rc, said, tries): +@pytest.mark.parametrize(("uv_rc", "said", "tries"), [(0, "RG=1", 1), (1, "RG=0", 2)]) +def test_the_search_package_never_fails_the_install(shell, tmp_path, uv_rc, said, tries): home, tools = shell - calls = tmp_path / "pip-calls" + calls = tmp_path / "uv-calls" nodes = tmp_path / "nodes" - _script(nodes / "0.1-x" / "bin" / "python", f'echo "$@" >> "{calls}"; exit {pip_rc}\n') + uv = _script(tools / "uv", f'echo "$@" >> "{calls}"; exit {uv_rc}\n') - search = dict(install.steps(_plan(node_dir=str(nodes), install_name="0.1-x")))["search"] + search = dict(install.steps(_plan(node_dir=str(nodes), install_name="0.1-x", uv=str(uv))))["search"] assert _sh(search, home, str(tools)).strip() == said asked = calls.read_text(encoding="utf-8").splitlines() assert len(asked) == tries - assert "--extra-index-url" not in asked[0], "the machine's own index first" + assert asked[0].startswith("pip install --quiet --python ") and "--extra-index-url" not in asked[0] if tries == 2: assert f"--extra-index-url {protocol.PUBLIC_INDEX}" in asked[1] @@ -590,7 +646,10 @@ def push(target: remote.Machine, command: str, data: bytes) -> tuple[int, str]: first = install.plan("box", node_dir=str(nodes), run=run) assert not first.present - p = dataclasses.replace(first, python=sys.executable, uv="") + uv = shutil.which("uv") + if uv is None: + pytest.skip("the install goes through uv, which this computer does not have") + p = dataclasses.replace(first, python=sys.executable, uv=uv) done = install.install(p, run=run, push=push) assert done.install_name == bundle.install_name() and done.rg is False and (done.size_kb or 0) > 1000 @@ -608,4 +667,5 @@ def push(target: remote.Machine, command: str, data: bytes) -> tuple[int, str]: assert install.plan("box", node_dir=str(nodes), run=run).present, "the next look finds it" assert install.prune(p, run=run) == ("0.0.1-old",) - assert sorted(path.name for path in nodes.iterdir()) == [bundle.install_name()] + # uv's cache stays beside the install, for the next one; nothing else does. + assert sorted(path.name for path in nodes.iterdir()) == [".uv-cache", bundle.install_name()] From 22feef5a79ea684fda60585f96bd02ea8157f697 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:17:23 +0800 Subject: [PATCH 05/44] fix(*): a remote run's reply names no tool this change lacks; two adds stay off the network The reply of a run on a machine pointed at read_file, list_dir, find, grep, write_file, edit_file and deliver_files for the files it left there, which on this change still read a path as this computer's: it now says only where the files are. Two tests of an add on a machine reached for the machine over a real ssh when the capability record ran after the stood-in ping, and waited out its connect timeout on CI: the record is stood in for too. The page's fixture gate lists the `machine` field of a sub-agent row as one its library does not send (every row there runs on this computer), and a daemon the stand-in ssh starts in a test leaves soon after the test does. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/acp_agent.py | 3 +-- tests/test_rpc_subagents.py | 6 ++++++ tests/test_subagent_acp_remote.py | 6 ++++-- ui-web/scripts/gates/fixture-shape.test.mjs | 3 +++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/raven/acp_client/acp_agent.py b/raven/acp_client/acp_agent.py index f6fea6d1e..404c48029 100644 --- a/raven/acp_client/acp_agent.py +++ b/raven/acp_client/acp_agent.py @@ -1832,8 +1832,7 @@ async def run( notes.append( f"Ran on machine {launched.target.label}, in " f"{remote.shown(launched.target, session_cwd)}; the files it wrote are there, " - "not on this computer -- read_file, list_dir, find, grep, write_file and edit_file take " - "that path as it is, and deliver_files fetches one by it" + "not on this computer" ) for note in filter(None, notes): notice = f"\n\n[raven] {note}." diff --git a/tests/test_rpc_subagents.py b/tests/test_rpc_subagents.py index 204d9a62c..737a777a2 100644 --- a/tests/test_rpc_subagents.py +++ b/tests/test_rpc_subagents.py @@ -3136,6 +3136,10 @@ async def _launch(**kwargs: object) -> object: ] +async def _no_record(cfg: object) -> None: + """A ``record_capabilities`` stand-in: the real one would open the machine's link, over a real ssh.""" + + @pytest.fixture def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: """A machine registry with one usable row, and the machine lane on.""" @@ -3159,6 +3163,7 @@ async def _capture(cfg: object) -> object: return await _pings_ok(cfg) monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _capture) + monkeypatch.setattr("raven.rpc.methods.subagents.record_capabilities", _no_record) out = await subagents_add({"preset": "claude_code", "machine": " ravenx2 ", "remote_cwd": " ~/work "}) @@ -3174,6 +3179,7 @@ async def test_add_on_a_machine_keeps_the_name_the_caller_chose( config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _pings_ok) + monkeypatch.setattr("raven.rpc.methods.subagents.record_capabilities", _no_record) out = await subagents_add({"preset": "claude_code", "machine": "ravenx2", "name": "lab"}) diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index d50f60378..1286cb10d 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -69,7 +69,8 @@ def fail(said): fail("worker@{host}: Permission denied (publickey).") command = args[args.index("--") + 1] if "--" in args else args[-1] os.chdir(home) -os.execve("/bin/sh", ["/bin/sh", "-c", command], {{"HOME": home, "SHELL": "/bin/sh", "PATH": os.environ["PATH"]}}) +# A daemon these tests start leaves soon after they do; the relay reads the idle time from its environment. +os.execve("/bin/sh", ["/bin/sh", "-c", command], {{"HOME": home, "SHELL": "/bin/sh", "PATH": os.environ["PATH"], "RAVEN_NODE_IDLE_S": "2"}}) """ @@ -187,7 +188,8 @@ async def test_a_dispatch_runs_on_the_machine_in_a_directory_made_there_and_says # Shown the way scp writes it, which the file tools read back as it is. assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in reply assert "not on this computer" in reply - assert "read_file, list_dir, find, grep, write_file and edit_file take that path as it is" in reply + assert "the files it wrote are there, not on this computer" in reply + assert "read_file" not in reply, "no tool is named that this change does not give a remote path to" assert not local.exists(), "the caller's workspace is never touched" for argv in box.calls(): assert str(local) not in " ".join(argv), "this computer's workspace path never reaches the machine" diff --git a/ui-web/scripts/gates/fixture-shape.test.mjs b/ui-web/scripts/gates/fixture-shape.test.mjs index 7f7cfc9ab..bddf5df3a 100644 --- a/ui-web/scripts/gates/fixture-shape.test.mjs +++ b/ui-web/scripts/gates/fixture-shape.test.mjs @@ -171,6 +171,9 @@ const UNSENT = new Set([ 'browser.watch.can_back', 'browser.watch.can_forward', 'browser.watch.error', 'browser.watch.headful', 'browser.watch.loading', 'browser.watch.title', 'browser.watch.url', 'browser.watch.vh', 'browser.watch.vw', 'browser.watch.watching', // channels.qr: 2 'channels.qr.qr', 'channels.qr.qr_text', + // subagents.list / subagents.probe: 1 each -- the machine an acp row runs on; the + // library's rows all run on this computer, where the server sends null. + 'subagents.list.rows[].machine', 'subagents.probe.rows[].machine', // config.set: 5 -- the model-switch group, which the offline library does not // model: its gateway always has a loop, so there is no restart to report. 'config.set.applies_to_session', 'config.set.needs_restart', 'config.set.scope', 'config.set.session_id', 'config.set.value', From 8a4da21a7c88a8beacec9148b1de8921e9fb5edb Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:43:30 +0800 Subject: [PATCH 06/44] fix(*): show a remote path only where the tool speaks; land a fetched file under its whole path A read's numbered lines, a search's matches and a diff's body came back with every registered root rewritten as an address, so the model edited against text the file does not hold: only the tool's own sentences -- its errors, its headers and footers, a write's account and a diff's two header lines -- are shown that way now. A delivered file landed under its bare name, so two files of one name from two directories landed on each other: it lands under its whole path on the machine. A file outside the directories this Raven's agents work in there was fetched on the strength of a read some conversation had been allowed, which is no evidence for this one: it is refused, and read there instead. The remote run's reply names the file tools again, now that they take its path. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/acp_agent.py | 3 ++- raven/agent/tools/deliver.py | 23 +++++++++-------- raven/agent/tools/remote_files.py | 43 ++++++++++++++++++++++++------- tests/test_deliver_files_tool.py | 24 ++++++++++++++--- tests/test_remote_file_tools.py | 41 +++++++++++++++++++++++++++++ tests/test_subagent_acp_remote.py | 2 +- 6 files changed, 111 insertions(+), 25 deletions(-) diff --git a/raven/acp_client/acp_agent.py b/raven/acp_client/acp_agent.py index 404c48029..f6fea6d1e 100644 --- a/raven/acp_client/acp_agent.py +++ b/raven/acp_client/acp_agent.py @@ -1832,7 +1832,8 @@ async def run( notes.append( f"Ran on machine {launched.target.label}, in " f"{remote.shown(launched.target, session_cwd)}; the files it wrote are there, " - "not on this computer" + "not on this computer -- read_file, list_dir, find, grep, write_file and edit_file take " + "that path as it is, and deliver_files fetches one by it" ) for note in filter(None, notes): notice = f"\n\n[raven] {note}." diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index 61d288588..65cb8056e 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -64,13 +64,15 @@ def _on_machine(raw: str) -> RemotePath | str | None: async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path | None, str]: - """``where``'s file, fetched from its machine to ``/pulled//``; or why it was not. - - Only a file this process may read without asking -- under the directories - this Raven's agents work in there, or one it has already read (and so was - allowed) -- is fetched here: a delivery is not a way around the read - check. One over :data:`PULL_FILE_BUDGET` stays, and is named with its - size. + """``where``'s file, fetched from its machine to ``/pulled//``; or why it was not. + + Only a file under the directories this Raven's agents work in on that + machine is fetched here, since this tool asks nothing: a delivery is not + a way around the read check, and what another conversation was allowed + to read is no evidence for this one. A file elsewhere is read there. One + over :data:`PULL_FILE_BUDGET` stays, and is named with its size. The file + lands under its whole path on the machine, so two files of one name from + two directories never land on each other. """ pool = remote_path.nodes() label = remote_path._label(where._row) @@ -80,11 +82,10 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path home = await pool.home(where.machine) place = where.absolute(home) roots = remote_path.reading_roots(where) - inside = any(remote_path.within(place, remote_path.absolute(root, home)) for root in roots) - if not inside and remote_path.last_read(where.machine, place) is None: + if not any(remote_path.within(place, remote_path.absolute(root, home)) for root in roots): return None, ( f"{where.display(place)} is outside the directories this Raven's agents work in on machine " - f"{label}; read it with read_file first" + f"{label}; read it there instead" ) fence = [remote_path.fence("read_file", where, place, home)] info = await pool.stat(where.machine, place, roots=fence) @@ -96,7 +97,7 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path f"{_human_size(size)} is over the {_human_size(PULL_FILE_BUDGET)} " f"limit for a file brought back from machine {label}; read it there instead" ) - local = (workspace or Path.cwd()) / "pulled" / where.machine / Path(place).name + local = (workspace or Path.cwd()) / "pulled" / where.machine / Path(place).relative_to("/") await pool.pull_file(where.machine, place, local, roots=fence) except remote_path.NodeCallError as exc: return None, str(exc) diff --git a/raven/agent/tools/remote_files.py b/raven/agent/tools/remote_files.py index b82da45c7..830151b26 100644 --- a/raven/agent/tools/remote_files.py +++ b/raven/agent/tools/remote_files.py @@ -15,7 +15,9 @@ Every failure is one sentence the tool returns: a path that names no registered machine, a machine without this Raven's node, one that does not -answer. +answer. The file's own lines -- a read's numbered lines, a search's matches, +a diff's body -- come back as they are: only what the tool says about paths +is shown as addresses, so the model edits against the bytes the file holds. """ from __future__ import annotations @@ -81,41 +83,64 @@ async def answer(tool: str, path: Any, params: dict[str, Any]) -> str | ToolResu remote_path.remember_read(where.machine, place, got["sha256"]) elif tool in ("write_file", "edit_file"): remote_path.forget_read(where.machine, place) - return _shown(got, where, [place, *roots]) + return _shown(got, where, [place, *roots], tool) -def _shown(got: dict[str, Any], where: RemotePath, paths: list[str]) -> str | ToolResult: - """The node's answer with each path it was given shown as scp writes it. +#: The lines of a tool's answer that are the file's own bytes, which no path +#: display may rewrite: a read's numbered lines, a search's matches and +#: context (``path:line:text`` and ``path-line-text``). What the tool says +#: about paths -- its errors, its headers and footers -- is shown as +#: addresses; the content stays what the model will edit against. +_CONTENT = { + "read_file": re.compile(r"^\d+\| "), + "grep": re.compile(r"^.+?[:-]\d+[:-]"), +} +_DIFF_HEADER = ("--- ", "+++ ") + + +def _shown(got: dict[str, Any], where: RemotePath, paths: list[str], tool: str) -> str | ToolResult: + """The node's answer with each path it was given shown as scp writes it, outside the file's own lines. One pass, longest path first: a shown path contains the plain one, and a - second pass over it would show it twice. + second pass over it would show it twice. A diff's content lines are left + alone too; only its two header lines name the file. """ swaps = sorted({p for p in paths if p.startswith("/")}, key=len, reverse=True) pattern = re.compile("|".join(re.escape(p) for p in swaps)) if swaps else None + content = _CONTENT.get(tool) def show(text: Any) -> str: out = str(text or "") return pattern.sub(lambda m: where.display(m.group(0)), out) if pattern else out - text = show(got.get("text")) + def shown(text: Any) -> str: + """``text`` with its paths shown, line by line, the content lines untouched.""" + if content is None: + return show(text) + return "\n".join(line if content.match(line) else show(line) for line in str(text or "").split("\n")) + + def shown_diff(diff: str) -> str: + return "\n".join(show(line) if line.startswith(_DIFF_HEADER) else line for line in diff.split("\n")) + + text = shown(got.get("text")) if "blocks" not in got: return text blocks = [] for block in got.get("blocks") or []: if isinstance(block, dict) and block.get("type") == "text": - block = {**block, "text": show(block.get("text"))} + block = {**block, "text": shown(block.get("text"))} blocks.append(block) display = got.get("display") change = got.get("file_change") return ToolResult( model_text=text, - display_text=show(display) if display is not None else None, + display_text=shown(display) if display is not None else None, ok=bool(got.get("ok", True)), blocks=blocks, # A write's account of what it changed, for a surface with a diff # view of its own: the file's whole content both ways, under the # path as scp writes it. - diff=show(got["diff"]) if isinstance(got.get("diff"), str) else None, + diff=shown_diff(got["diff"]) if isinstance(got.get("diff"), str) else None, file_change=FileChange( path=show(change.get("path")), after=str(change.get("after") or ""), before=change.get("before") ) diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 5875c1071..5861f348f 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -415,12 +415,28 @@ async def test_a_file_on_a_machine_is_brought_back_and_delivered(tool, tmp_path, manifest = tool.take_metadata()["raven_delivery"] assert manifest["invalid"] == [], summary (entry,) = manifest["files"] - landed = tmp_path / "chanwork" / "pulled" / "box" / "report.pdf" + landed = tmp_path / "chanwork" / "pulled" / "box" / "root" / "raven-work" / "t1" / "report.pdf" assert entry["name"] == "report.pdf" and entry["path"] == str(landed) and entry["size"] == 5 assert landed.read_bytes() == b"xxxxx" assert machines.pulled == [("/root/raven-work/t1/report.pdf", landed, ["/root/raven-work"])] +async def test_two_files_of_one_name_from_two_directories_land_apart(tool, tmp_path, machines) -> None: + summary = await tool.execute( + files=[{"path": "box:raven-work/a/report.pdf"}, {"path": "box:raven-work/b/report.pdf"}] + ) + + manifest = tool.take_metadata()["raven_delivery"] + assert manifest["invalid"] == [], summary + landed = [Path(entry["path"]) for entry in manifest["files"]] + pulled = tmp_path / "chanwork" / "pulled" / "box" / "root" / "raven-work" + assert landed == [pulled / "a" / "report.pdf", pulled / "b" / "report.pdf"] + assert [where for where, _, _ in machines.pulled] == [ + "/root/raven-work/a/report.pdf", + "/root/raven-work/b/report.pdf", + ] + + async def test_a_file_over_the_budget_stays_on_its_machine(tool, machines) -> None: machines.size = 21 * 1024 * 1024 @@ -439,6 +455,7 @@ async def test_what_was_not_delivered_says_why(tool, machines) -> None: outside = await tool.execute(files=[{"path": "box:.ssh/id_rsa"}]) remote_path.remember_read("box", "/root/.ssh/id_rsa", "ab" * 32) read_first = await tool.execute(files=[{"path": "box:.ssh/id_rsa"}]) + after_a_read = read_first nobody = await tool.execute(files=[{"path": "nobox:/etc/hostname"}]) machines.kind = "dir" folder = await tool.execute(files=[{"path": "box:raven-work/t1"}]) @@ -448,10 +465,11 @@ async def test_what_was_not_delivered_says_why(tool, machines) -> None: in big ) assert ( - "is outside the directories this Raven's agents work in on machine 'Lab box' (box); read it with read_file first" + "is outside the directories this Raven's agents work in on machine 'Lab box' (box); read it there instead" in outside ) - assert "id_rsa" in read_first and "outside" not in read_first, "a file already read was allowed to be" + assert "outside" in after_a_read, "a read by some conversation is no evidence for this one" + assert machines.pulled == [], "nothing outside the roots is fetched" assert "is written as a path on another machine" in nobody assert "not found or not a regular file on machine 'Lab box' (box)" in folder # The machine is named by its label in every sentence; its address appears diff --git a/tests/test_remote_file_tools.py b/tests/test_remote_file_tools.py index 7c7fb3a43..c6ab51917 100644 --- a/tests/test_remote_file_tools.py +++ b/tests/test_remote_file_tools.py @@ -310,6 +310,47 @@ def _writers(tmp_path: Path) -> dict[str, Any]: return {"write_file": WriteFileTool(**fenced), "edit_file": EditFileTool(**fenced)} +async def test_a_read_s_own_lines_come_back_as_they_are_and_only_the_tool_s_sentences_show_addresses( + registry, pool, tmp_path +): + pool.answer = { + "text": f"1| export ROOT={HOME}/raven-work\n2| cd {HOME}/raven-work/t1\n\n(Showing lines 1-2 of 2. Use offset=3 to continue.)", + "sha256": "ab" * 32, + } + + got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/env.sh") + + assert got.startswith(f"1| export ROOT={HOME}/raven-work\n2| cd {HOME}/raven-work/t1\n"), "the file's bytes" + assert SHOWN not in got.split("\n\n")[0] and "(Showing lines 1-2 of 2." in got + + +async def test_a_search_s_matches_and_a_diff_s_body_keep_their_lines(registry, pool, tmp_path): + pool.answer = { + "text": f"t1/env.sh:1:export ROOT={HOME}/raven-work\nt1/env.sh-2-cd {HOME}/raven-work/t1\n\n1 match in 1 file" + } + matches = await _tools(tmp_path)["grep"].execute("ROOT", "box:raven-work") + + place = f"{HOME}/raven-work/t1/env.sh" + pool.answer = { + "text": f"Successfully wrote 30 bytes to {place}", + "display": None, + "ok": True, + "blocks": [], + "diff": f"--- {place}\n+++ {place}\n@@ -0,0 +1 @@\n+export ROOT={HOME}/raven-work", + "file_change": {"path": place, "after": f"export ROOT={HOME}/raven-work\n", "before": None}, + } + written = await _writers(tmp_path)["write_file"].execute( + "box:raven-work/t1/env.sh", f"export ROOT={HOME}/raven-work\n" + ) + + assert matches.startswith(f"t1/env.sh:1:export ROOT={HOME}/raven-work\nt1/env.sh-2-cd {HOME}/raven-work/t1\n") + assert SHOWN not in matches + assert isinstance(written, ToolResult) + assert written.model_text == f"Successfully wrote 30 bytes to {SHOWN}{place}" + assert written.diff == f"--- {SHOWN}{place}\n+++ {SHOWN}{place}\n@@ -0,0 +1 @@\n+export ROOT={HOME}/raven-work" + assert written.file_change is not None and written.file_change.path == f"{SHOWN}{place}" + + async def test_a_write_and_an_edit_go_to_the_node_and_come_back_as_the_tool_s_result(registry, pool, tmp_path): place = f"{HOME}/raven-work/t1/a.md" pool.answer = { diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 1286cb10d..cbb74dd9f 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -189,7 +189,7 @@ async def test_a_dispatch_runs_on_the_machine_in_a_directory_made_there_and_says assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in reply assert "not on this computer" in reply assert "the files it wrote are there, not on this computer" in reply - assert "read_file" not in reply, "no tool is named that this change does not give a remote path to" + assert "read_file, list_dir, find, grep, write_file and edit_file take that path as it is" in reply assert not local.exists(), "the caller's workspace is never touched" for argv in box.calls(): assert str(local) not in " ".join(argv), "this computer's workspace path never reaches the machine" From 9223a583a10705ce9233fcae773f74d708811bd7 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:00:17 +0800 Subject: [PATCH 07/44] fix(*): the daemon's socket and records are the owner's alone; the ssh argv goes as it is The socket's fallback for a long node directory lay in the temp directory itself, so the relay forwarded the host's lines to whatever listened at that path, and a file there the user could not remove kept the daemon from starting: the fallback now lies in a directory of the user's own (0700, checked, never a link or another's), the relay forwards only to a socket of the user's, and a daemon refused its socket's place says so in one line and leaves. The task records -- the agent's frames as they were -- were made with the default mode: tasks/, each record and its three files are the owner's alone now, as the host's journal is, and the installer makes the node directory so. A page read, a digest, a chunk and a checked write ran on the daemon's one loop, which also answers the heartbeat: they run in a thread now, writers one at a time under a lock. The ssh argv went to the connection as one shell-quoted line, which a Windows host's own grammar split into twelve words: it goes as the argv, which no host reads again. A call on its way to a node counts as use, so a link is not closed under it as idle. A drive-relative or extended Windows path (C:file, \\?\C:\...) no longer reads as a machine's. The daemon's tests skip where there is no fcntl, the POSIX teardown tests lend a SIGKILL where the platform has none, the login-shell test checks a directory of its own instead of /nonexistent, a node is run with nothing importable beyond its requirements, and remote.py says what does reach a model: a path on the machine, written the way scp writes it. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 7 +- raven/acp_client/client.py | 22 +++-- raven/acp_client/remote.py | 36 ++++--- raven/node/__main__.py | 7 +- raven/node/client.py | 27 ++++-- raven/node/install.py | 5 +- raven/node/relay.py | 17 +++- raven/node/server.py | 129 +++++++++++++++++++++---- raven/ops/remote_path.py | 14 ++- tests/test_acp_client_launch.py | 36 +++++++ tests/test_acp_client_teardown.py | 10 +- tests/test_acp_remote.py | 25 ++--- tests/test_node_client.py | 45 ++++++++- tests/test_node_install.py | 8 +- tests/test_node_relay.py | 23 +++++ tests/test_node_server.py | 151 ++++++++++++++++++++++++++++++ tests/test_ops_remote_path.py | 24 ++++- 17 files changed, 506 insertions(+), 80 deletions(-) create mode 100644 tests/test_acp_client_launch.py diff --git a/CONTEXT.md b/CONTEXT.md index a37b54a54..524eeee65 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -408,7 +408,9 @@ implementation detail). The Raven process on a registered **Machine** that owns the agents the host runs there: a daemon (`python -I -m raven.node --serve --dir `) that the host's relay starts over ssh when none is up (`--attach`: the relay bridges the ssh session to the -daemon's socket under `node_dir/run/` and leaves with the ssh; the daemon leaves on its +daemon's socket -- under `node_dir/run/`, or in a directory of the user's own under the temp +directory when that path is too long for a socket; either way the relay forwards only to a +socket of the user's -- and leaves with the ssh; the daemon leaves on its own once idle). It does three things there and nothing else: answers file calls with Raven's own file tools (`read_file`, `list_dir`, `find`, `grep`, `write_file`, `edit_file`) fenced to the directories each call names, so a read of 50 lines of a large file moves @@ -416,7 +418,8 @@ those 50 lines, and fetches one file whole (`fs/pull`); starts the `acp` agents entries name the machine, each under a task id of the host's, in its own process group in the machine user's login environment, forwards their ACP frames on numbered channels of the connection that holds them and keeps those frames in the task's record -(`node_dir/tasks//`), so a connection that goes leaves the agent running and the next +(`node_dir/tasks//`, owner-only like the host's journal), so a connection that goes +leaves the agent running and the next one takes it over from the frame the host last read (`agent/attach`); and answers probes of the machine and of an agent's command. It holds no conversation, calls no model and opens no network port. It runs exactly the host's code, checked by protocol and digest at every diff --git a/raven/acp_client/client.py b/raven/acp_client/client.py index 5b388a544..02a018cc1 100644 --- a/raven/acp_client/client.py +++ b/raven/acp_client/client.py @@ -25,7 +25,7 @@ import signal import time from collections import deque -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Sequence from functools import partial from typing import Any, Protocol @@ -341,7 +341,7 @@ async def launch( cls, *, name: str, - command: str, + command: str | Sequence[str], cwd: str | None = None, env: dict[str, str] | None = None, on_request: RequestHandler | None = None, @@ -350,6 +350,13 @@ async def launch( ) -> "AcpClient": """Start the agent's ACP server on this computer and begin reading it. + ``command`` is the line a config stores, split the way this host reads + a line, or an argv a caller split already, taken as it is: a word of + it that is a command for another machine keeps that machine's + quoting, which this host's grammar would read apart (a Windows host's + parser made twelve words of a node's one remote-command word while it + went as a line; reviewed 2026-10-12). + Raises :class:`AcpConnectionError` if the process cannot be started; a server that starts but never speaks is a per-request timeout, not a launch failure, because the two need different operator action. @@ -360,10 +367,13 @@ async def launch( from raven.agent.subagent.backends.env import host_identity_env, login_shell_env, resolve_program from raven.agent.subagent.role import subagent_role_env - try: - argv = command_argv(command) - except ValueError as exc: - raise AcpConnectionError(f"acp agent {name!r}: command cannot be parsed: {exc}") from exc + if isinstance(command, str): + try: + argv = command_argv(command) + except ValueError as exc: + raise AcpConnectionError(f"acp agent {name!r}: command cannot be parsed: {exc}") from exc + else: + argv = [str(word) for word in command] if not argv: raise AcpConnectionError(f"acp agent {name!r}: command is empty") diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index 8c1ac3a6f..e122d6e91 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -14,9 +14,11 @@ - the registry lookup, refused up front for a row that cannot be used, so a dispatch fails before anything starts rather than as a network fault minutes in; -- the ssh line that carries a node, built from the registry at launch and - never stored, so no address, port or key path reaches a config file, a log - line or a model; +- the ssh argv that carries a node, built from the registry at launch and + never stored, so the way in -- the key path, ssh's options, the address as + ssh takes it -- reaches no config file, log line or model; the address does + reach a model as part of a path on the machine, written the way scp writes + it (:func:`shown`), because that is how a file there is named to the tools; - one sentence for an ssh failure, naming the machine and never its address; ssh's own messages, which do name it, go to an owner-only log rather than to the stderr the connection journals and quotes; @@ -71,9 +73,9 @@ class RemoteMachineError(RuntimeError): """A remote agent's machine could not be used. The message is safe to show. - It names the machine by its id and display name only: the registry keeps - the address out of anything a model reads (``connections.shown``), and an - error is read by one. + It names the machine by its id and display name only, never by its + address: an error is read by a model, and what an address does in front + of one is name a file there (:func:`shown`), not a fault. """ @@ -98,7 +100,7 @@ def field(self, name: str) -> Any: For what a caller needs to know about the machine itself -- where a Raven node is kept on it (``node_dir``), the directories its owner listed (``paths``). The address, port, account and key stay behind - :func:`ssh_line`, :func:`run` and :func:`push`. + :func:`ssh_command`, :func:`run` and :func:`push`. """ if name in _WAY_IN: raise KeyError(f"{name!r} is how raven reaches the machine, not a fact about it") @@ -163,13 +165,17 @@ def posix(command: str) -> str: return "exec /bin/sh -c " + shlex.quote(f"echo; {command}").replace("!", "'\\!'") -def ssh_line(target: Machine, command: str, *, ssh_log: str | os.PathLike[str], connect_timeout: int = 15) -> str: - """The ssh that keeps ``command`` running on ``target`` for as long as it is read: one shell-quoted line. +def ssh_command( + target: Machine, command: str, *, ssh_log: str | os.PathLike[str], connect_timeout: int = 15 +) -> list[str]: + """The ssh that keeps ``command`` running on ``target`` for as long as it is read: its argv. - What a Raven node is started with. A string because that is what the - connection takes (it ``shlex.split``\\ s it); ``shlex.join`` makes the - split give back exactly this argv. Built at launch and held in memory - only: it carries the address. ``command`` is sent through :func:`posix`, + What a Raven node is started with. An argv, handed to the connection as + it is, so no host's own grammar reads it again: the one word that is the + remote command keeps the POSIX quoting the far shell reads, whatever this + host is (a Windows host's parser made twelve words of it while it went as + a line; reviewed 2026-10-12). Built at launch and held in memory only: it + carries the address. ``command`` is sent through :func:`posix`, so the machine user's login shell may be any shell; it carries no variable and reads no login profile -- the node captures the login environment itself, for the agents it starts. @@ -185,7 +191,7 @@ def ssh_line(target: Machine, command: str, *, ssh_log: str | os.PathLike[str], # ``--`` so nothing in the remote command is read as an option to ssh, # which also parses options that follow the destination. argv += ["--", posix(command)] - return shlex.join(argv) + return argv def failure(target: Machine, log: str | os.PathLike[str], exc: BaseException) -> str | None: @@ -432,6 +438,6 @@ def _target(target: Machine) -> tuple[str, int, str, str]: "session_root", "shell_path", "shown", - "ssh_line", + "ssh_command", "ssh_log_path", ] diff --git a/raven/node/__main__.py b/raven/node/__main__.py index 9a37e5a10..d047d2cdb 100644 --- a/raven/node/__main__.py +++ b/raven/node/__main__.py @@ -42,7 +42,12 @@ def main(argv: list[str] | None = None) -> int: if args.serve: from raven.node.server import IDLE_EXIT_S, run_daemon - return run_daemon(root, idle_s=IDLE_EXIT_S if args.idle is None else args.idle) + try: + return run_daemon(root, idle_s=IDLE_EXIT_S if args.idle is None else args.idle) + except PermissionError as exc: + # The socket's place is not this user's to use; said in one line, in the serve log. + print(f"raven node: {exc}", file=sys.stderr) + return 1 from raven.node.relay import run_attach return run_attach(root) diff --git a/raven/node/client.py b/raven/node/client.py index 990e8bbb3..ec7f6c987 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -1,6 +1,6 @@ """The host's side of a Raven node: one live link per registered machine. -A node is reached over ssh (:func:`raven.acp_client.remote.ssh_line`): the +A node is reached over ssh (:func:`raven.acp_client.remote.ssh_command`): the relay the ssh runs bridges it to the daemon on the machine, starting one when none is up. It is spoken to with the client an ACP agent is (:class:`raven.acp_client.client.AcpClient`), @@ -298,6 +298,8 @@ class _Node: hello: dict[str, Any] channels: dict[int, ChannelEndpoint] used: float = field(default_factory=time.monotonic) + calls: int = 0 + """Requests on their way to the node, not yet answered: use, to the idle close, as much as an answer is.""" heartbeat: asyncio.Task[None] | None = None next_channel: int = 1 @@ -338,6 +340,7 @@ async def call( """``method`` on the node of ``machine_id``, starting one if none is live.""" node = await self._acquire(machine_id) label = node.target.label + node.calls += 1 try: result = await node.client.request(method, params, timeout=timeout) except AcpRemoteError as exc: @@ -355,7 +358,9 @@ async def call( said = self._ended(node.target, node.log, exc) await self._lost(machine_id, node, said) raise NodeError(said) from None - node.used = time.monotonic() + finally: + node.calls -= 1 + node.used = time.monotonic() return result if isinstance(result, dict) else {} async def hello(self, machine_id: str) -> dict[str, Any]: @@ -530,8 +535,8 @@ async def _acquire(self, machine_id: str) -> _Node: node.heartbeat = asyncio.create_task(self._heartbeat(machine_id, node), name=f"node-heartbeat-{machine_id}") return node - def _prepare(self, machine_id: str) -> tuple[remote.Machine, Path, str, str]: - """The machine, its ssh log (emptied), the ssh line and the install name. Blocking: reads the registry.""" + def _prepare(self, machine_id: str) -> tuple[remote.Machine, Path, list[str], str]: + """The machine, its ssh log (emptied), the ssh argv and the install name. Blocking: reads the registry.""" target = remote.machine(machine_id) install_name = bundle.install_name() log = remote.ssh_log_path(f"raven-node@{target.id}") @@ -556,21 +561,23 @@ def _prepare(self, machine_id: str) -> tuple[remote.Machine, Path, str, str]: f"--dir {remote.shell_path(node_dir(target))}" ) try: - line = remote.ssh_line(target, command, ssh_log=log) + argv = remote.ssh_command(target, command, ssh_log=log) except ValueError as exc: raise remote.RemoteMachineError(f"machine {target.label}: {exc}") from None - return target, log, line, install_name + return target, log, argv, install_name async def _launch(self, machine_id: str) -> _Node: try: - target, log, line, install_name = await asyncio.to_thread(self._prepare, machine_id) + target, log, argv, install_name = await asyncio.to_thread(self._prepare, machine_id) except remote.RemoteMachineError as exc: raise NodeError(str(exc)) from None channels: dict[int, ChannelEndpoint] = {} try: client = await AcpClient.launch( name=f"raven-node@{target.id}", - command=line, + # The argv itself: a line would be read by this host's own + # grammar, which is not the far shell's (Windows, 2026-10-12). + command=argv, # Fixed rather than any caller's workspace: the ssh needs none. cwd=str(Path.home()), env={}, @@ -765,9 +772,11 @@ async def _drop(self, machine_id: str, reason: str | None = None) -> None: await node.client.close() async def _close_idle(self, *, keep: str) -> None: + """Close the links to other machines that nothing uses: no agent on them, no call on its way to them.""" now = time.monotonic() for machine_id, node in list(self._nodes.items()): - if machine_id != keep and not node.channels and now - node.used > self._idle_close_s: + idle = not node.channels and not node.calls and now - node.used > self._idle_close_s + if machine_id != keep and idle: await self._drop(machine_id) diff --git a/raven/node/install.py b/raven/node/install.py index b64ad266c..43e95c9d3 100644 --- a/raven/node/install.py +++ b/raven/node/install.py @@ -216,12 +216,13 @@ def steps(p: Plan) -> list[tuple[str, str]]: out.append( ( "uv", - head + f'mkdir -p "$dir/.uv" && {fetch} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' + head + + f'mkdir -p "$dir/.uv" && chmod 700 "$dir" && {fetch} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' 'INSTALLER_NO_MODIFY_PATH=1 sh >/dev/null && test -x "$dir/.uv/uv"', ) ) python = shlex.quote(p.python) if p.how == "python" else "3.12" - venv = f'mkdir -p "$dir" && {env} "$uv" venv --quiet --python {python} "$node"' + venv = f'mkdir -p "$dir" && chmod 700 "$dir" && {env} "$uv" venv --quiet --python {python} "$node"' pip = f'{env} "$uv" pip install --quiet --python "$node/bin/python"' requirements = " ".join(shlex.quote(r) for r in bundle.pinned(protocol.REQUIREMENTS)) deps = f"{pip} {requirements}" diff --git a/raven/node/relay.py b/raven/node/relay.py index 3eecd3f8e..f23331a72 100644 --- a/raven/node/relay.py +++ b/raven/node/relay.py @@ -29,9 +29,7 @@ def start_daemon(root: Path, *, python: str | None = None) -> subprocess.Popen[bytes]: """Start the daemon under ``root`` in a session of its own; its words go to ``log/serve.log``.""" - log_dir = root / server.LOG_DIR - log_dir.mkdir(parents=True, exist_ok=True) - log = (log_dir / "serve.log").open("ab") + log = server.own_file(server.own_dir(root / server.LOG_DIR) / "serve.log") argv = [python or sys.executable, "-I", "-m", "raven.node", "--serve", "--dir", str(root)] # How long the daemon stays idle is the daemon's default, unless the # environment the relay runs in says otherwise (tests end theirs quickly). @@ -54,11 +52,20 @@ def start_daemon(root: Path, *, python: str | None = None) -> subprocess.Popen[b async def connect( root: Path, *, python: str | None = None, timeout_s: float = START_TIMEOUT_S ) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]: - """A connection to the daemon under ``root``, started here when none answers.""" + """A connection to the daemon under ``root``, started here when none answers. + + The socket's directory is this user's own (:func:`raven.node.server.own_dir`), + so what listens there is this user's daemon; a thing at the socket's place + that is somebody else's is not forwarded to, and no daemon is started + beside it. + """ sock = server.socket_path(root) + server.own_dir(sock.parent) deadline = time.monotonic() + timeout_s started = False while True: + if not server.owned(sock): + raise ConnectionError(f"{sock} is not this user's own; the relay forwards to this user's Raven daemon only") try: return await asyncio.open_unix_connection(str(sock), limit=_BLOCK) except (FileNotFoundError, ConnectionRefusedError, OSError): @@ -110,7 +117,7 @@ def run_attach(root: Path) -> int: async def main() -> int: try: reader, writer = await connect(root) - except ConnectionError as exc: + except (ConnectionError, PermissionError) as exc: sys.stderr.write(f"{exc}\n") return 1 loop = asyncio.get_running_loop() diff --git a/raven/node/server.py b/raven/node/server.py index 2a95a63e5..773c28ab9 100644 --- a/raven/node/server.py +++ b/raven/node/server.py @@ -47,6 +47,7 @@ import subprocess import sys import tempfile +import threading import time import uuid from collections.abc import Awaitable, Callable @@ -325,9 +326,10 @@ async def _write(params: dict[str, Any]) -> dict[str, Any]: ``expect`` is the file as the host last read it -- its digest, or ``absent`` -- and a file that is not so any more is left alone, with - :data:`raven.node.protocol.CHANGED`. The check and the write are one - stretch with no await between them, and this node's requests interleave - only at awaits, so what was checked is what is written over. + :data:`raven.node.protocol.CHANGED`. The check and the write run in one + worker thread under the daemon's write lock (:func:`_apart`), so no other + write comes between them: what was checked is what is written over, and + the loop is not held while a whole file is hashed. """ from raven.agent.tools.filesystem import WriteFileTool @@ -353,7 +355,7 @@ async def _write(params: dict[str, Any]) -> dict[str, Any]: async def _edit(params: dict[str, Any]) -> dict[str, Any]: - """The host's ``edit_file`` on this machine: one read, the replacement, one write, with nothing between.""" + """The host's ``edit_file`` on this machine: one read, the replacement, one write, with nothing between (under the write lock, :func:`_apart`).""" from raven.agent.tools.filesystem import EditFileTool tool = _tool(EditFileTool, params) @@ -543,17 +545,73 @@ def _end_tree(pid: int) -> None: def socket_path(root: Path) -> Path: - """Where the daemon of this install listens for ``root``: under it, or in the temp directory when that is too long. + """Where the daemon of this install listens for ``root``: under ``run/``, or when that path is too long for a socket, in a directory of this user's own in the temp directory. A unix socket's path is short by the system's rule, and a node directory - is wherever the owner put it; the fallback is named after ``root`` so a - relay and a daemon given the same directory meet, whichever they use. + is wherever the owner put it. The fallback is named after ``root`` so a + relay and a daemon given the same directory meet, and it lies in a + directory only this user may open (:func:`own_dir`), as ``run/`` is, so + what listens there is this user's own daemon and nothing else. """ under = root / RUN_DIR / f"{bundle.install_name()}.sock" if len(os.fsencode(under)) <= _SOCKET_PATH_MAX: return under tag = hashlib.sha256(os.fsencode(root.resolve())).hexdigest()[:16] - return Path(tempfile.gettempdir()) / f"raven-node-{tag}.sock" + return Path(tempfile.gettempdir()) / f"raven-node-{os.getuid()}" / f"{tag}.sock" + + +def own_dir(path: Path) -> Path: + """``path`` as a directory only this user may open: made so, or refused. + + What the socket's directory and the records' must be for what is in them + to be this user's alone: a directory of this user's, not a link, with no + group or other bits. One of this user's with wider bits is narrowed; + anything else is left as it is and named, never used. + """ + try: + st = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=True, exist_ok=True) + st = path.lstat() + except OSError as exc: + raise PermissionError(f"could not make {path}: {exc.strerror or exc}") from None + ours = filemode.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() + if ours and st.st_mode & 0o077: + try: + os.chmod(path, 0o700) + st = path.lstat() + except OSError as exc: + raise PermissionError(f"could not make {path} this user's own: {exc.strerror or exc}") from None + if not ours or st.st_mode & 0o077: + raise PermissionError( + f"{path} is not a directory only this user may open; remove it, or give the node a shorter directory" + ) + return path + + +def owned(path: Path) -> bool: + """Whether what is at ``path`` is this user's own, or nothing is there yet. + + What a relay checks before it forwards to a socket: in a directory of this + user's own (:func:`own_dir`) anything that is somebody else's is root's, + and the relay neither forwards to it nor removes it. + """ + try: + return path.lstat().st_uid == os.getuid() + except FileNotFoundError: + return True + + +def own_file(path: Path, mode: str = "ab") -> Any: + """``path`` opened to append (or, ``"w"``, to write over), made with no group or other bits when it is new. + + A task's record holds the agent's frames as they were -- its replies, its + tool results, which quote what it read -- so it is its owner's alone, as + the host's own journal is. + """ + flags = os.O_WRONLY | os.O_CREAT | (os.O_APPEND if "a" in mode else os.O_TRUNC) + return os.fdopen(os.open(path, flags, 0o600), mode, **({} if "b" in mode else {"encoding": "utf-8"})) def pid_path(root: Path) -> Path: @@ -614,8 +672,8 @@ def __init__( self.frames = 0 self.subscriber: tuple[Node, int] | None = None self._send_lock = asyncio.Lock() - self._frames_file = (record / "frames.jsonl").open("ab") - self._stderr_file = (record / "stderr.log").open("ab") + self._frames_file = own_file(record / "frames.jsonl") + self._stderr_file = own_file(record / "stderr.log") self.write_record("running") self._forwarders = [ asyncio.create_task(self._forward(proc.stdout, frames=True)), @@ -759,7 +817,8 @@ async def close(self) -> int | None: def _write_json(path: Path, data: dict[str, Any]) -> None: tmp = path.with_suffix(".json.tmp") - tmp.write_text(json.dumps(data), encoding="utf-8") + with own_file(tmp, "w") as handle: + handle.write(json.dumps(data)) os.replace(tmp, path) @@ -795,7 +854,7 @@ def __init__(self, root: Path, *, ephemeral: bool = False) -> None: self._env: dict[str, str] | None = None self._env_lock = asyncio.Lock() self.last_activity = time.monotonic() - (root / TASKS_DIR).mkdir(parents=True, exist_ok=True) + own_dir(root / TASKS_DIR) def touch(self) -> None: self.last_activity = time.monotonic() @@ -831,7 +890,7 @@ async def launch(self, node: Node, channel: int, params: dict[str, Any]) -> _Tas cwd.mkdir(parents=True, exist_ok=True) if record.exists(): shutil.rmtree(record) - record.mkdir(parents=True) + record.mkdir(mode=0o700, parents=True) except OSError as exc: raise _RefusedError(_INTERNAL_ERROR, f"could not make {cwd}: {exc.strerror or exc}") from None try: @@ -1046,6 +1105,27 @@ async def shutdown(self, params: dict[str, Any]) -> dict[str, Any]: return {} +#: The file calls that touch a whole file -- a page read beside the file's +#: digest, a stat with one, a chunk, a write checked against one -- answered +#: in a thread, so the loop that answers the heartbeat and forwards every +#: agent's frames is not held for as long as a large file takes (measured +#: 2026-10-12: 3.5 s for a 1 GiB file, with no ping answered meanwhile). +_WHOLE_FILE = frozenset({protocol.READ, protocol.STAT, protocol.WRITE, protocol.EDIT, protocol.PULL}) +#: The ones that change a file, one at a time: what a write checked is what it writes over. +_WRITERS = frozenset({protocol.WRITE, protocol.EDIT}) +_WRITE_LOCK = threading.Lock() + + +def _apart( + handler: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]], params: dict[str, Any], *, serial: bool +) -> dict[str, Any]: + """``handler`` run to completion in this worker thread, on a loop of its own; a writer one at a time.""" + if not serial: + return asyncio.run(handler(params)) + with _WRITE_LOCK: + return asyncio.run(handler(params)) + + _HANDLERS: dict[str, Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]] = { protocol.PING: _ping, protocol.PROBE_MACHINE: _probe_machine, @@ -1102,7 +1182,11 @@ async def handle(frame: Any, node: Node | None = None) -> dict[str, Any] | None: params = params or {} if not isinstance(params, dict): raise _RefusedError(_INVALID_PARAMS, "params must be an object") - return {"jsonrpc": "2.0", "id": request_id, "result": await handler(params)} + if method in _WHOLE_FILE: + result = await asyncio.to_thread(_apart, handler, params, serial=method in _WRITERS) + else: + result = await handler(params) + return {"jsonrpc": "2.0", "id": request_id, "result": result} except _RefusedError as exc: return {"jsonrpc": "2.0", "id": request_id, "error": {"code": exc.code, "message": exc.message}} except Exception as exc: # noqa: BLE001 - the host gets a reason; the daemon keeps serving @@ -1200,18 +1284,27 @@ async def _connection(daemon: Daemon, reader: asyncio.StreamReader, writer: asyn async def run_daemon_async(root: Path, *, idle_s: float = IDLE_EXIT_S) -> int: """Serve on the socket under ``root`` until idle for ``idle_s`` or told to stop; 0 when another daemon already does.""" - run_dir = root / RUN_DIR - run_dir.mkdir(parents=True, exist_ok=True) - os.chmod(run_dir, 0o700) + run_dir = own_dir(root / RUN_DIR) sock = socket_path(root) + if sock.parent != run_dir: + own_dir(sock.parent) lock = (run_dir / f"{bundle.install_name()}.lock").open("w") try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: lock.close() return 0 - with contextlib.suppress(FileNotFoundError): + try: sock.unlink() + except FileNotFoundError: + pass + except OSError as exc: + # Whatever is there is not this daemon's to remove -- in a directory + # of this user's own, root's -- and the daemon does not listen beside it. + lock.close() + raise PermissionError( + f"{sock} is in the way of the daemon and cannot be removed: {exc.strerror or exc}" + ) from None pid_path(root).write_text(str(os.getpid()), encoding="utf-8") daemon = Daemon(root) daemon.sweep() diff --git a/raven/ops/remote_path.py b/raven/ops/remote_path.py index 1bdab6359..f35ffb67b 100644 --- a/raven/ops/remote_path.py +++ b/raven/ops/remote_path.py @@ -33,6 +33,7 @@ from __future__ import annotations +import os import posixpath import re from collections.abc import Callable @@ -62,6 +63,9 @@ _URI = re.compile(r"^(?:scp|sftp)://", re.IGNORECASE) _SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://") _DRIVE = re.compile(r"^[A-Za-z]:(?:[\\/]|$)") +#: A drive-relative path (``C:file``), which only a Windows host reads as one: elsewhere ``c:file`` is scp's machine ``c``. +_DRIVE_RELATIVE = re.compile(r"^[A-Za-z]:") +_WINDOWS = os.name == "nt" _PORT = re.compile(r"^[0-9]{1,5}$") @@ -323,11 +327,15 @@ def looks_remote(text: str) -> bool: """Whether ``text`` is written as a remote path, before asking which machine it names. Local, as scp reads it: anything starting with ``/``, ``./``, ``../`` or - ``~``; a Windows drive (``C:\\`` or ``C:/``); a URL of another scheme; - and any path whose first ``/`` comes before its first ``:``. + ``~``; a Windows drive (``C:\\`` or ``C:/``), a UNC or extended Windows + path (``\\\\server\\share``, ``\\\\?\\C:\\...``) and, on a Windows host, + a drive-relative path (``C:file``); a URL of another scheme; and any + path whose first ``/`` comes before its first ``:``. """ raw = str(text or "").strip() - if not raw or raw.startswith(("/", "./", "../", "~")) or _DRIVE.match(raw): + if not raw or raw.startswith(("/", "./", "../", "~", "\\\\")) or _DRIVE.match(raw): + return False + if _WINDOWS and _DRIVE_RELATIVE.match(raw): return False if _URI.match(raw): return True diff --git a/tests/test_acp_client_launch.py b/tests/test_acp_client_launch.py new file mode 100644 index 000000000..565da9e83 --- /dev/null +++ b/tests/test_acp_client_launch.py @@ -0,0 +1,36 @@ +"""AcpClient.launch: a command line is split the way this host reads one; an argv is taken as it is.""" + +from __future__ import annotations + +import os +import sys + +import pytest + +from raven.acp_client import client as acp_client +from raven.agent.subagent.backends import env as backends_env + + +async def test_an_argv_is_started_as_it_is_and_no_host_grammar_reads_it_again(monkeypatch): + monkeypatch.setattr(acp_client, "command_argv", lambda command: pytest.fail("an argv is not split again")) + monkeypatch.setattr(backends_env, "login_shell_env", lambda: dict(os.environ)) + started: list[list[str]] = [] + + def as_given(argv: list[str], child_env: dict[str, str], *, batch_files: bool = False) -> list[str]: + started.append(list(argv)) + return argv + + monkeypatch.setattr(backends_env, "resolve_program", as_given) + # The last word is a command for another machine's shell: one word here, whatever this host would make of it. + argv = [sys.executable, "-c", "import sys; sys.stdin.read()", "--", "exec /bin/sh -c 'echo; node --attach'"] + + client = await acp_client.AcpClient.launch(name="argv", command=argv, env={}) + try: + assert started == [argv] + finally: + await client.close() + + +async def test_an_empty_argv_is_refused_like_an_empty_line(): + with pytest.raises(acp_client.AcpConnectionError, match="command is empty"): + await acp_client.AcpClient.launch(name="none", command=[], env={}) diff --git a/tests/test_acp_client_teardown.py b/tests/test_acp_client_teardown.py index 16cf5ef64..9a1475726 100644 --- a/tests/test_acp_client_teardown.py +++ b/tests/test_acp_client_teardown.py @@ -27,9 +27,15 @@ def kill(self) -> None: self.killed += 1 +def _as_posix(monkeypatch: pytest.MonkeyPatch) -> None: + """The POSIX branch, wherever the test runs: Windows has no SIGKILL to name, so one is lent for the read.""" + monkeypatch.setattr(acp_client, "_POSIX", True) + monkeypatch.setattr(acp_client.signal, "SIGKILL", getattr(acp_client.signal, "SIGKILL", 9), raising=False) + + def test_on_posix_the_process_group_is_killed(monkeypatch): swept: list[tuple[int, int]] = [] - monkeypatch.setattr(acp_client, "_POSIX", True) + _as_posix(monkeypatch) monkeypatch.setattr(os, "killpg", lambda pgid, sig: swept.append((pgid, sig)), raising=False) proc = _Proc() @@ -43,7 +49,7 @@ def test_a_group_already_gone_is_not_an_error(monkeypatch): def gone(pgid, sig): raise ProcessLookupError - monkeypatch.setattr(acp_client, "_POSIX", True) + _as_posix(monkeypatch) monkeypatch.setattr(os, "killpg", gone, raising=False) acp_client._end(4242, _Proc()) diff --git a/tests/test_acp_remote.py b/tests/test_acp_remote.py index 5b7df595d..5fe0894ff 100644 --- a/tests/test_acp_remote.py +++ b/tests/test_acp_remote.py @@ -75,11 +75,11 @@ def _run_as_sshd(command: str, home: Path) -> subprocess.CompletedProcess[str]: # --- the ssh line ------------------------------------------------------------- -def test_the_ssh_line_reaches_the_machine_with_the_registry_s_options_and_stays_open(tmp_path: Path): +def test_the_ssh_command_reaches_the_machine_with_the_registry_s_options_and_stays_open(tmp_path: Path): log = tmp_path / "box.log" - argv = shlex.split(remote.ssh_line(_box(), "node --stdio", ssh_log=log)) + argv = remote.ssh_command(_box(), "node --stdio", ssh_log=log) - assert argv[0] == "ssh" + assert isinstance(argv, list) and argv[0] == "ssh" assert argv[argv.index("-i") + 1] == KEY assert argv[argv.index("-p") + 1] == str(PORT) assert "BatchMode=yes" in argv @@ -98,22 +98,23 @@ def test_the_ssh_line_reaches_the_machine_with_the_registry_s_options_and_stays_ assert argv.index("-T") < dest and argv.index("-E") < dest -def test_the_ssh_line_carries_no_variable_and_reads_no_profile(tmp_path: Path): +def test_the_ssh_command_carries_no_variable_and_reads_no_profile(tmp_path: Path): # The node captures the login environment itself, for the agents it # starts; what carries it is one plain line any login shell reads. - line = remote.ssh_line(_box(), "~/.raven-node/x/bin/python -I -m raven.node --stdio", ssh_log=tmp_path / "l") + argv = remote.ssh_command(_box(), "~/.raven-node/x/bin/python -I -m raven.node --stdio", ssh_log=tmp_path / "l") - sent = shlex.split(line)[-1] + sent = argv[-1] + line = shlex.join(argv) assert sent == "exec /bin/sh -c 'echo; ~/.raven-node/x/bin/python -I -m raven.node --stdio'" assert "-lic" not in line and "mkdir" not in line and "read -r" not in line @_posix_shell -def test_the_ssh_line_s_command_runs_with_its_arguments_intact_from_the_login_directory(tmp_path: Path): +def test_the_ssh_command_s_command_runs_with_its_arguments_intact_from_the_login_directory(tmp_path: Path): # The trap measured 2026-10-10: `ssh host -- bash -lic 'qwen --acp'` loses # `--acp` to the far shell's re-parse and starts qwen interactive. command = f"{shlex.quote(sys.executable)} -c {shlex.quote(_PROBE)} --stdio 'two words'" - sent = shlex.split(remote.ssh_line(_box(), command, ssh_log=tmp_path / "box.log"))[-1] + sent = remote.ssh_command(_box(), command, ssh_log=tmp_path / "box.log")[-1] done = _run_as_sshd(sent, tmp_path) @@ -144,7 +145,7 @@ def test_an_entry_that_lost_its_address_is_refused_by_name_not_as_a_network_faul bare = Machine(id="box", display_name="Lab box", _row={"id": "box"}) with pytest.raises(RemoteMachineError, match=r"'Lab box' \(box\): .*has no address"): - remote.ssh_line(bare, "node", ssh_log=tmp_path / "l") + remote.ssh_command(bare, "node", ssh_log=tmp_path / "l") with pytest.raises(RemoteMachineError, match=r"'Lab box' \(box\): .*has no address"): remote.push(bare, "true", b"") @@ -153,7 +154,7 @@ def test_a_line_break_in_what_would_be_sent_is_refused_before_any_ssh(tmp_path: with pytest.raises(ValueError, match="must be one line"): remote.posix("echo a\necho b") with pytest.raises(ValueError, match="must be one line"): - remote.ssh_line(_box(), "node\n--stdio", ssh_log=tmp_path / "l") + remote.ssh_command(_box(), "node\n--stdio", ssh_log=tmp_path / "l") def test_an_absolute_root_is_quoted_and_a_bare_tilde_is_left_to_the_shell(): @@ -382,7 +383,9 @@ def _as_user_of(shell: str, command: str, home: Path) -> subprocess.CompletedPro @pytest.mark.parametrize("shell", _SHELLS) def test_a_command_reads_the_same_under_every_login_shell(tmp_path, shell): shell = _shell(shell) - command = "printf '%s\\n' 'a!b' \"it's\" '$HOME' && [ ! -d /nonexistent ] && echo done" + # A directory the test knows is absent: /nonexistent exists on some machines (nobody's home). + absent = shlex.quote(str(tmp_path / "absent")) + command = f"printf '%s\\n' 'a!b' \"it's\" '$HOME' && [ ! -d {absent} ] && echo done" done = _as_user_of(shell, remote.posix(command), tmp_path) diff --git a/tests/test_node_client.py b/tests/test_node_client.py index 1978c7625..6de917c63 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -389,11 +389,30 @@ async def test_a_machine_that_is_not_registered_is_refused_before_any_ssh(box, p assert box.launches() == [] -async def test_an_ssh_line_that_cannot_be_made_is_one_sentence(box, pool, monkeypatch): +async def test_the_ssh_argv_reaches_the_connection_as_it_is_for_no_host_grammar_to_read_again(box, pool, monkeypatch): + launched: list[Any] = [] + + class _Stub: + @classmethod + async def launch(cls, **kw: Any) -> None: + launched.append(kw["command"]) + raise AcpConnectionError("stood in") + + monkeypatch.setattr(node_client, "AcpClient", _Stub) + + with pytest.raises(NodeError, match="could not start ssh to machine 'Lab box'"): + await pool.hello("box") + + (argv,) = launched + assert isinstance(argv, list) and argv[0] == "ssh" and argv[-2] == "--", "an argv, not a line to split again" + assert argv[-1].startswith("exec /bin/sh -c 'echo; exec ") and "-m raven.node --attach --dir" in argv[-1] + + +async def test_an_ssh_command_that_cannot_be_made_is_one_sentence(box, pool, monkeypatch): def refuse(*args: Any, **kwargs: Any) -> str: raise ValueError("a command for a registered machine must be one line") - monkeypatch.setattr(remote, "ssh_line", refuse) + monkeypatch.setattr(remote, "ssh_command", refuse) with pytest.raises(NodeError, match=r"^machine 'Lab box' \(box\): a command .* must be one line$") as caught: await pool.hello("box") @@ -783,6 +802,28 @@ async def test_a_node_left_idle_is_closed_when_another_machine_is_asked_for_unle assert old.asked == [], "a link closed by choice is just closed; its agents stay with the machine" +async def test_a_call_on_its_way_to_a_node_keeps_its_link_however_long_ago_it_was_last_used(tmp_path): + pool = NodePool(idle_close_s=60.0) + + class _Slow(_Client): + async def request(self, method: str, params: dict[str, Any], *, timeout: float) -> Any: + await asyncio.sleep(0.3) + return await super().request(method, params, timeout=timeout) + + _live(pool, tmp_path, "slow") + pool._nodes["slow"].client = slow = _Slow() # type: ignore[assignment] + pool._nodes["slow"].used = time.monotonic() - 61.0 + _live(pool, tmp_path, "other") + on_its_way = asyncio.create_task(pool.call("slow", protocol.READ, {})) + await asyncio.sleep(0.05) + + await pool.call("other", protocol.READ, {}) + + assert slow.alive and "slow" in pool.live(), "a link with a call on its way is in use, not idle" + assert await on_its_way == {"text": "ok"} + assert pool._nodes["slow"].calls == 0 + + async def test_live_lists_only_the_nodes_still_running_and_close_ends_them(tmp_path): pool = NodePool() one = _live(pool, tmp_path, "one") diff --git a/tests/test_node_install.py b/tests/test_node_install.py index ee61ac064..806a3bf85 100644 --- a/tests/test_node_install.py +++ b/tests/test_node_install.py @@ -289,7 +289,8 @@ def test_the_steps_build_a_virtualenv_with_uv_then_fill_it_with_uv(): assert all(command.startswith(head) for _, command in steps) commands = dict(steps) assert commands["virtualenv"] == ( - head + f'mkdir -p "$dir" && {_UV_ENV} "$uv" venv --quiet --python \'/opt/py 3.12/bin/python3\' "$node"' + head + + f'mkdir -p "$dir" && chmod 700 "$dir" && {_UV_ENV} "$uv" venv --quiet --python \'/opt/py 3.12/bin/python3\' "$node"' ) pip = f'{_UV_ENV} "$uv" pip install --quiet --python "$node/bin/python"' assert commands["packages"] == head + f"{pip} " + " ".join(bundle.pinned(protocol.REQUIREMENTS)) @@ -308,7 +309,7 @@ def test_uv_makes_its_python_inside_the_node_dir(): commands = dict(install.steps(_plan(python="", uv="/root/.local/bin/uv"))) assert commands["virtualenv"].endswith( - f'uv=/root/.local/bin/uv; mkdir -p "$dir" && {_UV_ENV} "$uv" venv --quiet --python 3.12 "$node"' + f'uv=/root/.local/bin/uv; mkdir -p "$dir" && chmod 700 "$dir" && {_UV_ENV} "$uv" venv --quiet --python 3.12 "$node"' ) @@ -323,7 +324,8 @@ def test_a_machine_without_uv_fetches_it_under_the_node_dir_first(fetch, fetched head = "dir=~/.raven-node; id=" + bundle.install_name() + '; node="$dir/$id"; uv="$dir/.uv/uv"; ' assert all(command.startswith(head) for _, command in steps) assert dict(steps)["uv"] == ( - head + f'mkdir -p "$dir/.uv" && {fetched_with} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' + head + + f'mkdir -p "$dir/.uv" && chmod 700 "$dir" && {fetched_with} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' 'INSTALLER_NO_MODIFY_PATH=1 sh >/dev/null && test -x "$dir/.uv/uv"' ) assert dict(steps)["virtualenv"].endswith('"$uv" venv --quiet --python /usr/bin/python3.12 "$node"') diff --git a/tests/test_node_relay.py b/tests/test_node_relay.py index a2a65de6f..75185d080 100644 --- a/tests/test_node_relay.py +++ b/tests/test_node_relay.py @@ -8,11 +8,15 @@ import json import os import signal +import sys import time from pathlib import Path import pytest +if sys.platform == "win32": + pytest.skip("the daemon is POSIX: it locks with fcntl and listens on a unix socket", allow_module_level=True) + from raven.node import __main__ as node_main from raven.node import protocol, relay, server @@ -73,6 +77,25 @@ async def test_the_relay_starts_a_daemon_when_none_answers_and_bridges_bytes_bot os.kill(pid, signal.SIGTERM) +async def test_the_relay_forwards_to_nothing_that_is_not_this_user_s_own_and_starts_no_daemon_beside_it( + tmp_path, monkeypatch +): + monkeypatch.setattr(server.tempfile, "gettempdir", lambda: str(tmp_path / "tmp")) + monkeypatch.setattr(server, "owned", lambda path: False) + started: list[int] = [] + monkeypatch.setattr(relay, "start_daemon", lambda *a, **k: started.append(1)) + + with pytest.raises( + ConnectionError, match="is not this user's own; the relay forwards to this user's Raven daemon only" + ): + await relay.connect(tmp_path / "n", timeout_s=0.5) + + assert started == [] + assert (server.socket_path(tmp_path / "n").parent.stat().st_mode & 0o777) == 0o700, ( + "the socket's directory, made first" + ) + + async def test_the_relay_gives_up_when_no_daemon_comes_up(tmp_path, monkeypatch): monkeypatch.setattr(relay, "start_daemon", lambda *a, **k: None) diff --git a/tests/test_node_server.py b/tests/test_node_server.py index cabe357f1..fcc9eaaef 100644 --- a/tests/test_node_server.py +++ b/tests/test_node_server.py @@ -23,6 +23,9 @@ import pytest +if sys.platform == "win32": + pytest.skip("the daemon is POSIX: it locks with fcntl and listens on a unix socket", allow_module_level=True) + from raven.agent.tools.file_search import FindTool, GrepTool from raven.agent.tools.filesystem import ListDirTool, ReadFileTool from raven.node import __main__ as node_main @@ -341,6 +344,56 @@ def test_a_node_process_answers_until_its_stdin_closes(tree): assert "hay" in second["result"]["text"] +# A node with nothing importable beyond the standard library, its own code and +# the packages it is installed with: what the next change to raven/agent/tools +# must not break, since every installed node runs it. +_BUDGETED_NODE = """ +import sys +allowed = set(sys.stdlib_module_names) | set(sys.modules) | {"raven", *%r} + + +class _Budget: + def find_spec(self, name, path=None, target=None): + # The build's own sysconfig data is the standard library's, under a name of the platform's. + if name.partition(".")[0] in allowed or name.startswith("_sysconfigdata"): + return None + raise ImportError(f"{name} is not among a node's requirements") + + +sys.meta_path.insert(0, _Budget()) +from raven.node import server +sys.exit(server.run_stdio()) +""" +_MODULE_OF = {"pillow": "PIL"} + + +@pytest.mark.slow +def test_a_node_answers_every_file_call_with_only_its_requirements_importable(tree): + importable = tuple(_MODULE_OF.get(name, name) for name in protocol.REQUIREMENTS) + calls = [ + (protocol.READ, {"path": "b.md"}), + (protocol.LIST, {"path": "."}), + (protocol.FIND, {"pattern": "*.py", "path": "."}), + (protocol.GREP, {"pattern": "needle", "path": "."}), + (protocol.STAT, {"path": "b.md", "hash": True}), + (protocol.WRITE, {"path": "new.txt", "content": "x\n"}), + (protocol.EDIT, {"path": "new.txt", "old_text": "x", "new_text": "y"}), + (protocol.PULL, {"path": "b.md"}), + ] + requests = [{"jsonrpc": "2.0", "id": 1, "method": protocol.HELLO, "params": {"protocol": protocol.PROTOCOL}}] + requests += [ + {"jsonrpc": "2.0", "id": n, "method": method, "params": {**params, "roots": [str(tree)]}} + for n, (method, params) in enumerate(calls, start=2) + ] + + done = _node(["-c", _BUDGETED_NODE % (importable,)], requests, tree) + + assert done.returncode == 0, done.stderr + answers = [json.loads(line) for line in done.stdout.splitlines()] + assert [answer.get("error") for answer in answers] == [None] * len(requests), answers + assert answers[0]["result"]["digest"] == bundle.digest() + + @pytest.mark.slow def test_stdout_carries_only_frames_whatever_else_prints(tree): done = _node(["-c", _NOISY_NODE], [{"jsonrpc": "2.0", "id": 1, "method": "x/noisy"}], tree) @@ -385,6 +438,17 @@ def test_stdio_serves(monkeypatch): assert node_main.main(["--stdio"]) == 0 +def test_a_daemon_refused_its_socket_s_place_says_so_in_one_line_and_leaves(tmp_path, monkeypatch, capsys): + def refuse(root: Path, *, idle_s: float) -> int: + raise PermissionError(f"{root} is in the way of the daemon") + + monkeypatch.setattr("raven.node.server.run_daemon", refuse) + + assert node_main.main(["--serve", "--dir", str(tmp_path)]) == 1 + + assert f"raven node: {tmp_path} is in the way of the daemon" in capsys.readouterr().err + + @pytest.mark.parametrize("argv", [[], ["--stdio", "--version"]]) def test_a_node_is_started_for_exactly_one_purpose(argv, capsys): with pytest.raises(SystemExit) as caught: @@ -1016,6 +1080,30 @@ def test_sweep_drops_the_records_of_tasks_that_ended_long_ago_and_keeps_the_rest assert sorted(p.name for p in (tmp_path / "node" / server.TASKS_DIR).iterdir()) == ["t-recent", "t-still"] +@pytest.mark.slow +async def test_a_task_s_records_are_its_owner_s_alone(tmp_path, agent): + daemon = server.Daemon(tmp_path / "node") + conn = _Conn(daemon) + + started = await conn.ask( + protocol.AGENT_LAUNCH, {"channel": 1, "task": "t-own", "command": agent, "cwd": str(tmp_path / "w")} + ) + + try: + assert started["result"]["task"] == "t-own" + record = daemon.record_dir("t-own") + assert (tmp_path / "node" / server.TASKS_DIR).stat().st_mode & 0o777 == 0o700 + assert record.stat().st_mode & 0o777 == 0o700 + assert {p.name: p.stat().st_mode & 0o777 for p in record.iterdir()} == { + "task.json": 0o600, + "frames.jsonl": 0o600, + "stderr.log": 0o600, + }, "the frames hold the agent's output as it was" + finally: + daemon.tasks["t-own"].end() + await daemon.tasks["t-own"].proc.wait() + + @pytest.mark.slow async def test_shutdown_from_a_host_leaves_a_daemon_s_tasks_running(tmp_path, agent): daemon = server.Daemon(tmp_path / "node") @@ -1097,6 +1185,69 @@ async def test_a_second_daemon_under_the_same_directory_leaves_at_once(tmp_path, assert await asyncio.wait_for(first, timeout=10) == 0 +def test_the_socket_s_directory_is_this_user_s_own_or_is_not_used(tmp_path, monkeypatch): + monkeypatch.setattr(server.tempfile, "gettempdir", lambda: str(tmp_path / "tmp")) + assert server.socket_path(Path("/srv/node")) == Path("/srv/node") / server.RUN_DIR / f"{bundle.install_name()}.sock" + sock = server.socket_path(tmp_path / ("n" * 120)) + assert sock.parent == tmp_path / "tmp" / f"raven-node-{os.getuid()}" and sock.suffix == ".sock" + assert len(os.fsencode(sock)) < len(os.fsencode(tmp_path / ("n" * 120) / server.RUN_DIR)) + + assert server.own_dir(sock.parent) == sock.parent + assert sock.parent.stat().st_mode & 0o777 == 0o700 + os.chmod(sock.parent, 0o755) + server.own_dir(sock.parent) + assert sock.parent.stat().st_mode & 0o777 == 0o700, "this user's, with wider bits: narrowed" + + (tmp_path / "file").write_text("", encoding="utf-8") + (tmp_path / "real").mkdir() + (tmp_path / "link").symlink_to(tmp_path / "real") + for other in (tmp_path / "file", tmp_path / "link"): + with pytest.raises(PermissionError, match="is not a directory only this user may open"): + server.own_dir(other) + assert server.owned(tmp_path / "file") and server.owned(tmp_path / "nothing-there") + + me = os.getuid() + monkeypatch.setattr(server.os, "getuid", lambda: me + 1) + with pytest.raises(PermissionError, match="is not a directory only this user may open"): + server.own_dir(tmp_path / "real") + assert (tmp_path / "real").stat().st_mode & 0o777 == 0o755, "another user's: left as it is" + assert not server.owned(tmp_path / "file") and server.owned(tmp_path / "nothing-there") + + +@pytest.mark.slow +async def test_the_daemon_does_not_listen_beside_a_thing_it_cannot_remove_from_its_socket_s_place( + tmp_path, monkeypatch +): + monkeypatch.setattr(server.tempfile, "gettempdir", lambda: str(tmp_path / "tmp")) + root = tmp_path / "n" + sock = server.socket_path(root) + sock.parent.mkdir(parents=True, exist_ok=True) + sock.mkdir() + + with pytest.raises(PermissionError, match="is in the way of the daemon and cannot be removed"): + await server.run_daemon_async(root, idle_s=0.3) + + assert sock.is_dir() and not server.pid_path(root).exists() + + +async def test_a_whole_file_s_digest_is_taken_off_the_loop_the_heartbeat_is_answered_on(tree, monkeypatch): + digest = server._sha256 + + def slowly(path: Path) -> str: + time.sleep(0.4) + return digest(path) + + monkeypatch.setattr(server, "_sha256", slowly) + read = asyncio.create_task(_result(protocol.READ, {"path": str(tree / "b.md"), "roots": [str(tree)]})) + await asyncio.sleep(0.05) + before = time.monotonic() + + assert await _result(protocol.PING, {"t": 1}) == {"t": 1} + + assert time.monotonic() - before < 0.3, "answered while the digest was still being taken" + assert (await read)["sha256"] == hashlib.sha256(b"hay\n").hexdigest() + + @pytest.mark.parametrize("argv", [["--serve"], ["--attach"]]) def test_serve_and_attach_need_the_node_directory(argv, capsys): with pytest.raises(SystemExit): diff --git a/tests/test_ops_remote_path.py b/tests/test_ops_remote_path.py index d93d454ae..544670be1 100644 --- a/tests/test_ops_remote_path.py +++ b/tests/test_ops_remote_path.py @@ -112,13 +112,35 @@ def test_two_rows_on_one_address_and_port_are_named_by_id_instead(): @pytest.mark.parametrize( "text", - ["./a:b.txt", "/abs/a:b", "C:\\x", "c:/x", "C:", "http://x.org/a", "https://x.org/a:b", "~/x", "dir/a:b", ":x", ""], + [ + "./a:b.txt", + "/abs/a:b", + "C:\\x", + "c:/x", + "C:", + "\\\\?\\C:\\x\\review_marker.txt", + "\\\\server\\share\\f", + "http://x.org/a", + "https://x.org/a:b", + "~/x", + "dir/a:b", + ":x", + "", + ], ) def test_a_local_path_is_left_to_this_computer(text): assert not looks_remote(text) assert parse(text, ROWS) is None +def test_a_drive_relative_path_is_this_computer_s_on_a_windows_host_and_scp_s_elsewhere(monkeypatch): + monkeypatch.setattr(remote_path, "_WINDOWS", True) + assert not looks_remote("C:review_marker.txt") and parse("C:review_marker.txt", ROWS) is None + + monkeypatch.setattr(remote_path, "_WINDOWS", False) + assert looks_remote("c:review_marker.txt"), "what scp reads as machine c" + + # --- shown the way scp writes it ------------------------------------------------------ From a99db7ff7d90d5e076ea3fe0905eeb3ff0a36cb1 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:08:47 +0800 Subject: [PATCH 08/44] fix(agent): read a copy's program with Windows spellings; leave a Windows path form to this computer The copy refusal compared a POSIX basename with scp, sftp and rsync, so scp.exe, SCP.EXE and C:\Windows\System32\OpenSSH\scp.exe reached the executor and went around the file tools' budgets and read check: the program a word runs is read with the directory, the case and the suffix taken off, and on a Windows host the words are read with their backslashes kept, as the shell there reads them. A drive-relative or extended Windows path (C:file, \\?\C:\...) was refused by read_file as a machine nobody registered: it is this computer's, through the public tools too. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/machine_exec.py | 39 ++++++++++++++++++++----- tests/test_remote_file_tools.py | 12 ++++++++ tests/test_shell_machine_channel.py | 44 +++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/raven/agent/tools/machine_exec.py b/raven/agent/tools/machine_exec.py index 5efe27d29..1155d2fe3 100644 --- a/raven/agent/tools/machine_exec.py +++ b/raven/agent/tools/machine_exec.py @@ -45,9 +45,10 @@ from __future__ import annotations import asyncio +import os import re import shlex -from pathlib import PurePosixPath +from pathlib import PurePosixPath, PureWindowsPath from typing import Any # Long enough to tail a large log or hash a directory; far short of any solver @@ -102,6 +103,10 @@ def machines_registered() -> bool: # does to the command. A separator ends it, so the words after one belong to the # next command and not to this ssh. _COMMAND_SEPARATORS = frozenset({";", "&", "&&", "|", "||", "|&", "(", ")"}) +#: On a Windows host the shell that runs a command reads a backslash as a path's +#: own, not as an escape: the words are read that way too, so ``C:\...\scp.exe`` +#: keeps its name (reviewed 2026-10-12). +_WINDOWS = os.name == "nt" # A redirection does not end the command: `ssh 2>/dev/null -p 58717 root@host` # is a single ssh. Reading one as a terminator -- or, worse, reading `&>` as a @@ -155,6 +160,26 @@ def _read_option_group(word: str, tokens: list[str], index: int) -> tuple[int, s return index, None +#: What Windows appends to an executable's name, which names the same program. +_EXE_SUFFIXES = (".exe", ".cmd", ".bat") + + +def _program(word: str) -> str: + """The program a command word runs, as the refusals name one: ``ssh``, ``scp``, ``sftp``, ``rsync``. + + Written ``ssh``, ``/usr/bin/ssh`` or ``\\ssh`` (a backslash suppresses alias + lookup and still runs the client), and on Windows ``scp.exe``, ``SCP.EXE`` + or ``C:\\Windows\\System32\\OpenSSH\\scp.exe``, all of which reach the far + side: the directory, the case and the suffix are taken off (reviewed + 2026-10-12: the three Windows spellings went past the copy refusal). + """ + name = PureWindowsPath(PurePosixPath(word.strip("\"'").lstrip("\\")).name).name.lower() + for suffix in _EXE_SUFFIXES: + if name.endswith(suffix): + return name[: -len(suffix)] + return name + + def _ssh_destinations(command: str) -> list[tuple[str, int]]: """Every host and port an ``ssh`` word in a shell command would connect to. @@ -184,7 +209,7 @@ def _ssh_destinations(command: str) -> list[tuple[str, int]]: true -p 58717`` is 22) or a ``--``. """ try: - lexer = shlex.shlex(command, posix=True, punctuation_chars=True) + lexer = shlex.shlex(command, posix=not _WINDOWS, punctuation_chars=True) lexer.whitespace_split = True tokens = list(lexer) except ValueError: @@ -195,7 +220,7 @@ def _ssh_destinations(command: str) -> list[tuple[str, int]]: found: list[tuple[str, int]] = [] index = 0 while index < len(tokens): - if PurePosixPath(tokens[index].lstrip("\\")).name != "ssh": + if _program(tokens[index]) != "ssh": index += 1 continue index += 1 @@ -206,7 +231,7 @@ def _ssh_destinations(command: str) -> list[tuple[str, int]]: options_ended = False while index < len(tokens): word = tokens[index] - if word in _COMMAND_SEPARATORS or PurePosixPath(word.lstrip("\\")).name == "ssh": + if word in _COMMAND_SEPARATORS or _program(word) == "ssh": # The command ended, or the next one began; either way this # ssh's arguments are over and the token is left for the outer # loop to read. @@ -359,7 +384,7 @@ def _copy_destinations(command: str) -> list[tuple[str, int]]: remote path names the host, and all of one command's share its port. """ try: - lexer = shlex.shlex(command, posix=True, punctuation_chars=True) + lexer = shlex.shlex(command, posix=not _WINDOWS, punctuation_chars=True) lexer.whitespace_split = True tokens = list(lexer) except ValueError: @@ -367,7 +392,7 @@ def _copy_destinations(command: str) -> list[tuple[str, int]]: found: list[tuple[str, int]] = [] index = 0 while index < len(tokens): - program = PurePosixPath(tokens[index].lstrip("\\")).name + program = _program(tokens[index]) if program not in _COPIERS: index += 1 continue @@ -377,7 +402,7 @@ def _copy_destinations(command: str) -> list[tuple[str, int]]: options_ended = False while index < len(tokens): word = tokens[index] - if word in _COMMAND_SEPARATORS or PurePosixPath(word.lstrip("\\")).name in _COPIERS | {"ssh"}: + if word in _COMMAND_SEPARATORS or _program(word) in _COPIERS | {"ssh"}: break following = tokens[index + 1] if index + 1 < len(tokens) else None if word in _REDIRECTIONS: diff --git a/tests/test_remote_file_tools.py b/tests/test_remote_file_tools.py index c6ab51917..54fd464c7 100644 --- a/tests/test_remote_file_tools.py +++ b/tests/test_remote_file_tools.py @@ -218,6 +218,18 @@ async def test_a_local_path_is_read_here_as_before(registry, pool, tmp_path): assert pool.calls == [] +@pytest.mark.parametrize("path", ["\\\\?\\C:\\work\\review_marker.txt", "C:review_marker.txt", "\\\\box\\share\\f"]) +async def test_a_windows_path_form_is_this_computer_s_not_a_machine_s(registry, pool, tmp_path, monkeypatch, path): + # Drive-relative and extended paths, as a Windows host writes them (reviewed 2026-10-12): the lane + # on, each is left to this computer's tool rather than refused as a machine nobody registered. + monkeypatch.setattr(remote_path, "_WINDOWS", True) + + got = await _tools(tmp_path)["read_file"].execute(path) + + assert "registered machine" not in got and "machine" not in got.lower().split("error")[-1][:40], got + assert pool.calls == [] + + # --- the real node, behind a stand-in ssh ---------------------------------------------------- diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index 61f17cb5a..02d3f44a8 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -826,6 +826,9 @@ def served() -> dict: "rsync -av --port 58717 ./dir rsync://203.0.113.7/module", "scp scp://root@203.0.113.7:58717//tmp/a .", "true && \\scp -P 58717 a root@203.0.113.7:b", + "scp.exe -P 58717 a.txt root@203.0.113.7:/tmp/", + "SCP.EXE -P 58717 a.txt root@203.0.113.7:/tmp/", + "sftp.exe -P 58717 root@203.0.113.7", ], ) async def test_a_copy_to_a_registered_machine_is_refused_and_points_at_the_file_tools(registry, tmp_path, command): @@ -867,6 +870,47 @@ def test_copy_destinations_are_read_the_way_the_programs_read_them(command, expe assert machine_exec._copy_destinations(command) == expected +@pytest.mark.parametrize( + ("word", "program"), + [ + ("ssh", "ssh"), + ("/usr/bin/ssh", "ssh"), + ("\\ssh", "ssh"), + ("scp.exe", "scp"), + ("SCP.EXE", "scp"), + ("C:\\Windows\\System32\\OpenSSH\\scp.exe", "scp"), + ("rsync.cmd", "rsync"), + ("scpx", "scpx"), + ("my-scp.exe.bak", "my-scp.exe.bak"), + ], +) +def test_the_program_a_word_runs_is_read_with_windows_spellings_too(word, program): + assert machine_exec._program(word) == program + + +@pytest.mark.parametrize( + "command", + [ + "C:\\Windows\\System32\\OpenSSH\\scp.exe -P 58717 a.txt root@203.0.113.7:/tmp/", + '"C:\\Program Files\\OpenSSH\\scp.exe" -P 58717 a.txt root@203.0.113.7:/tmp/', + "C:\\Windows\\System32\\OpenSSH\\ssh.exe -p 58717 root@203.0.113.7 ls", + ], +) +async def test_on_a_windows_host_a_program_s_whole_path_keeps_its_backslashes_and_its_name( + registry, tmp_path, monkeypatch, command +): + # The shell there reads a backslash as a path's own; read as POSIX escapes, the name was lost. + monkeypatch.setattr(machine_exec, "_WINDOWS", True) + tool = ExecTool(working_dir=str(tmp_path)) + marker = tmp_path / "ran" + + out = await tool.execute(command=f"{command}; touch {marker}") + + assert out.startswith("Error:") and "the way scp writes it" in out and "Nothing was run." in out + assert "203.0.113.7" not in out.replace("user@host", "") + assert not marker.exists() + + async def test_a_copy_to_the_same_address_on_another_port_is_another_machine(registry): assert machine_exec.raw_copy_target("scp a.txt root@203.0.113.7:/tmp/") is None assert machine_exec.raw_copy_target("scp -P 58717 a.txt root@203.0.113.7:/tmp/") == dict(ROW) From 3b790620c54a4828b1067cc957e961668e2591c7 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:32:23 +0800 Subject: [PATCH 09/44] fix(node): open a record the way the type check reads; the ownership test sets its own mode The record opener handed os.fdopen a splatted keyword map, which no overload of it matches under the type check: the binary and the text mode are two calls now. The test of another user's directory asserted the mode mkdir leaves, which is the umask's (0775 under 0002, where it failed): the test sets 0755 first and asserts it is left so. Co-authored-by: Claude (claude-fable-5-1) --- raven/node/server.py | 5 ++++- tests/test_node_server.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/raven/node/server.py b/raven/node/server.py index 773c28ab9..905cc9d3c 100644 --- a/raven/node/server.py +++ b/raven/node/server.py @@ -611,7 +611,10 @@ def own_file(path: Path, mode: str = "ab") -> Any: the host's own journal is. """ flags = os.O_WRONLY | os.O_CREAT | (os.O_APPEND if "a" in mode else os.O_TRUNC) - return os.fdopen(os.open(path, flags, 0o600), mode, **({} if "b" in mode else {"encoding": "utf-8"})) + fd = os.open(path, flags, 0o600) + if "b" in mode: + return os.fdopen(fd, mode) + return os.fdopen(fd, mode, encoding="utf-8") def pid_path(root: Path) -> Path: diff --git a/tests/test_node_server.py b/tests/test_node_server.py index fcc9eaaef..6f7b60630 100644 --- a/tests/test_node_server.py +++ b/tests/test_node_server.py @@ -1200,6 +1200,7 @@ def test_the_socket_s_directory_is_this_user_s_own_or_is_not_used(tmp_path, monk (tmp_path / "file").write_text("", encoding="utf-8") (tmp_path / "real").mkdir() + os.chmod(tmp_path / "real", 0o755) # an explicit mode: mkdir's is whatever the umask leaves (tmp_path / "link").symlink_to(tmp_path / "real") for other in (tmp_path / "file", tmp_path / "link"): with pytest.raises(PermissionError, match="is not a directory only this user may open"): From e514df13230351e0d15f7ca1d6fd1f84dbc2b191 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:09:08 +0800 Subject: [PATCH 10/44] fix(ops): the colon-in-a-name test names a two-letter host, which no host reads as a drive On a Windows host one letter before the colon is a drive, so `a:b.txt` is this computer's there and the test that expects a refusal failed (reviewed 2026-10-12); a two-letter name reads as a machine's everywhere. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_ops_remote_path.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_ops_remote_path.py b/tests/test_ops_remote_path.py index 544670be1..f03e7c24d 100644 --- a/tests/test_ops_remote_path.py +++ b/tests/test_ops_remote_path.py @@ -93,8 +93,9 @@ def test_a_user_and_a_machine_id_are_compared_as_written(): ("scp://[2001:db8::5//x", "opens an address with '['"), ("scp://[2001:db8::5]x//x", "something after the address that is not a port"), ("nobox:/x", "write it as ./name or as an absolute path"), - # C27: a local name holding a colon reads as remote unless written as a path. - ("a:b.txt", "write it as ./name or as an absolute path"), + # C27: a local name holding a colon reads as remote unless written as a path + # (one letter before the colon is a drive on a Windows host: the drive-relative test). + ("ab:b.txt", "write it as ./name or as an absolute path"), ], ) def test_a_path_that_names_no_registered_machine_is_refused_before_any_ssh(text, said): From a010ad8d2e2e6ce3a899844ab89884058b14388b Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:10:32 +0800 Subject: [PATCH 11/44] fix(agent): a fetched file lands by names this computer keeps; a quoted word reads bare on Windows A fetched file landed under its path on the machine joined as this computer's path, and a Linux file name may hold backslashes: on a Windows host `..\` inside one name read as two steps up, and the file landed above the workspace (reviewed 2026-10-12). The path is read as POSIX now, each component written as one name here (the other platform's separator, what Windows reserves and control characters as their percent-codes), and a landing that would not resolve under the machine's delivery directory is refused before anything is made. On a Windows host the words of a command kept their quotes, so a quoted destination or port went unmatched and the command reached the executor: the quotes that shell takes off are taken off before the match. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/deliver.py | 47 ++++++++++++++++++++++++++--- raven/agent/tools/machine_exec.py | 44 +++++++++++++++++++-------- tests/test_deliver_files_tool.py | 28 +++++++++++++++++ tests/test_shell_machine_channel.py | 44 +++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 16 deletions(-) diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index 65cb8056e..1ae533cb2 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -3,10 +3,11 @@ from __future__ import annotations import mimetypes +import re import time from contextvars import ContextVar from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from urllib.parse import quote @@ -71,8 +72,10 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path a way around the read check, and what another conversation was allowed to read is no evidence for this one. A file elsewhere is read there. One over :data:`PULL_FILE_BUDGET` stays, and is named with its size. The file - lands under its whole path on the machine, so two files of one name from - two directories never land on each other. + lands under its whole path on the machine, each component written as one + name here (:func:`_landing`), so two files of one name from two + directories never land on each other and no name reaches above the + delivery directory. """ pool = remote_path.nodes() label = remote_path._label(where._row) @@ -97,7 +100,13 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path f"{_human_size(size)} is over the {_human_size(PULL_FILE_BUDGET)} " f"limit for a file brought back from machine {label}; read it there instead" ) - local = (workspace or Path.cwd()) / "pulled" / where.machine / Path(place).relative_to("/") + try: + local = _landing(workspace, where.machine, place) + except ValueError: + return None, ( + f"{where.display(place)} has a name this computer cannot keep under its delivery directory; " + "read it there instead" + ) await pool.pull_file(where.machine, place, local, roots=fence) except remote_path.NodeCallError as exc: return None, str(exc) @@ -109,6 +118,36 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path #: The most a file fetched from a registered machine for a delivery may weigh; a bigger one is read there. PULL_FILE_BUDGET = 20 * 1024 * 1024 +#: What a component of a fetched file's path may not hold as a name on this +#: computer: the other platform's separator, what Windows reserves in a name, +#: a control character. A Linux file name may hold any of them, and ``..\`` +#: inside one, read by a Windows host as two steps up, put a fetched file +#: above the workspace (reviewed 2026-10-12). Each is written as its +#: percent-code, the way a URL writes one. +_NOT_A_NAME_HERE = re.compile(r'[\\<>:"|?*\x00-\x1f]') + + +def _local_name(part: str) -> str: + """One component of a path on the machine, as one name on this computer's filesystem.""" + if part in (".", ".."): + return part.replace(".", "%2E") + return _NOT_A_NAME_HERE.sub(lambda m: f"%{ord(m.group(0)):02X}", part) + + +def _landing(workspace: Path | None, machine: str, place: str) -> Path: + """Where ``place`` on ``machine`` lands: under ``pulled//``, by its POSIX components, each one name here. + + Checked before anything is made or written: a landing that resolves + outside its machine's delivery directory is refused, whatever the name + did to get there. + """ + pulled = (workspace or Path.cwd()) / "pulled" / _local_name(machine) + parts = [_local_name(part) for part in PurePosixPath(place).parts if part != "/"] + local = pulled.joinpath(*parts) + if not parts or local.resolve() == pulled.resolve() or not local.resolve().is_relative_to(pulled.resolve()): + raise ValueError(f"{place} would not land under {pulled}") + return local + def _human_size(size: int) -> str: value = float(size) diff --git a/raven/agent/tools/machine_exec.py b/raven/agent/tools/machine_exec.py index 1155d2fe3..c2eeb0385 100644 --- a/raven/agent/tools/machine_exec.py +++ b/raven/agent/tools/machine_exec.py @@ -108,6 +108,34 @@ def machines_registered() -> bool: #: keeps its name (reviewed 2026-10-12). _WINDOWS = os.name == "nt" + +def _words(command: str) -> list[str]: + """The command's words as the shell that runs it reads them; none for a line it cannot read. + + POSIX quoting here. On a Windows host a backslash is a path's own and is + kept, and the quotes around a word, or inside one, are taken off the way + that shell takes them, so a quoted destination or port is matched like a + bare one (reviewed 2026-10-12: ``"root@host"`` and ``-P "58717"`` went + past the refusals with their quotes on). An unbalanced quote is not a + line this can read; the shell rejects it too, so nothing reaches a + machine either way. + """ + try: + lexer = shlex.shlex(command, posix=not _WINDOWS, punctuation_chars=True) + lexer.whitespace_split = True + words = list(lexer) + except ValueError: + return [] + return [_unquoted(word) for word in words] if _WINDOWS else words + + +def _unquoted(word: str) -> str: + """``word`` without the quotes a Windows shell takes off: a pair around it, or double quotes inside it.""" + if len(word) >= 2 and word[0] == word[-1] and word[0] in "\"'": + return word[1:-1] + return word.replace('"', "") + + # A redirection does not end the command: `ssh 2>/dev/null -p 58717 root@host` # is a single ssh. Reading one as a terminator -- or, worse, reading `&>` as a # word and taking it for the destination -- lost the registered host that came @@ -208,13 +236,8 @@ def _ssh_destinations(command: str) -> list[tuple[str, int]]: not an option (``ssh root@h -p 58717 true`` is port 58717, ``ssh root@h true -p 58717`` is 22) or a ``--``. """ - try: - lexer = shlex.shlex(command, posix=not _WINDOWS, punctuation_chars=True) - lexer.whitespace_split = True - tokens = list(lexer) - except ValueError: - # An unbalanced quote is not a shell line this can read; the shell will - # reject it too, so nothing reaches a machine either way. + tokens = _words(command) + if not tokens: return [] found: list[tuple[str, int]] = [] @@ -383,11 +406,8 @@ def _copy_destinations(command: str) -> list[tuple[str, int]]: or the ssh its ``-e``/``--rsh`` names, else 22; every operand written as a remote path names the host, and all of one command's share its port. """ - try: - lexer = shlex.shlex(command, posix=not _WINDOWS, punctuation_chars=True) - lexer.whitespace_split = True - tokens = list(lexer) - except ValueError: + tokens = _words(command) + if not tokens: return [] found: list[tuple[str, int]] = [] index = 0 diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 5861f348f..49d766df1 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -421,6 +421,34 @@ async def test_a_file_on_a_machine_is_brought_back_and_delivered(tool, tmp_path, assert machines.pulled == [("/root/raven-work/t1/report.pdf", landed, ["/root/raven-work"])] +async def test_a_name_holding_the_other_platform_s_separators_lands_as_one_name_under_the_delivery_directory( + tool, tmp_path, machines +) -> None: + # A Linux file name may hold backslashes, which a Windows host reads as steps up (reviewed 2026-10-12). + summary = await tool.execute(files=[{"path": "box:raven-work/t1/..\\..\\..\\review_escape.txt"}]) + + manifest = tool.take_metadata()["raven_delivery"] + assert manifest["invalid"] == [], summary + (landed,) = (Path(entry["path"]) for entry in manifest["files"]) + pulled = tmp_path / "chanwork" / "pulled" / "box" + assert landed == pulled / "root" / "raven-work" / "t1" / "..%5C..%5C..%5Creview_escape.txt" + assert landed.is_file() and not list(tmp_path.glob("**/review_escape.txt")) + assert machines.pulled[0][0] == "/root/raven-work/t1/..\\..\\..\\review_escape.txt" + + +async def test_nothing_is_fetched_to_a_landing_outside_the_delivery_directory( + tool, tmp_path, machines, monkeypatch +) -> None: + from raven.agent.tools import deliver + + monkeypatch.setattr(deliver, "_local_name", lambda part: "..") + + summary = await tool.execute(files=[{"path": "box:raven-work/t1/report.pdf"}]) + + assert "has a name this computer cannot keep under its delivery directory; read it there instead" in summary + assert machines.pulled == [] and not (tmp_path / "chanwork" / "pulled").exists() + + async def test_two_files_of_one_name_from_two_directories_land_apart(tool, tmp_path, machines) -> None: summary = await tool.execute( files=[{"path": "box:raven-work/a/report.pdf"}, {"path": "box:raven-work/b/report.pdf"}] diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index 02d3f44a8..6e0d19e3a 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -911,6 +911,50 @@ async def test_on_a_windows_host_a_program_s_whole_path_keeps_its_backslashes_an assert not marker.exists() +@pytest.mark.parametrize( + "command", + [ + 'ssh -p 58717 "root@203.0.113.7" true', + 'scp -P "58717" a.txt root@203.0.113.7:/tmp/a', + 'sftp -P "58717" "root@203.0.113.7"', + "rsync -av -e 'ssh -p 58717' ./dir \"root@203.0.113.7:/data/\"", + 'ssh -p"58717" root@203.0.113.7 true', + ], +) +async def test_on_a_windows_host_a_quoted_destination_or_port_is_matched_like_a_bare_one( + registry, tmp_path, monkeypatch, command +): + # That shell takes the quotes off before the program sees the word; read with them on, the + # host and the port went unmatched and the command reached the executor (reviewed 2026-10-12). + monkeypatch.setattr(machine_exec, "_WINDOWS", True) + tool = ExecTool(working_dir=str(tmp_path)) + marker = tmp_path / "ran" + + out = await tool.execute(command=f"{command}; touch {marker}") + + assert out.startswith("Error:") and "the way scp writes it" in out and "Nothing was run." in out + assert "203.0.113.7" not in out.replace("user@host", "") + assert not marker.exists() + + +def test_on_a_windows_host_the_words_are_read_without_their_quotes(monkeypatch): + monkeypatch.setattr(machine_exec, "_WINDOWS", True) + assert machine_exec._words('scp -P "58717" a.txt "root@h:/x" && -p"2222" \'two words\'') == [ + "scp", + "-P", + "58717", + "a.txt", + "root@h:/x", + "&&", + "-p2222", + "two words", + ] + assert machine_exec._words("scp 'unbalanced") == [] + + monkeypatch.setattr(machine_exec, "_WINDOWS", False) + assert machine_exec._words('scp -P "58717" a.txt "root@h:/x"') == ["scp", "-P", "58717", "a.txt", "root@h:/x"] + + async def test_a_copy_to_the_same_address_on_another_port_is_another_machine(registry): assert machine_exec.raw_copy_target("scp a.txt root@203.0.113.7:/tmp/") is None assert machine_exec.raw_copy_target("scp -P 58717 a.txt root@203.0.113.7:/tmp/") == dict(ROW) From 598cc8dfa487598ad20c297e0b153bfa064ffe14 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:52:15 +0800 Subject: [PATCH 12/44] fix(agent): encode a fetched name's percent sign too; a quoted piece of a Windows word stays in it A fetched name's backslash was written as its percent-code and a percent sign was not, so `a\b.txt` and `a%5Cb.txt` landed as one file (reviewed 2026-10-12): the percent sign is written out too, and no two names land as one. On a Windows host the words of a command were read with the quotes kept and taken off afterwards, so a quoted piece next to a bare one -- `"root@"host`, `"587"17` -- became two words and the host or port went unmatched: the words are read with POSIX quoting, pieces joined as that shell's argv has them, and only the backslash kept as a path's own. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/deliver.py | 6 ++++-- raven/agent/tools/machine_exec.py | 27 ++++++++++----------------- tests/test_deliver_files_tool.py | 18 ++++++++++++++++++ tests/test_shell_machine_channel.py | 16 ++++++++++++++++ 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index 1ae533cb2..d47abf05b 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -123,8 +123,10 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path #: a control character. A Linux file name may hold any of them, and ``..\`` #: inside one, read by a Windows host as two steps up, put a fetched file #: above the workspace (reviewed 2026-10-12). Each is written as its -#: percent-code, the way a URL writes one. -_NOT_A_NAME_HERE = re.compile(r'[\\<>:"|?*\x00-\x1f]') +#: percent-code, the way a URL writes one -- the percent sign itself too, so +#: two names that differ only by one written out never land as one +#: (``a\b`` and ``a%5Cb``; reviewed 2026-10-12). +_NOT_A_NAME_HERE = re.compile(r'[%\\<>:"|?*\x00-\x1f]') def _local_name(part: str) -> str: diff --git a/raven/agent/tools/machine_exec.py b/raven/agent/tools/machine_exec.py index c2eeb0385..05f080f92 100644 --- a/raven/agent/tools/machine_exec.py +++ b/raven/agent/tools/machine_exec.py @@ -112,28 +112,21 @@ def machines_registered() -> bool: def _words(command: str) -> list[str]: """The command's words as the shell that runs it reads them; none for a line it cannot read. - POSIX quoting here. On a Windows host a backslash is a path's own and is - kept, and the quotes around a word, or inside one, are taken off the way - that shell takes them, so a quoted destination or port is matched like a - bare one (reviewed 2026-10-12: ``"root@host"`` and ``-P "58717"`` went - past the refusals with their quotes on). An unbalanced quote is not a - line this can read; the shell rejects it too, so nothing reaches a - machine either way. + POSIX quoting here. On a Windows host the quotes come off the same way + -- around a word or inside it, a quoted piece and a bare one next to it + making one word, as that shell's argv has them (``"root@"host``, + ``"587"17``; reviewed 2026-10-12) -- but a backslash is a path's own and + is kept, never an escape. An unbalanced quote is not a line this can + read; the shell rejects it too, so nothing reaches a machine either way. """ try: - lexer = shlex.shlex(command, posix=not _WINDOWS, punctuation_chars=True) + lexer = shlex.shlex(command, posix=True, punctuation_chars=True) lexer.whitespace_split = True - words = list(lexer) + if _WINDOWS: + lexer.escape = "" + return list(lexer) except ValueError: return [] - return [_unquoted(word) for word in words] if _WINDOWS else words - - -def _unquoted(word: str) -> str: - """``word`` without the quotes a Windows shell takes off: a pair around it, or double quotes inside it.""" - if len(word) >= 2 and word[0] == word[-1] and word[0] in "\"'": - return word[1:-1] - return word.replace('"', "") # A redirection does not end the command: `ssh 2>/dev/null -p 58717 root@host` diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 49d766df1..844f749c5 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -436,6 +436,24 @@ async def test_a_name_holding_the_other_platform_s_separators_lands_as_one_name_ assert machines.pulled[0][0] == "/root/raven-work/t1/..\\..\\..\\review_escape.txt" +async def test_two_names_that_differ_only_by_one_written_out_land_apart(tool, tmp_path, machines) -> None: + # `a\b.txt` is written `a%5Cb.txt` here, so a file named `a%5Cb.txt` must not land on it (reviewed 2026-10-12). + summary = await tool.execute( + files=[{"path": "box:raven-work/t1/a\\b.txt"}, {"path": "box:raven-work/t1/a%5Cb.txt"}] + ) + + manifest = tool.take_metadata()["raven_delivery"] + assert manifest["invalid"] == [], summary + landed = [Path(entry["path"]) for entry in manifest["files"]] + pulled = tmp_path / "chanwork" / "pulled" / "box" / "root" / "raven-work" / "t1" + assert landed == [pulled / "a%5Cb.txt", pulled / "a%255Cb.txt"] + assert all(place.is_file() for place in landed) + assert [where for where, _, _ in machines.pulled] == [ + "/root/raven-work/t1/a\\b.txt", + "/root/raven-work/t1/a%5Cb.txt", + ] + + async def test_nothing_is_fetched_to_a_landing_outside_the_delivery_directory( tool, tmp_path, machines, monkeypatch ) -> None: diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index 6e0d19e3a..c7e262190 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -919,6 +919,9 @@ async def test_on_a_windows_host_a_program_s_whole_path_keeps_its_backslashes_an 'sftp -P "58717" "root@203.0.113.7"', "rsync -av -e 'ssh -p 58717' ./dir \"root@203.0.113.7:/data/\"", 'ssh -p"58717" root@203.0.113.7 true', + 'ssh -p 58717 "root@"203.0.113.7 true', + 'ssh -p "587"17 root@203.0.113.7 true', + 'scp -P 58717 a.txt "root@203.0.113.7:"/tmp/a', ], ) async def test_on_a_windows_host_a_quoted_destination_or_port_is_matched_like_a_bare_one( @@ -949,6 +952,19 @@ def test_on_a_windows_host_the_words_are_read_without_their_quotes(monkeypatch): "-p2222", "two words", ] + # A quoted piece and a bare one next to it are one word, as that shell's argv has them; a backslash is kept. + assert machine_exec._words('ssh -p "587"17 "root@"h true 2>&1 C:\\dir\\scp.exe "C:\\Program Files\\x"') == [ + "ssh", + "-p", + "58717", + "root@h", + "true", + "2", + ">&", + "1", + "C:\\dir\\scp.exe", + "C:\\Program Files\\x", + ] assert machine_exec._words("scp 'unbalanced") == [] monkeypatch.setattr(machine_exec, "_WINDOWS", False) From 2e9f7dd1105da5ce715cef3b41209cba3dada649 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:52:41 +0800 Subject: [PATCH 13/44] fix(node): the reconnect test finds the attach among the new link's requests, not as its last The test of a link that comes back read the new link's last request as the attach, and its heartbeat, every 10 ms there, can follow the attach before the check: it now looks for the attach among the requests (reproduced on Windows, 2026-10-12). Co-authored-by: Claude (claude-fable-5-1) --- tests/test_node_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_node_client.py b/tests/test_node_client.py index 6de917c63..334f69b7f 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -898,7 +898,9 @@ async def launch(machine_id: str) -> _Node: ) await _until(lambda: len(tried) == 2 and pool.live() == ["box"] and endpoint.detached is None) again = pool._nodes["box"].client - assert again.asked[-1][:2] == (protocol.AGENT_ATTACH, {"channel": 1, "task": endpoint.task, "after": 0}) + # Among the new link's requests, not its last: a heartbeat may well follow the attach (reviewed 2026-10-12). + attach = (protocol.AGENT_ATTACH, {"channel": 1, "task": endpoint.task, "after": 0}) + assert attach in [asked[:2] for asked in again.asked] assert pool.status("box").state == "connected" and pool._nodes["box"].channels[1] is endpoint await pool.close_all() From ba86d7edf6f3f52a5627db3fc391d11ea4a3adae Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:59:12 +0800 Subject: [PATCH 14/44] chore(agent): say what a comment knows without the day it was reviewed A review date in a source comment is the task's context, not the code's: the five comments that carried one keep their reason and drop the day. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/deliver.py | 4 ++-- raven/agent/tools/machine_exec.py | 8 ++++---- tests/test_deliver_files_tool.py | 4 ++-- tests/test_remote_file_tools.py | 2 +- tests/test_shell_machine_channel.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index d47abf05b..d9ba80b50 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -122,10 +122,10 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path #: computer: the other platform's separator, what Windows reserves in a name, #: a control character. A Linux file name may hold any of them, and ``..\`` #: inside one, read by a Windows host as two steps up, put a fetched file -#: above the workspace (reviewed 2026-10-12). Each is written as its +#: above the workspace. Each is written as its #: percent-code, the way a URL writes one -- the percent sign itself too, so #: two names that differ only by one written out never land as one -#: (``a\b`` and ``a%5Cb``; reviewed 2026-10-12). +#: (``a\b`` and ``a%5Cb``). _NOT_A_NAME_HERE = re.compile(r'[%\\<>:"|?*\x00-\x1f]') diff --git a/raven/agent/tools/machine_exec.py b/raven/agent/tools/machine_exec.py index 05f080f92..775b5f793 100644 --- a/raven/agent/tools/machine_exec.py +++ b/raven/agent/tools/machine_exec.py @@ -105,7 +105,7 @@ def machines_registered() -> bool: _COMMAND_SEPARATORS = frozenset({";", "&", "&&", "|", "||", "|&", "(", ")"}) #: On a Windows host the shell that runs a command reads a backslash as a path's #: own, not as an escape: the words are read that way too, so ``C:\...\scp.exe`` -#: keeps its name (reviewed 2026-10-12). +#: keeps its name. _WINDOWS = os.name == "nt" @@ -115,7 +115,7 @@ def _words(command: str) -> list[str]: POSIX quoting here. On a Windows host the quotes come off the same way -- around a word or inside it, a quoted piece and a bare one next to it making one word, as that shell's argv has them (``"root@"host``, - ``"587"17``; reviewed 2026-10-12) -- but a backslash is a path's own and + ``"587"17``) -- but a backslash is a path's own and is kept, never an escape. An unbalanced quote is not a line this can read; the shell rejects it too, so nothing reaches a machine either way. """ @@ -191,8 +191,8 @@ def _program(word: str) -> str: Written ``ssh``, ``/usr/bin/ssh`` or ``\\ssh`` (a backslash suppresses alias lookup and still runs the client), and on Windows ``scp.exe``, ``SCP.EXE`` or ``C:\\Windows\\System32\\OpenSSH\\scp.exe``, all of which reach the far - side: the directory, the case and the suffix are taken off (reviewed - 2026-10-12: the three Windows spellings went past the copy refusal). + side: the directory, the case and the suffix are taken off (before + this, the three Windows spellings went past the copy refusal). """ name = PureWindowsPath(PurePosixPath(word.strip("\"'").lstrip("\\")).name).name.lower() for suffix in _EXE_SUFFIXES: diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 844f749c5..5bf5f0317 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -424,7 +424,7 @@ async def test_a_file_on_a_machine_is_brought_back_and_delivered(tool, tmp_path, async def test_a_name_holding_the_other_platform_s_separators_lands_as_one_name_under_the_delivery_directory( tool, tmp_path, machines ) -> None: - # A Linux file name may hold backslashes, which a Windows host reads as steps up (reviewed 2026-10-12). + # A Linux file name may hold backslashes, which a Windows host reads as steps up. summary = await tool.execute(files=[{"path": "box:raven-work/t1/..\\..\\..\\review_escape.txt"}]) manifest = tool.take_metadata()["raven_delivery"] @@ -437,7 +437,7 @@ async def test_a_name_holding_the_other_platform_s_separators_lands_as_one_name_ async def test_two_names_that_differ_only_by_one_written_out_land_apart(tool, tmp_path, machines) -> None: - # `a\b.txt` is written `a%5Cb.txt` here, so a file named `a%5Cb.txt` must not land on it (reviewed 2026-10-12). + # `a\b.txt` is written `a%5Cb.txt` here, so a file named `a%5Cb.txt` must not land on it. summary = await tool.execute( files=[{"path": "box:raven-work/t1/a\\b.txt"}, {"path": "box:raven-work/t1/a%5Cb.txt"}] ) diff --git a/tests/test_remote_file_tools.py b/tests/test_remote_file_tools.py index 54fd464c7..d14d1597d 100644 --- a/tests/test_remote_file_tools.py +++ b/tests/test_remote_file_tools.py @@ -220,7 +220,7 @@ async def test_a_local_path_is_read_here_as_before(registry, pool, tmp_path): @pytest.mark.parametrize("path", ["\\\\?\\C:\\work\\review_marker.txt", "C:review_marker.txt", "\\\\box\\share\\f"]) async def test_a_windows_path_form_is_this_computer_s_not_a_machine_s(registry, pool, tmp_path, monkeypatch, path): - # Drive-relative and extended paths, as a Windows host writes them (reviewed 2026-10-12): the lane + # Drive-relative and extended paths, as a Windows host writes them: the lane # on, each is left to this computer's tool rather than refused as a machine nobody registered. monkeypatch.setattr(remote_path, "_WINDOWS", True) diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index c7e262190..17cdcfb29 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -928,7 +928,7 @@ async def test_on_a_windows_host_a_quoted_destination_or_port_is_matched_like_a_ registry, tmp_path, monkeypatch, command ): # That shell takes the quotes off before the program sees the word; read with them on, the - # host and the port went unmatched and the command reached the executor (reviewed 2026-10-12). + # host and the port went unmatched and the command reached the executor. monkeypatch.setattr(machine_exec, "_WINDOWS", True) tool = ExecTool(working_dir=str(tmp_path)) marker = tmp_path / "ran" From 4ce6e585ad6545a5c6992789e1a58c3edef5ce9f Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:59:13 +0800 Subject: [PATCH 15/44] chore(*): say what a comment knows without the day it was reviewed A review date in a source comment is the task's context, not the code's: the four comments that carried one keep their reason and drop the day. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/client.py | 2 +- raven/acp_client/remote.py | 2 +- raven/node/client.py | 2 +- tests/test_node_client.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/raven/acp_client/client.py b/raven/acp_client/client.py index 02a018cc1..b519ed035 100644 --- a/raven/acp_client/client.py +++ b/raven/acp_client/client.py @@ -355,7 +355,7 @@ async def launch( it that is a command for another machine keeps that machine's quoting, which this host's grammar would read apart (a Windows host's parser made twelve words of a node's one remote-command word while it - went as a line; reviewed 2026-10-12). + went as a line). Raises :class:`AcpConnectionError` if the process cannot be started; a server that starts but never speaks is a per-request timeout, not a diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index e122d6e91..682dec0c3 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -174,7 +174,7 @@ def ssh_command( it is, so no host's own grammar reads it again: the one word that is the remote command keeps the POSIX quoting the far shell reads, whatever this host is (a Windows host's parser made twelve words of it while it went as - a line; reviewed 2026-10-12). Built at launch and held in memory only: it + a line). Built at launch and held in memory only: it carries the address. ``command`` is sent through :func:`posix`, so the machine user's login shell may be any shell; it carries no variable and reads no login profile -- the node captures the login diff --git a/raven/node/client.py b/raven/node/client.py index ec7f6c987..1f6cbda41 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -576,7 +576,7 @@ async def _launch(self, machine_id: str) -> _Node: client = await AcpClient.launch( name=f"raven-node@{target.id}", # The argv itself: a line would be read by this host's own - # grammar, which is not the far shell's (Windows, 2026-10-12). + # grammar, which is not the far shell's (a Windows host's is not). command=argv, # Fixed rather than any caller's workspace: the ssh needs none. cwd=str(Path.home()), diff --git a/tests/test_node_client.py b/tests/test_node_client.py index 334f69b7f..d600ba57f 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -898,7 +898,7 @@ async def launch(machine_id: str) -> _Node: ) await _until(lambda: len(tried) == 2 and pool.live() == ["box"] and endpoint.detached is None) again = pool._nodes["box"].client - # Among the new link's requests, not its last: a heartbeat may well follow the attach (reviewed 2026-10-12). + # Among the new link's requests, not its last: a heartbeat may well follow the attach. attach = (protocol.AGENT_ATTACH, {"channel": 1, "task": endpoint.task, "after": 0}) assert attach in [asked[:2] for asked in again.asked] assert pool.status("box").state == "connected" and pool._nodes["box"].channels[1] is endpoint From 2ff2c789e192f7cac143092139dc990db26fe508 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:23:35 +0800 Subject: [PATCH 16/44] fix(agent): a fetched file lands under a key of its path, so no two paths land as one name Two paths on a machine landed under their own names here, and a filesystem that reads two names as one -- `Report.txt` and `report.txt` on Windows or macOS, `report.txt` and `report.txt.` on Windows -- put the second file's bytes under the first's entry. A file now lands under `pulled///`, the key a digest of its path on the machine, so two paths never share a landing, and the name beside it stays readable. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/deliver.py | 37 ++++++++++++++++----------- tests/test_deliver_files_tool.py | 43 ++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index d9ba80b50..77fe5db4c 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import mimetypes import re import time @@ -65,16 +66,16 @@ def _on_machine(raw: str) -> RemotePath | str | None: async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path | None, str]: - """``where``'s file, fetched from its machine to ``/pulled//``; or why it was not. + """``where``'s file, fetched from its machine to ``/pulled///``; or why it was not. Only a file under the directories this Raven's agents work in on that machine is fetched here, since this tool asks nothing: a delivery is not a way around the read check, and what another conversation was allowed to read is no evidence for this one. A file elsewhere is read there. One over :data:`PULL_FILE_BUDGET` stays, and is named with its size. The file - lands under its whole path on the machine, each component written as one - name here (:func:`_landing`), so two files of one name from two - directories never land on each other and no name reaches above the + lands under a key of its path on the machine, by its name + (:func:`_landing`), so two paths never land on each other, however this + computer's filesystem reads their names, and no name reaches above the delivery directory. """ pool = remote_path.nodes() @@ -118,8 +119,8 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path #: The most a file fetched from a registered machine for a delivery may weigh; a bigger one is read there. PULL_FILE_BUDGET = 20 * 1024 * 1024 -#: What a component of a fetched file's path may not hold as a name on this -#: computer: the other platform's separator, what Windows reserves in a name, +#: What a fetched file's name may not hold as a name on this computer: the +#: other platform's separator, what Windows reserves in a name, #: a control character. A Linux file name may hold any of them, and ``..\`` #: inside one, read by a Windows host as two steps up, put a fetched file #: above the workspace. Each is written as its @@ -130,23 +131,29 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path def _local_name(part: str) -> str: - """One component of a path on the machine, as one name on this computer's filesystem.""" + """A name on the machine, as one name on this computer's filesystem.""" if part in (".", ".."): return part.replace(".", "%2E") return _NOT_A_NAME_HERE.sub(lambda m: f"%{ord(m.group(0)):02X}", part) def _landing(workspace: Path | None, machine: str, place: str) -> Path: - """Where ``place`` on ``machine`` lands: under ``pulled//``, by its POSIX components, each one name here. - - Checked before anything is made or written: a landing that resolves - outside its machine's delivery directory is refused, whatever the name - did to get there. + """Where ``place`` on ``machine`` lands: ``pulled///``, the key a digest of the path there. + + The key keeps apart two paths that this computer's filesystem would read + as one name -- ``Report.txt`` and ``report.txt`` on Windows or macOS, + ``report.txt`` and ``report.txt.`` on Windows -- as well as two files of + one name from two directories; the name, written as one this computer + keeps (:func:`_local_name`), stays readable beside it. Checked before + anything is made or written: a landing that resolves outside its + machine's delivery directory is refused, whatever the name did to get + there. """ pulled = (workspace or Path.cwd()) / "pulled" / _local_name(machine) - parts = [_local_name(part) for part in PurePosixPath(place).parts if part != "/"] - local = pulled.joinpath(*parts) - if not parts or local.resolve() == pulled.resolve() or not local.resolve().is_relative_to(pulled.resolve()): + key = hashlib.sha256(f"{machine}\0{place}".encode()).hexdigest()[:16] + name = _local_name(PurePosixPath(place).name) + local = pulled / key / name + if not name or local.resolve() == pulled.resolve() or not local.resolve().is_relative_to(pulled.resolve()): raise ValueError(f"{place} would not land under {pulled}") return local diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 5bf5f0317..43cded64f 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -3,9 +3,10 @@ from __future__ import annotations import asyncio +import hashlib import json import time -from pathlib import Path +from pathlib import Path, PurePosixPath import pytest @@ -388,6 +389,12 @@ def machines(tmp_path, monkeypatch): return pool +def _landed(tmp_path: Path, place: str, *, name: str | None = None, machine: str = "box") -> Path: + """Where a file of the machine lands, as deliver.py lays it out: under a key of its path, by its name.""" + key = hashlib.sha256(f"{machine}\0{place}".encode()).hexdigest()[:16] + return tmp_path / "chanwork" / "pulled" / machine / key / (name or PurePosixPath(place).name) + + class _RemotePool: """A node pool with one file of a given size, which it hands over as x's.""" @@ -415,7 +422,7 @@ async def test_a_file_on_a_machine_is_brought_back_and_delivered(tool, tmp_path, manifest = tool.take_metadata()["raven_delivery"] assert manifest["invalid"] == [], summary (entry,) = manifest["files"] - landed = tmp_path / "chanwork" / "pulled" / "box" / "root" / "raven-work" / "t1" / "report.pdf" + landed = _landed(tmp_path, "/root/raven-work/t1/report.pdf") assert entry["name"] == "report.pdf" and entry["path"] == str(landed) and entry["size"] == 5 assert landed.read_bytes() == b"xxxxx" assert machines.pulled == [("/root/raven-work/t1/report.pdf", landed, ["/root/raven-work"])] @@ -430,8 +437,9 @@ async def test_a_name_holding_the_other_platform_s_separators_lands_as_one_name_ manifest = tool.take_metadata()["raven_delivery"] assert manifest["invalid"] == [], summary (landed,) = (Path(entry["path"]) for entry in manifest["files"]) - pulled = tmp_path / "chanwork" / "pulled" / "box" - assert landed == pulled / "root" / "raven-work" / "t1" / "..%5C..%5C..%5Creview_escape.txt" + assert landed == _landed( + tmp_path, "/root/raven-work/t1/..\\..\\..\\review_escape.txt", name="..%5C..%5C..%5Creview_escape.txt" + ) assert landed.is_file() and not list(tmp_path.glob("**/review_escape.txt")) assert machines.pulled[0][0] == "/root/raven-work/t1/..\\..\\..\\review_escape.txt" @@ -445,8 +453,10 @@ async def test_two_names_that_differ_only_by_one_written_out_land_apart(tool, tm manifest = tool.take_metadata()["raven_delivery"] assert manifest["invalid"] == [], summary landed = [Path(entry["path"]) for entry in manifest["files"]] - pulled = tmp_path / "chanwork" / "pulled" / "box" / "root" / "raven-work" / "t1" - assert landed == [pulled / "a%5Cb.txt", pulled / "a%255Cb.txt"] + assert landed == [ + _landed(tmp_path, "/root/raven-work/t1/a\\b.txt", name="a%5Cb.txt"), + _landed(tmp_path, "/root/raven-work/t1/a%5Cb.txt", name="a%255Cb.txt"), + ] assert all(place.is_file() for place in landed) assert [where for where, _, _ in machines.pulled] == [ "/root/raven-work/t1/a\\b.txt", @@ -454,6 +464,21 @@ async def test_two_names_that_differ_only_by_one_written_out_land_apart(tool, tm ] +async def test_two_paths_this_filesystem_would_read_as_one_name_land_apart(tool, tmp_path, machines) -> None: + # Windows and macOS read `Report.txt` and `report.txt` as one name, and Windows drops a trailing dot: the + # key of each path keeps them apart, whatever the filesystem makes of the name. + names = ("Report.txt", "report.txt", "report.txt.") + + summary = await tool.execute(files=[{"path": f"box:raven-work/t1/{name}"} for name in names]) + + manifest = tool.take_metadata()["raven_delivery"] + assert manifest["invalid"] == [], summary + landed = [Path(entry["path"]) for entry in manifest["files"]] + assert landed == [_landed(tmp_path, f"/root/raven-work/t1/{name}") for name in names] + assert len({place.parent for place in landed}) == 3 and all(place.is_file() for place in landed) + assert [where for where, _, _ in machines.pulled] == [f"/root/raven-work/t1/{name}" for name in names] + + async def test_nothing_is_fetched_to_a_landing_outside_the_delivery_directory( tool, tmp_path, machines, monkeypatch ) -> None: @@ -475,8 +500,10 @@ async def test_two_files_of_one_name_from_two_directories_land_apart(tool, tmp_p manifest = tool.take_metadata()["raven_delivery"] assert manifest["invalid"] == [], summary landed = [Path(entry["path"]) for entry in manifest["files"]] - pulled = tmp_path / "chanwork" / "pulled" / "box" / "root" / "raven-work" - assert landed == [pulled / "a" / "report.pdf", pulled / "b" / "report.pdf"] + assert landed == [ + _landed(tmp_path, "/root/raven-work/a/report.pdf"), + _landed(tmp_path, "/root/raven-work/b/report.pdf"), + ] assert [where for where, _, _ in machines.pulled] == [ "/root/raven-work/a/report.pdf", "/root/raven-work/b/report.pdf", From 482c704fc0357b757eed072d80576b455583dec6 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:37:08 +0800 Subject: [PATCH 17/44] feat(*): pick up a turn left on a registered machine when this computer stops A turn on a registered machine ended with this computer: stopping the gateway cancelled it and closed the agent, and a hard kill left it running there with nobody to read its answer. Now a run on a machine is remembered here while it is in flight -- the machine, the task its daemon runs the agent under, the session, the prompt awaited, the frames read, the ids used, the agent's handshake -- in an owner-only file under the raven home, kept current as frames arrive and removed when the turn ends here. When this process stops, the agents on machines stay with their daemons: no session/cancel, the connection's close ends the endpoint only, and the manager leaves the dispatch's record open. At the next start the runs are picked up: the task attached from the frame last read, the connection adopted into the pool with ids carried on, the prompt's answer awaited, the reply built as a turn ended here is, the record finished, the registry and node claim closed, and the result announced into the conversation that asked. A machine that cannot be asked keeps the record for a later start; a run the machine no longer has, or whose agent ended meanwhile, is announced as the error it is. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 8 +- raven/acp_client/acp_agent.py | 81 +++++++- raven/acp_client/client.py | 61 ++++++ raven/acp_client/pool.py | 49 +++++ raven/acp_client/remote.py | 31 +++ raven/acp_client/remote_resume.py | 144 +++++++++++++ raven/acp_client/remote_runs.py | 197 ++++++++++++++++++ raven/agent/subagent/manager.py | 18 +- raven/agent/subagent/resume_remote.py | 163 +++++++++++++++ raven/cli/gateway_commands.py | 11 + raven/node/client.py | 83 +++++++- raven/rpc/bootstrap.py | 8 + tests/test_acp_client_resume.py | 199 ++++++++++++++++++ tests/test_node_client.py | 101 ++++++++++ tests/test_remote_resume.py | 206 +++++++++++++++++++ tests/test_remote_runs.py | 132 ++++++++++++ tests/test_resume_remote.py | 278 ++++++++++++++++++++++++++ tests/test_subagent_acp_remote.py | 108 +++++++++- tests/test_subagent_manager.py | 57 ++++++ 19 files changed, 1924 insertions(+), 11 deletions(-) create mode 100644 raven/acp_client/remote_resume.py create mode 100644 raven/acp_client/remote_runs.py create mode 100644 raven/agent/subagent/resume_remote.py create mode 100644 tests/test_acp_client_resume.py create mode 100644 tests/test_remote_resume.py create mode 100644 tests/test_remote_runs.py create mode 100644 tests/test_resume_remote.py diff --git a/CONTEXT.md b/CONTEXT.md index 524eeee65..d40d6107c 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -429,8 +429,12 @@ named by the code's version and digest -- its install name, `bundle.install_name host keeps at most one live link per machine and watches it with a heartbeat (`raven/node/client.py`): a link silent for 15 s is lost, the agents on it wait, and the link is tried again on a backoff for as long as any waits; a link that comes back attaches -each agent again by its task id. Only `--stdio` (tests, a fallback) ties the agents to the -connection. +each agent again by its task id. A turn running on a machine when this computer stops is +left with the daemon (`remote.leave_agents_running()`), remembered while in flight in +`/machines/runs/` (`raven/acp_client/remote_runs.py`), and picked up by the +next start (`raven/agent/subagent/resume_remote.py`): attached from the frame last read, +its reply read off the machine and announced into the conversation that asked. Only +`--stdio` (tests, a fallback) ties the agents to the connection. _Avoid_: "node" alone (a DAG node is a `run_subagent_dag` step, and the Machine entry already rules the bare word out for a computer); "node id" for the install name (a **Node id** is a delegated task's address); "agent" or "server" (it diff --git a/raven/acp_client/acp_agent.py b/raven/acp_client/acp_agent.py index f6fea6d1e..f393938f2 100644 --- a/raven/acp_client/acp_agent.py +++ b/raven/acp_client/acp_agent.py @@ -36,7 +36,7 @@ from loguru import logger -from raven.acp_client import remote +from raven.acp_client import remote, remote_runs from raven.acp_client.acp_dialects import AcpDialect, ToolCall, content_texts, dialect_for from raven.acp_client.ask_user import AskUserResponder, clarify_responder from raven.acp_client.capabilities import ( @@ -1658,6 +1658,23 @@ async def run( } ) record_session(span, session_id=session_id, resumed=resumed) + # A run on a machine is remembered here while it is in flight, + # so a process after this one can pick it up where this one + # left it (remote_runs): removed when the turn ends here, kept + # when this process stops and leaves the agent to the machine. + tracker = ( + self._track_on_machine( + client, + connection, + session_id=session_id, + skey=skey, + handle=handle, + task_id=task_id, + session_cwd=session_cwd, + ) + if launched is not None + else None + ) # Snapshotted before the prompt so the error below names what this # turn ran into rather than what the connection has ever refused. @@ -1719,8 +1736,14 @@ async def run( }, timeout=_prompt_deadline(self.timeout), cancel_session=session_id, + sent=tracker.prompt_sent if tracker is not None else None, ) except asyncio.CancelledError: + if tracker is not None and remote.leaving(): + # This process is stopping: the turn goes on under + # the machine's daemon, and the manager reads this + # dispatch as left, not cancelled. + remote_runs.mark_left(task_id) # An UNSETTLED cancel means the turn outlived its cancel # budget, so it is still running on the agent while this lock # is about to be released -- prompting the same session again @@ -1752,6 +1775,8 @@ async def run( record_frames(span, cancelled) raise finally: + if tracker is not None: + self._untrack(client, tracker, keep=remote.leaving()) activity.offer_steer(collector._run, None) connection.router.detach(session_id, collector) connection.elicitors.detach(session_id, elicitor) @@ -1879,6 +1904,60 @@ async def launcher() -> Any: return _OnMachine(target=target, root=root, session_cwd=session_cwd, since=since, link=link, launcher=launcher) + def _track_on_machine( + self, + client: Any, + connection: Any, + *, + session_id: str, + skey: str, + handle: str, + task_id: str, + session_cwd: str, + ) -> remote_runs.Tracker | None: + """The record of this turn on its machine, kept current as the agent's frames arrive.""" + endpoint = client.endpoint + task = getattr(endpoint, "task", None) + watchers = getattr(endpoint, "watchers", None) + if not isinstance(task, str) or watchers is None: + return None + run = remote_runs.RemoteRun( + machine=self.machine or "", + task=task, + agent=self.name, + session_key=skey, + handle=handle, + task_id=task_id, + session_id=session_id, + prompt_id=0, + next_id=client.next_request_id, + initialize=connection.initialize, + launch_key=connection.launch_key, + config_key=connection.config_key, + session_cwd=session_cwd, + stateful=self.is_stateful, + received=int(getattr(endpoint, "received", 0) or 0), + ) + try: + tracker = remote_runs.Tracker(run) + except OSError as exc: + logger.warning( + "acp agent {!r}: the run on machine {} could not be recorded: {}", self.name, self.machine, exc + ) + return None + watchers.append(tracker.received) + return tracker + + @staticmethod + def _untrack(client: Any, tracker: remote_runs.Tracker, *, keep: bool) -> None: + watchers = getattr(client.endpoint, "watchers", None) + if watchers is not None: + try: + watchers.remove(tracker.received) + except ValueError: + pass + tracker.close(keep=keep) + @property def can_steer(self) -> bool: """Whether this agent takes text mid-turn (``_raven/session/steer``). diff --git a/raven/acp_client/client.py b/raven/acp_client/client.py index b519ed035..019493c43 100644 --- a/raven/acp_client/client.py +++ b/raven/acp_client/client.py @@ -593,9 +593,13 @@ async def request( *, timeout: float | None = None, cancel_session: str | None = None, + sent: Callable[[int], None] | None = None, ) -> Any: """Send a request and await its result. + ``sent`` hears the request's id as it goes out: what a run on a + machine records, so a process after this one can await the answer. + Raises :class:`AcpRemoteError` if the agent answers with an error, :class:`AcpTimeoutError` on budget expiry, :class:`AcpConnectionError` if the connection is gone. @@ -630,6 +634,8 @@ async def request( self._request_sessions[request_id] = (params or {}).get("sessionId") if isinstance(params, dict) else None self._request_methods[request_id] = method try: + if sent is not None: + sent(request_id) await self._send(protocol.request(request_id, method, params)) if timeout is None: return await asyncio.shield(future) @@ -647,6 +653,51 @@ async def request( self._request_sessions.pop(request_id, None) self._request_methods.pop(request_id, None) + @property + def endpoint(self) -> Endpoint: + """What this connection runs over: a child process here, or a channel to an agent on a machine.""" + return self._endpoint + + @property + def next_request_id(self) -> int: + """The id the next request from here carries.""" + return self._next_id + 1 + + def continue_ids_from(self, next_id: int) -> None: + """Carry request ids on from ``next_id``: for a connection taken over from a process that used the ones before it.""" + self._next_id = max(self._next_id, int(next_id) - 1) + + async def await_response( + self, request_id: int, *, method: str, session: str | None = None, timeout: float | None = None + ) -> Any: + """The answer to a request another process of this Raven sent on this connection: awaited, never sent. + + What a run picked up after a restart waits on: the ``session/prompt`` + the process before this one sent, whose answer the agent still owes. + Raises as :meth:`request` does. + """ + if not self.alive: + if self._closed: + raise AcpConnectionError(f"acp agent {self.name!r}: connection is not open") + await self._settle_exit() + raise self._ended_error() + future: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + self._pending[request_id] = future + self._request_sessions[request_id] = session + self._request_methods[request_id] = method + try: + if timeout is None: + return await asyncio.shield(future) + return await asyncio.wait_for(asyncio.shield(future), timeout=timeout) + except asyncio.TimeoutError: + raise AcpTimeoutError( + f"acp agent {self.name!r}: {method} timed out after {timeout}s", method=method + ) from None + finally: + self._pending.pop(request_id, None) + self._request_sessions.pop(request_id, None) + self._request_methods.pop(request_id, None) + @property def prompting(self) -> bool: """Whether a turn is in flight on this connection right now. @@ -701,6 +752,10 @@ async def _cancel_turn(self, session_id: str, future: asyncio.Future) -> None: this codebase cancels once and then gathers -- the same property ``CliAgentBackend._kill_process_group`` relies on to await the child. """ + if getattr(self._endpoint, "stays", False): + # This process is stopping and the agent stays with its machine's + # daemon: the turn goes on there, and the next process picks it up. + return try: await self.notify("session/cancel", {"sessionId": session_id}) except Exception: # noqa: BLE001 - a connection already gone has nothing to settle @@ -773,6 +828,12 @@ async def _read_stdout(self) -> None: if self._journal is not None: self._journal.note("in", frame=frame, session=self._frame_session(frame)) await self._dispatch(frame) + # Told after the frame is dealt with, so what an endpoint + # records as read is what was read: a channel to a machine + # keeps the count a later attach goes on from. + consumed = getattr(self._endpoint, "consumed", None) + if consumed is not None: + consumed() except asyncio.CancelledError: raise except Exception as exc: # noqa: BLE001 - the loop must not die silently diff --git a/raven/acp_client/pool.py b/raven/acp_client/pool.py index fb39f8f8b..ee5855121 100644 --- a/raven/acp_client/pool.py +++ b/raven/acp_client/pool.py @@ -528,6 +528,55 @@ async def acquire( held[bkey] = connection return connection + async def adopt( + self, + *, + name: str, + endpoint: Any, + initialize: Any, + launch: str, + config: str, + next_id: int, + ) -> tuple[_Connection, bool]: + """A connection over an agent a process before this one started on a machine and left running there. + + No handshake: the agent answered ``initialize`` to that process, and + its answer comes with the run's record. Request ids go on from + ``next_id``, past the ones that process used, so a frame the daemon + replays answers nothing sent from here. Held under the name for later + dispatches, as a connection started here would be -- unless one of + the name is already live, in which case this one serves the pick-up + only and the second value says so. + """ + async with self._lock_for(name): + router = _SessionRouter(name) + elicitors = _SessionElicitors(name) + responders = _SessionResponders(name) + journal = open_journal(name) + answering = request_dispatcher(name, router.dispatch, elicitors=elicitors) + notifying = notification_dispatcher(name, router.dispatch, responders=responders) + client = AcpClient.over( + endpoint, name=name, on_request=answering, on_notification=notifying, journal=journal + ) + client.continue_ids_from(next_id) + connection = _Connection( + client, + router, + elicitors, + responders, + launch, + initialize, + handshake_bytes=journal.offset if journal is not None else 0, + config=config, + ) + held = self._connections.setdefault(name, {}) + bkey = _binding_key(None) + existing = held.get(bkey) + if existing is not None and existing.alive: + return connection, False + held[bkey] = connection + return connection, True + async def drop(self, name: str) -> None: """Close and forget one agent's connections, if any.""" async with self._lock_for(name): diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index 682dec0c3..fb1212ce0 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -194,6 +194,34 @@ def ssh_command( return argv +# Whether this process is stopping and leaves the agents on registered machines to their daemons. +_LEAVING = False + + +def leave_agents_running() -> None: + """From here on, closing a link leaves the agents on the machines running: this process is stopping. + + A machine's daemon owns the agent (:mod:`raven.node.server`); the link is + only the way to it. Said once by the host's shutdown, before the sub-agents + are cancelled and the pools closed, so neither a turn's cancel nor a link's + close reaches an agent there, and the next process picks the run up + (:mod:`raven.acp_client.remote_runs`). + """ + global _LEAVING + _LEAVING = True + + +def leaving() -> bool: + """Whether this process is stopping and leaves the agents on the machines running.""" + return _LEAVING + + +def staying() -> None: + """The opposite of :func:`leave_agents_running`: for a process that goes on (tests).""" + global _LEAVING + _LEAVING = False + + def failure(target: Machine, log: str | os.PathLike[str], exc: BaseException) -> str | None: """The sentence for a connection that ended, when ssh or the far shell ended it. @@ -439,5 +467,8 @@ def _target(target: Machine) -> tuple[str, int, str, str]: "shell_path", "shown", "ssh_command", + "leave_agents_running", + "leaving", + "staying", "ssh_log_path", ] diff --git a/raven/acp_client/remote_resume.py b/raven/acp_client/remote_resume.py new file mode 100644 index 000000000..8a9f6e05a --- /dev/null +++ b/raven/acp_client/remote_resume.py @@ -0,0 +1,144 @@ +"""Picking a turn up that a process before this one left running on a registered machine. + +The run's record (:mod:`raven.acp_client.remote_runs`) says which machine, +which task its daemon runs the agent under, which ACP session the turn is in, +which prompt is awaited and how many frames that process had read. This +module turns that into the reply the turn owes: the machine's daemon replays +what was never read and hands the rest over as it comes (``agent/attach``), +a connection over that channel is taken into the pool as if started here, the +turn's collector reads the agent's messages off it, and the answer to the +prompt -- sent by the other process, awaited by this one -- ends the wait. What +to do with the reply is the manager's business (:mod:`raven.agent.subagent.resume_remote`). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from loguru import logger + +from raven.acp_client import remote +from raven.acp_client.acp_agent import AcpAgentBackend, _prompt_deadline, _TurnCollector +from raven.acp_client.acp_dialects import dialect_for +from raven.acp_client.pool import get_pool +from raven.acp_client.protocol import AcpConnectionError, AcpTimeoutError +from raven.acp_client.remote_runs import RemoteRun +from raven.ops import remote_path + +#: What the reply of a picked-up turn ends with: the reader learns it crossed a restart. +PICKED_UP_NOTE = "picked up after this computer restarted" + + +class UnreachableError(RuntimeError): + """The machine could not be asked: the record stays, for a later start.""" + + +@dataclass(frozen=True) +class PickedUp: + status: str + """``ok`` with the reply, or ``error`` with the sentence that says why there is none.""" + reply: str + stop_reason: Any = None + + +class _Span: + """What :meth:`AcpAgentBackend._finished` tells about a turn cut short; there is no trace here.""" + + def error(self, said: str) -> None: + logger.warning("picked-up turn: {}", said) + + +async def pick_up(run: RemoteRun, backend: AcpAgentBackend, *, pool: Any = None) -> PickedUp: + """The reply the turn in ``run`` owes, read off the machine; or why there is none. + + Raises :class:`UnreachableError` when the machine cannot be asked at all, so + the caller keeps the record for a later start; every other outcome is a + :class:`PickedUp`, and the record is the caller's to remove. + """ + link = pool if pool is not None else remote_path.nodes() + if link is None: + raise UnreachableError( + f"this process does not reach registered machines, so the run on machine {run.machine} waits" + ) + try: + target = remote.machine(run.machine) + except remote.RemoteMachineError as exc: + return PickedUp("error", f"the run of {run.agent!r} cannot be picked up: {exc}") + try: + rows = await link.tasks(run.machine) + except remote_path.NodeCallError as exc: + raise UnreachableError(str(exc)) from None + if not any(row.get("task") == run.task for row in rows): + return PickedUp( + "error", + f"machine {target.label} no longer has the run of {run.agent!r} (task {run.task}); " + "its record there was removed, or the machine's node directory was", + ) + if run.prompt_id <= 0: + return PickedUp("error", f"the run of {run.agent!r} on machine {target.label} ended before its prompt was sent") + try: + endpoint, attached = await link.attach(run.machine, run.task, name=run.agent, after=run.received) + except remote_path.NodeCallError as exc: + if exc.reason in ("unknown_task", None) and "no longer runs" in str(exc): + return PickedUp("error", str(exc)) + raise UnreachableError(str(exc)) from None + connection, held = await get_pool().adopt( + name=run.agent, + endpoint=endpoint, + initialize=run.initialize, + launch=run.launch_key, + config=run.config_key, + # Past the ids the other process used, with room: a replayed answer to + # one of its requests must resolve nothing sent from here. + next_id=run.next_id + _ID_ROOM, + ) + client = connection.client + collector = _TurnCollector(dialect=dialect_for(run.initialize), limit=backend.max_output_chars, track_files=False) + try: + async with connection.session_lock(run.session_id): + connection.router.reserve(run.session_id) + await connection.router.take_over(run.session_id, collector) + try: + result = await client.await_response( + run.prompt_id, + method="session/prompt", + session=run.session_id, + timeout=_prompt_deadline(backend.timeout), + ) + except AcpConnectionError as exc: + rc = attached.get("rc") + ended = f" (exit {rc})" if isinstance(rc, int) else "" + return PickedUp( + "error", + f"the run of {run.agent!r} on machine {target.label} ended while this computer was away{ended}: {exc}", + ) + except AcpTimeoutError as exc: + return PickedUp("error", f"the run of {run.agent!r} on machine {target.label} was waited out: {exc}") + finally: + connection.router.detach(run.session_id, collector) + stop_reason = result.get("stopReason") if isinstance(result, dict) else None + if not collector.text: + return PickedUp( + "error", + f"acp agent {run.agent!r} ended its turn with no content (stopReason={stop_reason!r}); " + f"its frames are on machine {target.label}, task {run.task}", + stop_reason, + ) + reply = await backend._finished(collector.reply, stop_reason=stop_reason, span=_Span(), sink=None) + reply += ( + f"\n\n[raven] Ran on machine {target.label}, in {remote.shown(target, run.session_cwd)}, " + f"{PICKED_UP_NOTE}; the files it wrote are there, not on this computer -- read_file, list_dir, " + "find, grep, write_file and edit_file take that path as it is, and deliver_files fetches one by it." + ) + return PickedUp("ok", reply, stop_reason) + finally: + if not held: + # Another connection of the name is live: this one served the pick-up only. + await client.close() + + +#: How far past the other process's last request id this one starts. +_ID_ROOM = 1000 + +__all__ = ["PICKED_UP_NOTE", "PickedUp", "UnreachableError", "pick_up"] diff --git a/raven/acp_client/remote_runs.py b/raven/acp_client/remote_runs.py new file mode 100644 index 000000000..3fa846231 --- /dev/null +++ b/raven/acp_client/remote_runs.py @@ -0,0 +1,197 @@ +"""What this computer keeps of a run on a registered machine, for as long as the run is in flight. + +A run on a machine belongs to the machine's daemon (:mod:`raven.node.server`): +it goes on, and its frames are kept there, when this process is gone -- killed, +restarted with ``/restart``, or stopped on purpose. What this process has to +keep is what the machine does not know, so that the next process can pick the +run up where this one left it: whose run it is (the Raven session, the agent, +the instance handle, the dispatch), the ACP session the turn runs in, the id +of the prompt whose answer is awaited, how many of the agent's frames reached +here, the ids this side has used, and what the agent answered to +``initialize`` on the connection. One file per run under +``/machines/runs/.json``, owner-only, written when the turn +starts, kept current as frames arrive, and removed when the turn ends here. A +file left behind is a run to pick up (:mod:`raven.acp_client.remote_resume`). +No address is in it: the machine is named by its registry id. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from loguru import logger + +from raven.home import raven_home + +#: Where the records live under the raven home. +RUNS_DIR = ("machines", "runs") +VERSION = 1 + + +@dataclass +class RemoteRun: + """One turn running on a machine, as this computer must remember it to pick it up again.""" + + machine: str + """The registry id of the machine.""" + task: str + """The task id the machine's daemon runs the agent under.""" + agent: str + session_key: str + handle: str + task_id: str + """The dispatch's id on this computer (the manager's), which the announce names.""" + session_id: str + """The ACP session the turn runs in.""" + prompt_id: int + """The JSON-RPC id of the ``session/prompt`` whose answer is awaited.""" + next_id: int + """The id the request after the prompt would carry; a later process goes on past it.""" + initialize: Any + """The agent's answer to ``initialize`` on this connection: the capabilities of the process serving it.""" + launch_key: str + config_key: str + session_cwd: str + stateful: bool + received: int = 0 + """How many of the agent's frames this computer has read on the connection: where an attach goes on from.""" + started: float = field(default_factory=time.time) + + def as_json(self) -> dict[str, Any]: + return {"version": VERSION, **asdict(self)} + + @classmethod + def from_json(cls, data: Any) -> RemoteRun | None: + """A record read back, or ``None`` for one this version does not read.""" + if not isinstance(data, dict) or data.get("version") != VERSION: + return None + fields = {name for name in cls.__dataclass_fields__} + try: + run = cls(**{key: value for key, value in data.items() if key in fields}) + except TypeError: + return None + if not (run.machine and run.task and run.agent and run.session_id): + return None + return run + + +def runs_dir() -> Path: + """The directory of the records, made owner-only.""" + path = raven_home().joinpath(*RUNS_DIR) + path.mkdir(parents=True, exist_ok=True) + with contextlib.suppress(OSError): + path.chmod(0o700) + return path + + +def path_of(task: str) -> Path: + return runs_dir() / f"{task}.json" + + +def record(run: RemoteRun) -> Path: + """Write ``run``'s record, whole and owner-only, in place of any earlier one.""" + path = path_of(run.task) + tmp = path.with_suffix(".json.tmp") + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(run.as_json(), handle) + os.replace(tmp, path) + return path + + +def forget(task: str) -> None: + """Remove the record of ``task``: the turn ended here.""" + with contextlib.suppress(FileNotFoundError, OSError): + path_of(task).unlink() + + +def pending() -> list[RemoteRun]: + """Every run a process of this Raven left on a machine, oldest first; a record that cannot be read is skipped and named.""" + out: list[RemoteRun] = [] + for path in sorted(runs_dir().glob("*.json")): + try: + run = RemoteRun.from_json(json.loads(path.read_text(encoding="utf-8"))) + except (OSError, ValueError) as exc: + logger.warning("remote run record {} could not be read: {}", path.name, exc) + continue + if run is None: + logger.warning("remote run record {} is not one this version reads; left alone", path.name) + continue + out.append(run) + return sorted(out, key=lambda run: run.started) + + +class Tracker: + """One run's record, kept current as the agent's frames arrive, and removed when the turn ends here. + + ``received`` is what a channel endpoint's watcher calls after every frame + the connection has handed on (:class:`raven.node.client.ChannelEndpoint`), + so a later attach replays only what this computer never read. + """ + + def __init__(self, run: RemoteRun) -> None: + self.run = run + self.kept = False + record(run) + + def prompt_sent(self, request_id: int) -> None: + """The prompt went out under ``request_id``: what a later process awaits the answer to.""" + self.run.prompt_id = request_id + self.run.next_id = request_id + 1 + try: + record(self.run) + except OSError as exc: + logger.warning("remote run record {} could not be updated: {}", self.run.task, exc) + + def received(self, n: int) -> None: + if n <= self.run.received: + return + self.run.received = n + try: + record(self.run) + except OSError as exc: + logger.warning("remote run record {} could not be updated: {}", self.run.task, exc) + + def close(self, *, keep: bool) -> None: + """The turn ended here (``keep`` false) or goes on without this process (``keep`` true).""" + self.kept = keep + if not keep: + forget(self.run.task) + + +# The dispatches whose turns this process left running on a machine, by the manager's task id: the +# manager reads one as left, not cancelled, when the run's task is cancelled at shutdown. +_LEFT: set[str] = set() + + +def mark_left(task_id: str) -> None: + _LEFT.add(task_id) + + +def was_left(task_id: str) -> bool: + """Whether the dispatch ``task_id`` was left running on its machine; read once.""" + try: + _LEFT.remove(task_id) + except KeyError: + return False + return True + + +__all__ = [ + "RUNS_DIR", + "RemoteRun", + "Tracker", + "forget", + "mark_left", + "path_of", + "pending", + "record", + "runs_dir", + "was_left", +] diff --git a/raven/agent/subagent/manager.py b/raven/agent/subagent/manager.py index 67f274682..d9edee0b5 100644 --- a/raven/agent/subagent/manager.py +++ b/raven/agent/subagent/manager.py @@ -11,6 +11,7 @@ from loguru import logger +from raven.acp_client import remote_runs from raven.agent import workdir from raven.agent.subagent import activity from raven.agent.subagent.attachments import retarget_note, with_attachment_note, with_undeliverable_note @@ -1795,6 +1796,7 @@ async def _run_subagent_inner( # the same instance a direct chat talks to, and watching it there is # the same question. cancelled = False + left = False # The record's own id, not its directory's name: the artifacts are a # filename prefix in the shared node root now, so the directory names # the namespace rather than the call. @@ -1885,6 +1887,16 @@ async def _run_subagent_inner( except asyncio.CancelledError: cancelled = True reason = self._cancel_reasons.pop(task_id, UNEXPLAINED_CANCEL) + if remote_runs.was_left(task_id): + # This process is stopping and the turn goes on under the + # machine's daemon: nothing is finished or announced here; + # the next process picks the run up and does both + # (raven.agent.subagent.resume_remote). + left = True + logger.info( + "Subagent [{}] stays running on its machine; picked up when this computer starts again", task_id + ) + raise await _write_spawn_status(session_key, agent, handle, "cancelled") self._emit_status( origin, task_id, task_summary, "cancelled", call_id=call_id, ended_at=int(time.time() * 1000) @@ -1933,8 +1945,10 @@ async def _run_subagent_inner( # nothing can free, so the registry has to be closed on the way # out whichever branch above ran. `finish` is idempotent and has # already written the record by here, so the file it reports is - # the one that decides `has_output`. - await self._finalize_node_claim(session_key, record) + # the one that decides `has_output`. Not for a run left on its + # machine: its claim stays open for the process that picks it up. + if not left: + await self._finalize_node_claim(session_key, record) # Not for a cancelled call: this poller would be created after # the cancellation sweep took its snapshot, so nothing could # reap it (see cancel_all / cancel_by_session). diff --git a/raven/agent/subagent/resume_remote.py b/raven/agent/subagent/resume_remote.py new file mode 100644 index 000000000..82fab294a --- /dev/null +++ b/raven/agent/subagent/resume_remote.py @@ -0,0 +1,163 @@ +"""Runs left on registered machines by a process before this one, picked up and reported to their sessions. + +Once per process, soon after start: every record a run on a machine left +behind (:mod:`raven.acp_client.remote_runs`) is a turn this computer still +owes an answer for. Its reply is read off the machine +(:func:`raven.acp_client.remote_resume.pick_up`), the dispatch's record under +the session is finished with it, the registry row and the node index are +closed, and the result is announced into the conversation that asked, as a +turn that ended here would have been. A machine that cannot be asked keeps +its record for the next start. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +from loguru import logger + +from raven.acp_client import remote_resume, remote_runs +from raven.acp_client.acp_agent import AcpAgentBackend +from raven.agent.subagent.history import SpawnRecord, nodes_root +from raven.agent.subagent.instances import get_registry + +#: How long after start the pick-up begins: the channels and the page are up by then, so an announce has a reader. +START_DELAY_S = 2.0 + +_SCHEDULED = False +_TASKS: set[asyncio.Task[Any]] = set() + + +def schedule_remote_resume(manager: Any, *, delay_s: float = START_DELAY_S) -> asyncio.Task[Any] | None: + """Pick up the runs left on machines, once per process, in the background.""" + global _SCHEDULED + if _SCHEDULED or not hasattr(manager, "session_dir_for"): + return None + _SCHEDULED = True + + async def _later() -> None: + await asyncio.sleep(delay_s) + try: + await resume_remote_runs(manager) + except Exception: # noqa: BLE001 - a pick-up that fails must not take the host down + logger.opt(exception=True).warning("picking up the runs left on machines failed") + + task = asyncio.create_task(_later(), name="remote-resume") + _TASKS.add(task) + task.add_done_callback(_TASKS.discard) + return task + + +async def resume_remote_runs(manager: Any) -> int: + """Every run left on a machine, picked up and reported; how many were concluded here.""" + concluded = 0 + for run in remote_runs.pending(): + backend = _backend_of(manager, run) + if backend is None: + await _conclude( + manager, + run, + "error", + f"the run of {run.agent!r} left on machine {run.machine} cannot be picked up: the agent is no " + "longer on the agent table as an acp agent on that machine; its frames are on the machine, " + f"task {run.task}", + ) + concluded += 1 + continue + try: + picked = await remote_resume.pick_up(run, backend) + except remote_resume.UnreachableError as exc: + logger.warning("the run of {!r} on machine {} waits for a later start: {}", run.agent, run.machine, exc) + continue + except Exception as exc: # noqa: BLE001 - one run's failure is reported as its own, the next is tried + logger.opt(exception=True).warning( + "picking up the run of {!r} on machine {} failed", run.agent, run.machine + ) + picked = remote_resume.PickedUp("error", f"the run of {run.agent!r} could not be picked up: {exc}") + await _conclude(manager, run, picked.status, picked.reply) + concluded += 1 + return concluded + + +def _backend_of(manager: Any, run: remote_runs.RemoteRun) -> AcpAgentBackend | None: + registry = getattr(manager, "registry", None) + backend = registry.backend(run.agent) if registry is not None else None + if not isinstance(backend, AcpAgentBackend) or backend.machine != run.machine: + return None + return backend + + +async def _conclude(manager: Any, run: remote_runs.RemoteRun, status: str, reply: str) -> None: + """Close what the dispatch left open here and announce the outcome into its conversation.""" + remote_runs.forget(run.task) + ok = status == "ok" + session_dir = Path(manager.session_dir_for(run.session_key)) + record, meta = _spawn_record(session_dir, run) + if record is not None: + if ok: + record.finish(status="completed", output=reply) + else: + record.finish(status="failed", error=f"Error: {reply}") + await manager._finalize_node_claim(run.session_key, record) + try: + if ok and run.stateful: + await get_registry().commit(run.session_key, run.agent, run.handle, run.session_id, kind="acp") + else: + await get_registry().upsert_spawn(run.session_key, run.agent, run.handle, "completed" if ok else "failed") + except Exception: # noqa: BLE001 - the registry row must not keep the result from its reader + logger.opt(exception=True).warning("the registry row of {}/{} could not be closed", run.agent, run.handle) + channel, sep, chat_id = run.session_key.partition(":") + if not (sep and channel and chat_id): + logger.warning("the run of {!r} has no conversation to report to (session {!r})", run.agent, run.session_key) + return + manager.remember_session(run.session_key) + origin = {"session_key": run.session_key, "channel": channel, "chat_id": chat_id} + summary = str(meta.get("task_summary") or f"{run.agent} on machine {run.machine}") + task_text = "" + if record is not None: + try: + task_text = record.file("prompt.md").read_text(encoding="utf-8") + except OSError: + task_text = "" + await manager._announce_result( + run.task_id, + summary, + task_text, + reply if ok else f"Error: {reply}", + origin, + "ok" if ok else "error", + record_path=str(record.file("out.md")) if record is not None else None, + ) + + +def _spawn_record(session_dir: Path, run: remote_runs.RemoteRun) -> tuple[SpawnRecord | None, dict[str, Any]]: + """The dispatch's record under the session, still open: the latest of this agent and handle left running.""" + root = nodes_root(session_dir) + best: tuple[int, str, dict[str, Any]] | None = None + for path in root.glob("*.meta.json") if root.is_dir() else []: + try: + meta = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if not isinstance(meta, dict) or meta.get("status") != "running": + continue + if meta.get("agent") != run.agent or str(meta.get("handle") or "") != run.handle: + continue + started = int(meta.get("started_at_ms") or 0) + if best is None or started > best[0]: + best = (started, path.name[: -len(".meta.json")], meta) + if best is None: + return None, {} + _, node_id, meta = best + task = "" + try: + task = (root / f"{node_id}.prompt.md").read_text(encoding="utf-8") + except OSError: + pass + return SpawnRecord(root, session_dir=session_dir, task=task, node_id=node_id), meta + + +__all__ = ["START_DELAY_S", "resume_remote_runs", "schedule_remote_resume"] diff --git a/raven/cli/gateway_commands.py b/raven/cli/gateway_commands.py index ebd3e7053..7bdfa4136 100644 --- a/raven/cli/gateway_commands.py +++ b/raven/cli/gateway_commands.py @@ -1235,6 +1235,12 @@ async def _when_idle() -> None: publish_control_endpoint(bound_host, bound_port, control_token) console.print(f"[green]✓[/green] Control plane: ws://{bound_host}:{bound_port}/ws") await _bind_generation() + # The runs a gateway before this one left on registered machines + # are picked up and reported to their conversations; after the + # generation is bound, so the announce has a spine to go to. + from raven.agent.subagent.resume_remote import schedule_remote_resume + + schedule_remote_resume(agent.subagents) await cron.start() await heartbeat.start() if sentinel_runner is not None: @@ -1301,6 +1307,7 @@ async def _when_idle() -> None: await control.stop() from raven.acp_client.client import begin_drain from raven.acp_client.pool import close_pool + from raven.acp_client.remote import leave_agents_running from raven.node.client import close_node_pool # Before anything tears a transport down: an ACP connection @@ -1311,6 +1318,10 @@ async def _when_idle() -> None: # either. Draining first, because the pool close below kills # every ACP server anyway and no cancelled turn is worth waiting # on when its process is about to go. + # The agents on registered machines stay with their daemons: + # neither the cancel below nor the pool close reaches them, and + # the next gateway picks their runs up (remote_runs). + leave_agents_running() begin_drain() await agent.subagents.cancel_all(reason="the gateway stopped") # The page's spine seals after the sub-agents are cancelled, the diff --git a/raven/node/client.py b/raven/node/client.py index 1f6cbda41..b6d9e99ac 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -36,7 +36,7 @@ import os import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from functools import partial from pathlib import Path @@ -171,7 +171,11 @@ def __init__( self.pid: int | None = None self.received = 0 """How many of the agent's frames reached here: where an attach goes on from.""" - self._frames: asyncio.Queue[bytes | None] = asyncio.Queue() + self.watchers: list[Callable[[int], None]] = [] + """Told, after each frame the connection has dealt with, how many it has read: what a record of the + run keeps, so a process that comes after this one attaches from there (:mod:`raven.acp_client.remote_runs`).""" + self._handing = 0 + self._frames: asyncio.Queue[tuple[int, bytes] | None] = asyncio.Queue() self._stderr: asyncio.Queue[bytes | None] = asyncio.Queue() self._rc: int | None = None self._lost: str | None = None @@ -195,6 +199,19 @@ def detached(self) -> str | None: """Why the endpoint is waiting for its link, or ``None`` while it has one.""" return self._detached + @property + def stays(self) -> bool: + """Whether the agent stays with its machine when this endpoint is ended: this process is stopping.""" + return remote.leaving() + + def consumed(self) -> None: + """The connection has dealt with the frame last handed over: the watchers hear how many it has read.""" + for watcher in list(self.watchers): + try: + watcher(self._handing) + except Exception as exc: # noqa: BLE001 - a record that cannot be kept must not stop the frames + logger.warning("raven node: a watcher of {} failed: {}", self.describe(), exc) + async def write(self, data: bytes) -> None: if self.returncode is not None: raise BrokenPipeError(f"the agent on {self.describe()} has ended") @@ -218,16 +235,28 @@ def _hold(self, line: str) -> None: async def readline(self) -> bytes: item = await self._frames.get() - return b"" if item is None else item + if item is None: + return b"" + self._handing, line = item + return line async def readerr(self) -> bytes: item = await self._stderr.get() return b"" if item is None else item async def end(self) -> None: - """End the agent and what it started, through the node; while the link is down, as soon as it is back.""" + """End the agent and what it started, through the node; while the link is down, as soon as it is back. + + Not while this process is stopping (:func:`raven.acp_client.remote.leave_agents_running`): + then the agent stays with the machine's daemon, this end is only the + endpoint's, and the run is picked up by the next process. + """ if self.returncode is not None: return + if self.stays: + self._lost = f"left running on {self.describe()}: this computer stopped" + self._settle() + return if self._detached is not None: self._cancel_wanted = True return @@ -249,7 +278,7 @@ def release(self) -> None: def feed_frame(self, line: Any, n: Any = None) -> None: if isinstance(line, str) and line and not self._done.is_set(): self.received = n if isinstance(n, int) and not isinstance(n, bool) else self.received + 1 - self._frames.put_nowait(line.encode("utf-8")) + self._frames.put_nowait((self.received, line.encode("utf-8"))) def feed_stderr(self, text: Any) -> None: if isinstance(text, str) and not self._done.is_set(): @@ -477,6 +506,50 @@ async def open_channel( logger.info("acp agent {!r}: started on {}", name, endpoint.describe()) return endpoint + async def attach( + self, machine_id: str, task: str, *, name: str, after: int = 0 + ) -> tuple[ChannelEndpoint, dict[str, Any]]: + """``task`` on the machine, onto a channel of its link, from the frame past the ``after``-th. + + What a process that comes after the one that started the agent uses + to pick the run up: the daemon replays what this computer never read + and hands the rest over as it comes, or, for a task that has ended, + replays its record and says so. The endpoint returned is what an + :class:`AcpClient` runs over; the answer says the pid, the frames + and the exit code (``None`` while the agent runs). + """ + node = await self._acquire(machine_id) + label = node.target.label + channel = node.next_channel + node.next_channel += 1 + endpoint = ChannelEndpoint(node.client, channel, target=node.target, agent=name, task=task) + endpoint.received = after + node.channels[channel] = endpoint + params = {"channel": channel, "task": task, "after": after} + try: + answer = await node.client.request(protocol.AGENT_ATTACH, params, timeout=_ATTACH_TIMEOUT_S) + except AcpRemoteError as exc: + node.channels.pop(channel, None) + raise NodeError( + f"machine {label} no longer runs {name!r} (task {task}): {exc.message}", reason=_REASONS.get(exc.code) + ) from None + except AcpTimeoutError: + node.channels.pop(channel, None) + said = f"the Raven node on machine {label} did not answer {protocol.AGENT_ATTACH}" + await self._lost(machine_id, node, said) + raise NodeError(said) from None + except AcpConnectionError as exc: + node.channels.pop(channel, None) + said = self._ended(node.target, node.log, exc) + await self._lost(machine_id, node, said) + raise NodeError(said) from None + answer = answer if isinstance(answer, dict) else {} + if isinstance(answer.get("pid"), int): + endpoint.pid = answer["pid"] + node.used = time.monotonic() + logger.info("acp agent {!r}: picked up on {} from frame {}", name, endpoint.describe(), after) + return endpoint, answer + def status(self, machine_id: str) -> MachineStatus: """The link's state, with no round trip: for a listing.""" node = self._nodes.get(machine_id) diff --git a/raven/rpc/bootstrap.py b/raven/rpc/bootstrap.py index 419f60e39..045294ff8 100644 --- a/raven/rpc/bootstrap.py +++ b/raven/rpc/bootstrap.py @@ -396,8 +396,12 @@ async def build_rpc_stack( # row launching another child), and the host's own stack does it. if channel != "acp": from raven.agent.subagent.probe import schedule_snapshot_verification + from raven.agent.subagent.resume_remote import schedule_remote_resume schedule_snapshot_verification(agent_loop.subagents) + # The runs a process before this one left on registered machines + # are picked up and reported to their conversations. + schedule_remote_resume(agent_loop.subagents) # Per-server MCP events, broadcast rather than conversation-scoped: a # server connecting is not part of anybody's turn. Clients already listen @@ -550,7 +554,11 @@ async def teardown() -> None: if owns_loop: try: from raven.acp_client.client import begin_drain + from raven.acp_client.remote import leave_agents_running + # The agents on registered machines stay with their daemons + # (see the gateway's shutdown): the next process picks them up. + leave_agents_running() begin_drain() if agent_loop is not None: await agent_loop.subagents.cancel_all(reason="the server stopped") diff --git a/tests/test_acp_client_resume.py b/tests/test_acp_client_resume.py new file mode 100644 index 000000000..fe5613a92 --- /dev/null +++ b/tests/test_acp_client_resume.py @@ -0,0 +1,199 @@ +"""AcpClient: what a connection taken over from a process before this one needs -- ids to go on from, an answer to await, the id of what goes out.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest + +from raven.acp_client import remote +from raven.acp_client.client import AcpClient +from raven.acp_client.protocol import AcpConnectionError, AcpTimeoutError + + +class _Pipe: + """An endpoint fed by the test: what the agent says goes in, what raven says comes out.""" + + pid = 1 + + def __init__(self) -> None: + self._in: asyncio.Queue[bytes | None] = asyncio.Queue() + self._err: asyncio.Queue[bytes | None] = asyncio.Queue() + self.written: list[dict[str, Any]] = [] + self.returncode: int | None = None + self._done = asyncio.Event() + + def describe(self) -> str: + return "a pipe" + + async def write(self, data: bytes) -> None: + self.written.append(json.loads(data.decode().strip())) + + async def readline(self) -> bytes: + item = await self._in.get() + return b"" if item is None else item + + async def readerr(self) -> bytes: + item = await self._err.get() + return b"" if item is None else item + + async def end(self) -> None: + self.returncode = 0 + self._in.put_nowait(None) + self._err.put_nowait(None) + self._done.set() + + async def wait(self) -> int | None: + await self._done.wait() + return self.returncode + + def ended(self) -> str | None: + return None + + def release(self) -> None: + pass + + def says(self, frame: dict[str, Any]) -> None: + self._in.put_nowait(json.dumps(frame).encode() + b"\n") + + +@pytest.fixture +async def pipe(): + endpoint = _Pipe() + client = AcpClient.over(endpoint, name="taken-over") + yield endpoint, client + await client.close() + + +async def test_an_answer_to_a_request_another_process_sent_is_awaited_here(pipe): + endpoint, client = pipe + waiting = asyncio.ensure_future(client.await_response(7, method="session/prompt", session="s1", timeout=5)) + await asyncio.sleep(0.01) + assert client.prompting, "the awaited prompt is the turn in flight" + + endpoint.says({"jsonrpc": "2.0", "id": 7, "result": {"stopReason": "end_turn"}}) + + assert await waiting == {"stopReason": "end_turn"} + assert endpoint.written == [], "nothing was sent to get it" + assert not client.prompting + + +async def test_an_answer_that_does_not_come_in_time_is_a_timeout_and_a_closed_connection_is_said_so(pipe): + endpoint, client = pipe + with pytest.raises(AcpTimeoutError, match="session/prompt timed out"): + await client.await_response(8, method="session/prompt", timeout=0.05) + + await client.close() + with pytest.raises(AcpConnectionError, match="connection is not open"): + await client.await_response(9, method="session/prompt") + + +async def test_request_ids_go_on_from_where_the_other_process_left_them(pipe): + endpoint, client = pipe + assert client.next_request_id == 1 + client.continue_ids_from(41) + assert client.next_request_id == 41 + client.continue_ids_from(3) + assert client.next_request_id == 41, "never back" + gone_out: list[int] = [] + + asked = asyncio.ensure_future(client.request("ping", {}, timeout=5, sent=gone_out.append)) + await asyncio.sleep(0.01) + endpoint.says({"jsonrpc": "2.0", "id": 41, "result": {}}) + + assert await asked == {} and gone_out == [41] and endpoint.written[0]["id"] == 41 + + +async def test_a_cancelled_prompt_is_not_cancelled_on_the_agent_while_this_process_is_leaving(pipe): + endpoint, client = pipe + endpoint.stays = True + asked = asyncio.ensure_future(client.request("session/prompt", {"sessionId": "s1"}, timeout=5, cancel_session="s1")) + await asyncio.sleep(0.01) + + asked.cancel() + with pytest.raises(asyncio.CancelledError): + await asked + + assert [frame["method"] for frame in endpoint.written] == ["session/prompt"], "no session/cancel went out" + + +def test_leaving_is_said_once_and_can_be_unsaid(): + assert remote.leaving() is False + remote.leave_agents_running() + try: + assert remote.leaving() is True + finally: + remote.staying() + assert remote.leaving() is False + + +async def test_an_answer_awaited_on_a_connection_whose_far_side_ended_is_said_so(): + endpoint = _Pipe() + client = AcpClient.over(endpoint, name="ended") + await endpoint.end() + await asyncio.sleep(0.05) + + with pytest.raises(AcpConnectionError, match="connection ended"): + await client.await_response(3, method="session/prompt") + await client.close() + + +async def test_a_second_adopted_connection_of_a_name_serves_its_pick_up_only(): + from raven.acp_client.pool import close_pool, get_pool + + pool = get_pool() + try: + first, held = await pool.adopt( + name="adopted", endpoint=_Pipe(), initialize={}, launch="k", config="c", next_id=50 + ) + second, held_too = await pool.adopt( + name="adopted", endpoint=_Pipe(), initialize={}, launch="k", config="c", next_id=50 + ) + + assert held is True and held_too is False + assert pool.connections("adopted") == [first] and first.client.next_request_id == 50 + await second.client.close() + finally: + await close_pool() + + +def test_a_run_is_tracked_only_over_a_channel_to_a_machine_and_a_record_that_cannot_be_written_is_named( + tmp_path, monkeypatch +): + from types import SimpleNamespace + + from loguru import logger + + from raven.acp_client import remote_runs + from raven.acp_client.acp_agent import AcpAgentBackend + + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + backend = AcpAgentBackend(name="stub@box", command="stub", machine="box") + connection = SimpleNamespace(initialize={}, launch_key="k", config_key="c") + track = dict(session_id="s", skey="tui:s1", handle="h", task_id="t", session_cwd="/w") + + local = SimpleNamespace(endpoint=SimpleNamespace(pid=1), next_request_id=1) + assert backend._track_on_machine(local, connection, **track) is None, "a child process here is no run on a machine" + + channel = SimpleNamespace(task="stub@box-1", watchers=[], received=0) + over_channel = SimpleNamespace(endpoint=channel, next_request_id=1) + tracker = backend._track_on_machine(over_channel, connection, **track) + assert tracker is not None and channel.watchers == [tracker.received] + assert [run.task for run in remote_runs.pending()] == ["stub@box-1"] + backend._untrack(over_channel, tracker, keep=False) + assert channel.watchers == [] and remote_runs.pending() == [] + backend._untrack(over_channel, tracker, keep=False) # a watcher already gone is not an error + + def refuse(run): + raise OSError("read-only home") + + records: list[str] = [] + sink = logger.add(lambda message: records.append(message.record["message"]), level="WARNING") + try: + monkeypatch.setattr(remote_runs, "Tracker", refuse) + assert backend._track_on_machine(over_channel, connection, **track) is None + finally: + logger.remove(sink) + assert any("could not be recorded: read-only home" in message for message in records) diff --git a/tests/test_node_client.py b/tests/test_node_client.py index d600ba57f..d30de466a 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -662,6 +662,65 @@ async def test_closing_the_link_on_purpose_leaves_the_agent_on_the_machine_for_t await client.close() +@pytest.mark.slow +async def test_while_this_computer_is_leaving_the_endpoint_s_end_leaves_the_agent_with_the_machine(box, pool): + """The host is stopping: closing the connection ends the endpoint only, and the daemon keeps the agent.""" + box.node() + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + client = AcpClient.over(endpoint, name="echo@box") + assert await client.request("ping", {}, timeout=10) == {"echo": {}} + agent = endpoint.pid + + remote.leave_agents_running() + try: + await client.close() + finally: + remote.staying() + + assert endpoint.returncode == node_client.LOST_RC and "left running on task" in (endpoint.ended() or "") + await asyncio.sleep(0.3) + assert not _gone(agent), "the agent stays with the daemon" + assert [(row["task"], row["state"]) for row in await pool.tasks("box")] == [(endpoint.task, "running")] + picked, answer = await pool.attach("box", endpoint.task, name="echo@box", after=endpoint.received) + assert answer["pid"] == agent and answer["rc"] is None + await picked.end() + await _until(lambda: _gone(agent)) + + +@pytest.mark.slow +async def test_a_task_is_picked_up_from_the_frame_last_read_and_the_watchers_hear_each_frame_dealt_with(box, pool): + box.node() + endpoint = await pool.open_channel("box", name="echo@box", command=box.agent(), cwd="~", env={}) + heard: list[int] = [] + endpoint.watchers.append(heard.append) + client = AcpClient.over(endpoint, name="echo@box") + await client.request("ping", {"n": 1}, timeout=10) + await client.request("ping", {"n": 2}, timeout=10) + await asyncio.sleep(0.05) + assert heard == [1, 2] and endpoint.received == 2 + agent = endpoint.pid + await pool.close("box") + + picked, answer = await pool.attach("box", endpoint.task, name="echo@box", after=2) + + assert answer["pid"] == agent and answer["replayed"] == 0 and answer["rc"] is None + again = AcpClient.over(picked, name="echo@box") + again.continue_ids_from(100) + assert await again.request("ping", {"n": 3}, timeout=10) == {"echo": {"n": 3}} + assert picked.received == 3, "numbered on from the frames the first link read" + await again.close() + await _until(lambda: _gone(agent)) + await client.close() + + +@pytest.mark.slow +async def test_picking_up_a_task_the_machine_does_not_know_is_one_sentence(box, pool): + box.node() + + with pytest.raises(NodeError, match=r"machine 'Lab box' \(box\) no longer runs 'echo@box' \(task nothing-here\)"): + await pool.attach("box", "nothing-here", name="echo@box") + + # --- a live node's calls, with the connection stood in for ------------------------- @@ -1047,3 +1106,45 @@ async def test_an_attach_the_link_cannot_carry_parks_every_waiting_endpoint_agai assert len(client.asked) == 1, "the second was not tried on a link that had just failed" assert pool.status("box").state == "lost" and pool.status("box").agents == 2 await pool.close_all() + + +async def test_a_pick_up_the_machine_refuses_or_cannot_carry_is_one_sentence_each(tmp_path): + pool = NodePool(backoff_s=()) + gone = AcpRemoteError(protocol.AGENT_ATTACH, protocol.UNKNOWN_TASK, "no task 't' is running or recorded") + _live(pool, tmp_path, "box", raises=gone) + with pytest.raises(NodeError, match=r"machine 'Lab box' \(box\) no longer runs 'echo@box' \(task t\): no task"): + await pool.attach("box", "t", name="echo@box") + assert pool._nodes["box"].channels == {}, "the channel opened for it is closed again" + + slow = _live(pool, tmp_path, "slow", raises=AcpTimeoutError("no answer", method=protocol.AGENT_ATTACH)) + with pytest.raises(NodeError, match=r"did not answer agent/attach"): + await pool.attach("slow", "t", name="echo@box") + assert not slow.alive, "a link that does not answer is lost" + + dead = _live(pool, tmp_path, "dead", raises=AcpConnectionError("connection ended (exit 255)")) + with pytest.raises(NodeError, match=r"machine 'Lab box' \(dead\)"): + await pool.attach("dead", "t", name="echo@box") + assert not dead.alive + await pool.close_all() + + +async def test_a_watcher_that_fails_is_named_and_the_frames_go_on(tmp_path): + from loguru import logger + + endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + heard: list[int] = [] + + def bad(n: int) -> None: + raise RuntimeError("no disk") + + endpoint.watchers.extend([bad, heard.append]) + records: list[str] = [] + sink = logger.add(lambda message: records.append(message.record["message"]), level="WARNING") + try: + endpoint.feed_frame('{"jsonrpc":"2.0","id":1,"result":{}}', 1) + assert await endpoint.readline() == b'{"jsonrpc":"2.0","id":1,"result":{}}' + endpoint.consumed() + finally: + logger.remove(sink) + + assert heard == [1] and any("a watcher of task" in message and "no disk" in message for message in records) diff --git a/tests/test_remote_resume.py b/tests/test_remote_resume.py new file mode 100644 index 000000000..7727e4842 --- /dev/null +++ b/tests/test_remote_resume.py @@ -0,0 +1,206 @@ +"""raven.acp_client.remote_resume: the reply a turn left on a machine owes, read off the machine's daemon.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest + +from raven.acp_client import remote_resume, remote_runs +from raven.acp_client.acp_agent import AcpAgentBackend +from raven.acp_client.pool import close_pool, get_pool +from raven.ops import remote_path +from tests.test_acp_client_resume import _Pipe +from tests.test_node_client import row + +HOST, PORT = "203.0.113.7", 58717 + + +@pytest.fixture(autouse=True) +def _machines(tmp_path, monkeypatch): + registry = tmp_path / "connections.json" + registry.write_text(json.dumps({"connections": [row()]}), encoding="utf-8") + monkeypatch.setenv("RAVEN_CONNECTIONS", str(registry)) + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + monkeypatch.setattr(remote_path, "_NODES", None) + + +@pytest.fixture +async def _pool(): + yield get_pool() + await close_pool() + + +class _Link: + """A node pool with one task, whose attach hands over a pipe the test speaks the agent's part into.""" + + def __init__(self, *, tasks: list[dict[str, Any]] | None = None, rc: int | None = None) -> None: + self.rows = tasks if tasks is not None else [{"task": "stub@box-1", "state": "running"}] + self.rc = rc + self.pipe = _Pipe() + self.attached: list[tuple[str, str, int]] = [] + self.refuse: Exception | None = None + + async def tasks(self, machine_id: str) -> list[dict[str, Any]]: + if isinstance(self.rows, Exception): + raise self.rows + return self.rows + + async def attach(self, machine_id: str, task: str, *, name: str, after: int = 0): + if self.refuse is not None: + raise self.refuse + self.attached.append((machine_id, task, after)) + return self.pipe, {"pid": 7, "frames": 9, "replayed": 0, "rc": self.rc} + + +def _run(**kw) -> remote_runs.RemoteRun: + fields = dict( + machine="box", + task="stub@box-1", + agent="stub@box", + session_key="tui:s1", + handle="t1", + task_id="t1", + session_id="sess-1", + prompt_id=3, + next_id=4, + initialize={}, + launch_key="k", + config_key="c", + session_cwd="/home/worker/raven-work/t1", + stateful=False, + received=5, + ) + fields.update(kw) + return remote_runs.RemoteRun(**fields) + + +def _backend() -> AcpAgentBackend: + return AcpAgentBackend(name="stub@box", command="stub", machine="box") + + +def _chunk(text: str, session: str = "sess-1") -> dict[str, Any]: + update = {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": text}} + return {"jsonrpc": "2.0", "method": "session/update", "params": {"sessionId": session, "update": update}} + + +async def test_the_reply_is_read_off_the_frames_the_other_process_never_saw_and_the_agent_is_the_pool_s(_pool): + link = _Link() + picking = asyncio.ensure_future(remote_resume.pick_up(_run(), _backend(), pool=link)) + await asyncio.sleep(0.05) + assert link.attached == [("box", "stub@box-1", 5)], "from the frame the other process last read" + link.pipe.says(_chunk("po")) + link.pipe.says(_chunk("ng")) + link.pipe.says(_chunk("not mine", session="other")) + link.pipe.says({"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "end_turn"}}) + + picked = await asyncio.wait_for(picking, timeout=5) + + assert picked.status == "ok" and picked.stop_reason == "end_turn" + assert picked.reply.startswith("pong\n\n[raven] Ran on machine 'Lab box' (box), in scp://worker@") + assert ( + "picked up after this computer restarted" in picked.reply and "deliver_files fetches one by it" in picked.reply + ) + (connection,) = _pool.connections("stub@box") + assert connection.client.endpoint is link.pipe and connection.client.next_request_id == 4 + remote_resume._ID_ROOM + assert connection.initialize == {} and connection.launch_key == "k" + + +async def test_a_turn_cut_short_is_marked_partial_as_a_turn_ended_here_would_be(_pool): + link = _Link() + picking = asyncio.ensure_future(remote_resume.pick_up(_run(), _backend(), pool=link)) + await asyncio.sleep(0.05) + link.pipe.says(_chunk("half")) + link.pipe.says({"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "max_tokens"}}) + + picked = await asyncio.wait_for(picking, timeout=5) + + assert picked.status == "ok" and picked.reply.startswith("half\n\n[raven] stub@box stopped before finishing") + + +async def test_a_machine_that_cannot_be_asked_or_a_link_that_cannot_attach_keeps_the_record(_pool): + link = _Link(tasks=remote_path.NodeCallError("the Raven node on machine 'Lab box' (box) did not answer")) + with pytest.raises(remote_resume.UnreachableError, match="did not answer"): + await remote_resume.pick_up(_run(), _backend(), pool=link) + + link = _Link() + link.refuse = remote_path.NodeCallError("the Raven node on machine 'Lab box' (box) did not answer agent/attach") + with pytest.raises(remote_resume.UnreachableError, match="agent/attach"): + await remote_resume.pick_up(_run(), _backend(), pool=link) + + with pytest.raises(remote_resume.UnreachableError, match="does not reach registered machines"): + await remote_resume.pick_up(_run(), _backend(), pool=None) + + +async def test_a_run_the_machine_no_longer_has_or_never_got_a_prompt_for_is_an_error_without_an_attach(_pool): + link = _Link(tasks=[]) + picked = await remote_resume.pick_up(_run(), _backend(), pool=link) + assert picked.status == "error" and "no longer has the run of 'stub@box' (task stub@box-1)" in picked.reply + + link = _Link() + picked = await remote_resume.pick_up(_run(prompt_id=0), _backend(), pool=link) + assert picked.status == "error" and "ended before its prompt was sent" in picked.reply + assert link.attached == [] + + link = _Link() + link.refuse = remote_path.NodeCallError("machine 'Lab box' (box) no longer runs 'stub@box' (task stub@box-1): gone") + picked = await remote_resume.pick_up(_run(), _backend(), pool=link) + assert picked.status == "error" and picked.reply.startswith("machine 'Lab box' (box) no longer runs") + + +async def test_an_unregistered_machine_is_an_error_in_one_sentence(_pool): + picked = await remote_resume.pick_up(_run(machine="nobox"), _backend(), pool=_Link()) + + assert picked.status == "error" and "cannot be picked up" in picked.reply and "nobox" in picked.reply + + +async def test_an_agent_that_ended_while_this_computer_was_away_is_said_so_with_its_exit(_pool): + link = _Link(rc=137) + picking = asyncio.ensure_future(remote_resume.pick_up(_run(), _backend(), pool=link)) + await asyncio.sleep(0.05) + link.pipe.says(_chunk("started")) + await link.pipe.end() + + picked = await asyncio.wait_for(picking, timeout=5) + + assert picked.status == "error" + assert picked.reply.startswith( + "the run of 'stub@box' on machine 'Lab box' (box) ended while this computer was away (exit 137)" + ) + + +async def test_an_answer_that_never_comes_is_waited_out_and_said_so(_pool, monkeypatch): + monkeypatch.setattr(remote_resume, "_prompt_deadline", lambda configured: 0.1) + picked = await asyncio.wait_for(remote_resume.pick_up(_run(), _backend(), pool=_Link()), timeout=5) + + assert picked.status == "error" and "was waited out" in picked.reply + + +async def test_a_turn_that_said_nothing_is_an_error_that_names_where_its_frames_are(_pool): + link = _Link() + picking = asyncio.ensure_future(remote_resume.pick_up(_run(), _backend(), pool=link)) + await asyncio.sleep(0.05) + link.pipe.says({"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "end_turn"}}) + + picked = await asyncio.wait_for(picking, timeout=5) + + assert picked.status == "error" and "ended its turn with no content" in picked.reply + assert "task stub@box-1" in picked.reply + + +async def test_a_pick_up_beside_a_live_connection_of_the_name_closes_its_own_afterwards(_pool): + live, _ = await _pool.adopt(name="stub@box", endpoint=_Pipe(), initialize={}, launch="k", config="c", next_id=1) + link = _Link() + picking = asyncio.ensure_future(remote_resume.pick_up(_run(), _backend(), pool=link)) + await asyncio.sleep(0.05) + link.pipe.says(_chunk("pong")) + link.pipe.says({"jsonrpc": "2.0", "id": 3, "result": {"stopReason": "end_turn"}}) + + picked = await asyncio.wait_for(picking, timeout=5) + + assert picked.status == "ok" + assert _pool.connections("stub@box") == [live], "the live connection stays the pool's" + assert link.pipe.returncode == 0, "the pick-up's own connection was closed" diff --git a/tests/test_remote_runs.py b/tests/test_remote_runs.py new file mode 100644 index 000000000..5c7c7270a --- /dev/null +++ b/tests/test_remote_runs.py @@ -0,0 +1,132 @@ +"""raven.acp_client.remote_runs: what this computer keeps of a run on a machine, while it is in flight.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from raven.acp_client import remote_runs + + +@pytest.fixture(autouse=True) +def _home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + return tmp_path / "home" + + +@pytest.fixture +def logs(): + """What raven logged while the test ran, as loguru's sink hears it.""" + from loguru import logger + + records: list[str] = [] + sink = logger.add(lambda message: records.append(message.record["message"]), level="INFO") + try: + yield records + finally: + logger.remove(sink) + + +def _run(task: str = "echo@box-1a2b3c4d", **kw) -> remote_runs.RemoteRun: + fields = dict( + machine="box", + task=task, + agent="echo@box", + session_key="tui:s1", + handle="t1", + task_id="t1", + session_id="sess-1", + prompt_id=3, + next_id=4, + initialize={"agentCapabilities": {}}, + launch_key="k1", + config_key="c1", + session_cwd="/root/raven-work/t1", + stateful=False, + ) + fields.update(kw) + return remote_runs.RemoteRun(**fields) + + +def test_a_record_is_written_owner_only_and_read_back_whole(_home: Path) -> None: + path = remote_runs.record(_run(received=7, started=1.5)) + + assert path == _home / "machines" / "runs" / "echo@box-1a2b3c4d.json" + assert path.stat().st_mode & 0o777 == 0o600 and path.parent.stat().st_mode & 0o777 == 0o700 + (back,) = remote_runs.pending() + assert back == _run(received=7, started=1.5) + assert json.loads(path.read_text(encoding="utf-8"))["version"] == remote_runs.VERSION + + +def test_pending_lists_the_runs_oldest_first_and_names_what_it_cannot_read(_home: Path, logs) -> None: + remote_runs.record(_run("later", started=20.0)) + remote_runs.record(_run("earlier", started=10.0)) + (remote_runs.runs_dir() / "broken.json").write_text("{not json", encoding="utf-8") + (remote_runs.runs_dir() / "other-version.json").write_text(json.dumps({"version": 99}), encoding="utf-8") + (remote_runs.runs_dir() / "bare.json").write_text(json.dumps({"version": 1, "machine": "box"}), encoding="utf-8") + + assert [run.task for run in remote_runs.pending()] == ["earlier", "later"] + assert "broken.json could not be read" in "\n".join(logs) + assert "other-version.json is not one this version reads" in "\n".join(logs) + + +def test_forget_removes_the_record_and_is_quiet_about_one_already_gone() -> None: + remote_runs.record(_run()) + + remote_runs.forget("echo@box-1a2b3c4d") + remote_runs.forget("echo@box-1a2b3c4d") + + assert remote_runs.pending() == [] + + +def test_the_tracker_keeps_the_prompt_s_id_and_the_frames_read_current_and_removes_the_record_when_the_turn_ends() -> ( + None +): + tracker = remote_runs.Tracker(_run(prompt_id=0, next_id=1)) + assert remote_runs.pending()[0].prompt_id == 0 + + tracker.prompt_sent(5) + tracker.received(3) + tracker.received(2) # a frame counted twice is not a step back + + (kept,) = remote_runs.pending() + assert (kept.prompt_id, kept.next_id, kept.received) == (5, 6, 3) + tracker.close(keep=False) + assert remote_runs.pending() == [] and tracker.kept is False + + +def test_a_tracker_closed_to_keep_leaves_the_record_for_the_next_start() -> None: + tracker = remote_runs.Tracker(_run()) + + tracker.close(keep=True) + + assert [run.task for run in remote_runs.pending()] == ["echo@box-1a2b3c4d"] and tracker.kept is True + + +def test_a_record_that_cannot_be_updated_is_named_not_raised(logs, monkeypatch) -> None: + tracker = remote_runs.Tracker(_run()) + + def refuse(run): + raise OSError("disk full") + + monkeypatch.setattr(remote_runs, "record", refuse) + tracker.received(9) + tracker.prompt_sent(9) + + assert "\n".join(logs).count("could not be updated: disk full") == 2 + + +def test_a_dispatch_left_on_its_machine_is_read_as_left_once() -> None: + remote_runs.mark_left("t9") + + assert remote_runs.was_left("t9") is True + assert remote_runs.was_left("t9") is False + assert remote_runs.was_left("never") is False + + +def test_the_directory_is_made_under_the_raven_home(_home: Path) -> None: + assert remote_runs.runs_dir() == _home / "machines" / "runs" + assert os.path.isdir(_home / "machines" / "runs") diff --git a/tests/test_resume_remote.py b/tests/test_resume_remote.py new file mode 100644 index 000000000..81db61abc --- /dev/null +++ b/tests/test_resume_remote.py @@ -0,0 +1,278 @@ +"""raven.agent.subagent.resume_remote: the runs a process before this one left on machines, concluded here.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from raven.acp_client import remote_resume, remote_runs +from raven.acp_client.acp_agent import AcpAgentBackend +from raven.agent.subagent import resume_remote +from raven.agent.subagent.history import SpawnRecord, nodes_root + + +@pytest.fixture(autouse=True) +def _home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + monkeypatch.setattr(resume_remote, "_SCHEDULED", False) + return tmp_path / "home" + + +@pytest.fixture +def logs(): + """What raven logged while the test ran, as loguru's sink hears it.""" + from loguru import logger + + records: list[str] = [] + sink = logger.add(lambda message: records.append(message.record["message"]), level="INFO") + try: + yield records + finally: + logger.remove(sink) + + +class _Registry: + def __init__(self) -> None: + self.committed: list[tuple[str, str, str, str, str]] = [] + self.statuses: list[tuple[str, str, str, str]] = [] + + async def commit(self, session_key, agent, handle, agent_id, *, kind="cli"): + self.committed.append((session_key, agent, handle, agent_id, kind)) + + async def upsert_spawn(self, session_key, agent, handle, status, agent_id=None): + self.statuses.append((session_key, agent, handle, status)) + + +class _Manager: + def __init__(self, backend: Any, session_dir: Path) -> None: + self.registry = SimpleNamespace( + backend=lambda name: backend if backend is not None and name == backend.name else None + ) + self._session_dir = session_dir + self.announced: list[dict[str, Any]] = [] + self.finalized: list[str] = [] + self.remembered: list[str] = [] + + def session_dir_for(self, session_key: str) -> Path: + return self._session_dir + + async def _finalize_node_claim(self, session_key, record) -> None: + self.finalized.append(record.node_id) + + def remember_session(self, session_key: str) -> None: + self.remembered.append(session_key) + + async def _announce_result(self, task_id, summary, task, result, origin, status, record_path=None, activity=None): + self.announced.append( + { + "task_id": task_id, + "summary": summary, + "task": task, + "result": result, + "origin": origin, + "status": status, + "record": record_path, + } + ) + + +def _backend(name: str = "stub@box", machine: str = "box") -> AcpAgentBackend: + return AcpAgentBackend(name=name, command="stub", machine=machine) + + +def _run(**kw) -> remote_runs.RemoteRun: + fields = dict( + machine="box", + task="stub@box-1a2b3c4d", + agent="stub@box", + session_key="tui:s1", + handle="t1", + task_id="t1", + session_id="sess-1", + prompt_id=3, + next_id=4, + initialize={}, + launch_key="k", + config_key="c", + session_cwd="/root/raven-work/t1", + stateful=False, + ) + fields.update(kw) + run = remote_runs.RemoteRun(**fields) + remote_runs.record(run) + return run + + +def _open_record(session_dir: Path, *, agent: str = "stub@box", handle: str = "t1", node_id: str = "n1") -> SpawnRecord: + meta = {"call_id": node_id, "session_key": "tui:s1", "agent": agent, "handle": handle, "task_summary": "say hi"} + return SpawnRecord.open(session_dir, task_id="t1", task="say hi to the machine", meta=meta, node_id=node_id) + + +async def test_a_picked_up_reply_finishes_the_dispatch_s_record_and_is_announced_to_its_conversation( + tmp_path, monkeypatch +): + backend = _backend() + manager = _Manager(backend, tmp_path / "s1") + registry = _Registry() + monkeypatch.setattr(resume_remote, "get_registry", lambda: registry) + _open_record(tmp_path / "s1") + run = _run(stateful=True) + + async def picked(got_run, got_backend, *, pool=None): + assert got_run == run and got_backend is backend + return remote_resume.PickedUp( + "ok", "pong\n\n[raven] Ran on machine 'Lab box' (box), picked up after this computer restarted." + ) + + monkeypatch.setattr(remote_resume, "pick_up", picked) + + assert await resume_remote.resume_remote_runs(manager) == 1 + + meta = json.loads((nodes_root(tmp_path / "s1") / "n1.meta.json").read_text(encoding="utf-8")) + assert meta["status"] == "completed" and meta["agent"] == "stub@box" + assert (nodes_root(tmp_path / "s1") / "n1.out.md").read_text(encoding="utf-8").startswith("pong") + assert manager.finalized == ["n1"] and manager.remembered == ["tui:s1"] + assert registry.committed == [("tui:s1", "stub@box", "t1", "sess-1", "acp")] + (said,) = manager.announced + assert said["status"] == "ok" and said["result"].startswith("pong") and said["task"] == "say hi to the machine" + assert ( + said["origin"] == {"session_key": "tui:s1", "channel": "tui", "chat_id": "s1"} and said["summary"] == "say hi" + ) + assert said["record"].endswith("n1.out.md") + assert remote_runs.pending() == [] + + +async def test_a_run_that_could_not_be_picked_up_is_announced_as_an_error_and_its_record_closed(tmp_path, monkeypatch): + backend = _backend() + manager = _Manager(backend, tmp_path / "s1") + registry = _Registry() + monkeypatch.setattr(resume_remote, "get_registry", lambda: registry) + _open_record(tmp_path / "s1") + _run() + + async def picked(run, backend, *, pool=None): + return remote_resume.PickedUp("error", "machine 'Lab box' (box) no longer has the run") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + + await resume_remote.resume_remote_runs(manager) + + meta = json.loads((nodes_root(tmp_path / "s1") / "n1.meta.json").read_text(encoding="utf-8")) + assert meta["status"] == "failed" + assert (nodes_root(tmp_path / "s1") / "n1.error.md").read_text(encoding="utf-8").startswith("Error: machine") + assert registry.statuses == [("tui:s1", "stub@box", "t1", "failed")] and registry.committed == [] + (said,) = manager.announced + assert said["status"] == "error" and said["result"].startswith("Error: machine") + assert remote_runs.pending() == [] + + +async def test_a_machine_that_cannot_be_asked_keeps_the_record_for_a_later_start(tmp_path, monkeypatch, logs): + manager = _Manager(_backend(), tmp_path / "s1") + _run() + + async def picked(run, backend, *, pool=None): + raise remote_resume.UnreachableError("the Raven node on machine 'Lab box' (box) did not answer") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + + assert await resume_remote.resume_remote_runs(manager) == 0 + + assert [run.task for run in remote_runs.pending()] == ["stub@box-1a2b3c4d"] + assert manager.announced == [] and "waits for a later start" in "\n".join(logs) + + +async def test_a_run_whose_agent_left_the_table_is_concluded_as_an_error_without_the_machine(tmp_path, monkeypatch): + registry = _Registry() + monkeypatch.setattr(resume_remote, "get_registry", lambda: registry) + asked: list[Any] = [] + + async def picked(run, backend, *, pool=None): + asked.append(run) + return remote_resume.PickedUp("ok", "never") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + _run() + for manager in (_Manager(None, tmp_path / "s1"), _Manager(_backend(machine="other"), tmp_path / "s1")): + remote_runs.record(_run()) + await resume_remote.resume_remote_runs(manager) + (said,) = manager.announced + assert ( + said["status"] == "error" + and "no longer on the agent table as an acp agent on that machine" in said["result"] + ) + assert asked == [] and remote_runs.pending() == [] + + +async def test_a_failure_inside_the_pick_up_is_the_run_s_own_and_the_next_run_is_tried(tmp_path, monkeypatch): + backend = _backend() + manager = _Manager(backend, tmp_path / "s1") + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + _run(task="first", started=1.0) + _run(task="second", started=2.0) + + async def picked(run, backend, *, pool=None): + if run.task == "first": + raise RuntimeError("boom") + return remote_resume.PickedUp("ok", "pong") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + + assert await resume_remote.resume_remote_runs(manager) == 2 + + assert [said["status"] for said in manager.announced] == ["error", "ok"] + assert "could not be picked up: boom" in manager.announced[0]["result"] + + +async def test_a_run_with_no_conversation_to_report_to_is_closed_quietly(tmp_path, monkeypatch, logs): + backend = _backend() + manager = _Manager(backend, tmp_path / "s1") + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + _run(session_key="default") + + async def picked(run, backend, *, pool=None): + return remote_resume.PickedUp("ok", "pong") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + + await resume_remote.resume_remote_runs(manager) + + assert manager.announced == [] and "has no conversation to report to" in "\n".join(logs) + + +async def test_the_pick_up_is_scheduled_once_per_process_after_a_delay_and_not_for_a_stand_in_manager( + tmp_path, monkeypatch +): + ran: list[Any] = [] + + async def fake(manager): + ran.append(manager) + return 0 + + monkeypatch.setattr(resume_remote, "resume_remote_runs", fake) + manager = _Manager(_backend(), tmp_path / "s1") + + assert resume_remote.schedule_remote_resume(SimpleNamespace()) is None, ( + "a stand-in without sessions schedules nothing" + ) + task = resume_remote.schedule_remote_resume(manager, delay_s=0.01) + assert task is not None and resume_remote.schedule_remote_resume(manager, delay_s=0.01) is None + await asyncio.wait_for(task, timeout=5) + + assert ran == [manager] + + +async def test_a_pick_up_that_raises_in_the_background_is_logged_not_raised(tmp_path, monkeypatch, logs): + async def fail(manager): + raise RuntimeError("no machines today") + + monkeypatch.setattr(resume_remote, "resume_remote_runs", fail) + task = resume_remote.schedule_remote_resume(_Manager(_backend(), tmp_path / "s1"), delay_s=0.01) + + await asyncio.wait_for(task, timeout=5) + + assert "picking up the runs left on machines failed" in "\n".join(logs) diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index cbb74dd9f..db5aaf1d8 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -17,14 +17,16 @@ import sys import time from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest +from raven.acp_client import remote, remote_runs from raven.acp_client.capabilities import snapshot_fingerprint, verify_agent from raven.acp_client.pool import close_pool, get_pool from raven.acp_client.protocol import AcpConnectionError -from raven.agent.subagent import probe +from raven.agent.subagent import probe, resume_remote from raven.agent.subagent.backends import ( agent_meta, build_third_party_backend, @@ -32,6 +34,7 @@ session_mcp_effective, ) from raven.agent.subagent.backends import env as backend_env +from raven.agent.subagent.history import SpawnRecord, nodes_root from raven.agent.subagent.mcp_grant import GrantedServer, McpGrant from raven.agent.subagent.probe_state import fingerprint from raven.agent.subagent.registry import _row_for @@ -446,6 +449,109 @@ async def test_a_link_that_goes_mid_turn_is_waited_out_and_the_turn_finishes_whe assert len([argv for argv in box.calls() if "-T" in argv]) == 2 +def _alive(pid: int | None) -> bool: + if pid is None: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + +class _Manager: + """What the pick-up asks of the sub-agent manager, recorded.""" + + def __init__(self, backend: Any, session_dir: Path) -> None: + self.registry = SimpleNamespace(backend=lambda name: backend if name == backend.name else None) + self.session_dir = session_dir + self.announced: list[dict[str, Any]] = [] + self.finalized: list[str] = [] + + def session_dir_for(self, session_key: str) -> Path: + return self.session_dir + + async def _finalize_node_claim(self, session_key: str, record: Any) -> None: + self.finalized.append(record.node_id) + + def remember_session(self, session_key: str) -> None: + pass + + async def _announce_result(self, task_id, summary, task, result, origin, status, record_path=None, activity=None): + self.announced.append({"task_id": task_id, "result": result, "status": status, "origin": origin}) + + +class _Registry: + def __init__(self) -> None: + self.rows: list[tuple[str, ...]] = [] + + async def commit(self, session_key, agent, handle, agent_id, *, kind="cli"): + self.rows.append(("commit", session_key, agent, handle, agent_id)) + + async def upsert_spawn(self, session_key, agent, handle, status, agent_id=None): + self.rows.append(("status", session_key, agent, handle, status)) + + +async def test_a_run_left_on_the_machine_when_this_computer_stops_is_picked_up_by_the_next_start( + box, tmp_path, monkeypatch +): + """The agent stays with the machine's daemon when this process stops; the next one reads its answer and reports it.""" + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + backend = build_third_party_backend(remote_config(env={"ACP_STUB_PROMPT_DELAY_S": "3"})) + turn = asyncio.ensure_future( + backend.run("hi", task_id="t1", workspace=tmp_path, executor=None, session_key="tui:s1") + ) + deadline = time.monotonic() + 20 + while not any(run.prompt_id > 0 for run in remote_runs.pending()): + assert time.monotonic() < deadline and not turn.done(), "the prompt never went out" + await asyncio.sleep(0.05) + agent = _agent_pid() + (run,) = remote_runs.pending() + assert (run.machine, run.agent, run.session_key, run.handle, run.task_id) == ( + "box", + "stub@box", + "tui:s1", + "t1", + "t1", + ) + assert run.session_id and run.initialize and run.next_id == run.prompt_id + 1 + + # This computer stops: the turn is cancelled the way the shutdown does it, then the pools are closed. + remote.leave_agents_running() + try: + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await turn + assert remote_runs.was_left("t1"), "the manager would read this dispatch as left, not cancelled" + await close_pool() + await box.nodes.close_all() + finally: + remote.staying() + assert _alive(agent), "the agent stays with the machine's daemon" + assert [kept.task for kept in remote_runs.pending()] == [run.task] + + # The next start: the dispatch's record is still open under its session, as the manager leaves one. + session_dir = tmp_path / "sessions" / "s1" + meta = {"call_id": "n1", "session_key": "tui:s1", "agent": backend.name, "handle": "t1", "task_summary": "say hi"} + SpawnRecord.open(session_dir, task_id="t1", task="hi", meta=meta, node_id="n1") + manager = _Manager(backend, session_dir) + registry = _Registry() + monkeypatch.setattr(resume_remote, "get_registry", lambda: registry) + + assert await asyncio.wait_for(resume_remote.resume_remote_runs(manager), timeout=30) == 1 + + (said,) = manager.announced + assert said["status"] == "ok" and said["task_id"] == "t1" and said["origin"]["chat_id"] == "s1" + assert said["result"].split("\n\n[raven]")[0] == "pong", "the agent's answer, read after the restart" + assert "picked up after this computer restarted" in said["result"] + meta_back = json.loads((nodes_root(session_dir) / "n1.meta.json").read_text(encoding="utf-8")) + assert meta_back["status"] == "completed" and manager.finalized == ["n1"] + assert (nodes_root(session_dir) / "n1.out.md").read_text(encoding="utf-8").startswith("pong") + assert registry.rows == [("status", "tui:s1", "stub@box", "t1", "completed")] + assert remote_runs.pending() == [] + assert _agent_pid() == agent, "the agent's process is the pool's connection again" + + async def test_a_daemon_that_stops_mid_turn_fails_the_turn_in_one_sentence(box, tmp_path): box.nodes._heartbeat_s = 0.1 box.nodes._lost_after_s = 1.0 diff --git a/tests/test_subagent_manager.py b/tests/test_subagent_manager.py index a13f636c3..4cca140e4 100644 --- a/tests/test_subagent_manager.py +++ b/tests/test_subagent_manager.py @@ -5178,3 +5178,60 @@ def bind_pin(self, model, provider_name=None): mgr.registry.backend("local")._pin() assert asked == [("deepseek-ai/DeepSeek-V3", None), ("mylocal/m", "mylocal")] + + +async def test_a_dispatch_left_running_on_its_machine_when_the_host_stops_is_neither_finished_nor_announced( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Shutdown cancels every run; one on a machine goes on under its daemon, and the next host picks it up.""" + from raven.acp_client import remote_runs + from raven.agent.subagent.history import nodes_root + from raven.agent.subagent.spawn_tool import SpawnTool + from raven.config.schema import ThirdPartyCliSubagentConfig + from raven.session.manager import SessionManager + + home = tmp_path / "home" + submitted: list[Any] = [] + mgr = SubagentManager( + provider=_StubProvider(), + workspace=home, + session_dir=lambda key: SessionManager(home).session_dir(key), + max_concurrent=2, + agents=[ThirdPartyCliSubagentConfig(name="Remote", command="cat {prompt_file}")], + ) + mgr.set_submit(submitted.append) + finalized: list[str] = [] + + async def finalize(session_key: Any, record: Any) -> None: + finalized.append(record.node_id) + + monkeypatch.setattr(mgr, "_finalize_node_claim", finalize) + started = asyncio.Event() + + class _OnMachine: + async def run(self, task: str, **kwargs: Any) -> str: + started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + # What the acp backend does for a run on a machine while the host is leaving. + remote_runs.mark_left(kwargs["task_id"]) + raise + return "never" + + mgr.registry._backends["Remote"] = _OnMachine() + monkeypatch.setattr(manager_mod, "build_executor", lambda *a, **k: _DummyExecutor()) + tool = SpawnTool(manager=mgr) + tool.set_context("cli", "direct", "cli:direct") + await tool.execute(node_id="r1", task_summary="Run on the machine", task="Go.", subagent="Remote") + await asyncio.wait_for(started.wait(), timeout=5) + + assert await mgr.cancel_all(reason="the gateway stopped") == 1 + + nodes = nodes_root(SessionManager(home).session_dir("cli:direct")) + meta = json.loads((nodes / "r1.meta.json").read_text(encoding="utf-8")) + assert meta["status"] == "running" and "ended_at_ms" not in meta, ( + "the record stays open for the host that picks it up" + ) + assert not (nodes / "r1.out.md").exists() and not (nodes / "r1.error.md").exists() + assert submitted == [] and finalized == [], "nothing is announced or closed here" From 00c671d9e16c0e796459f557df9331671cd53776 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:50:25 +0800 Subject: [PATCH 18/44] test(agent): the probe listing test waits for the agent's exit to reach the link The agent's exit reaches the link a beat after the close that asked for it, and on a slow runner the listing read the link before it did: the test waits for the link to read idle, as the listing does. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_subagent_acp_remote.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index db5aaf1d8..9d0e82948 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -424,6 +424,10 @@ async def test_the_free_probe_shows_the_link_s_state_once_a_dispatch_has_opened_ assert "runs on machine 'Lab box' (box) (connected, 1 agent running)" in result.detail await close_pool() + # The agent's exit reaches the link a beat after the close that asked for it; the listing reads the link. + deadline = time.monotonic() + 10 + while box.nodes.status("box").agents and time.monotonic() < deadline: + await asyncio.sleep(0.05) assert "(connected, idle)" in (await probe.probe_one(cfg, source="config")).detail From bdd5f55d15bdbb84234419e7923e7efc1831b1c9 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:15:41 +0800 Subject: [PATCH 19/44] feat(*): run a sub-agent of every kind on a registered machine, not only an acp one A cli, openai or builtin row that named a machine lost the key on load without a word and ran on this computer, since only the acp config declared it. Now every kind carries `machine` and runs there its own way: - cli: the command is started on the machine by its Raven node, in the run's directory there (`/`), and read back over the link. The prompt reaches it the three ways it does here -- the file is written there under the run's `.raven/`, stdin travels on the link as the new `agent/stdin` (protocol 4), which also closes it, since the node opens a stdin for every agent and a command that reads to the end would wait. MCP servers are not lent (the file there names none, the reply says so), the files that came with the task are named by address, this computer's shell, home and model binding stay here, and a cancel or timeout ends the process tree there. - openai: each turn opens an ssh tunnel to the machine (`-N -L`, held for the turn) and posts through it, with `Host` as written and no proxy; http only, refused in one sentence before any ssh otherwise; `remoteCwd` is refused on write and dropped with a warning on load. - builtin: declared so a row naming one is told loudly and cleared; Raven's own loop runs here and reaches a machine as the main loop does. `agent_meta` reads the machine off every kind, so the roster tag, the registry's lane gate and the dispatch agree; the free probe of a cli or openai row on a machine reaches nothing and says to run a test, which runs there; `subagents.add` takes any preset but a built-in one, with `remote_cwd` refused for an openai one; `raven_config` lists the machines for every kind. With it, the review's three notes: a machine row's direct-chat attachments are kept for the backend to name by address and the page's bullets are spelled the same way, with the comment and docstring saying so; `raven_config` says exec can look on the machine instead of saying nothing reaches it; a failed run's announce no longer says its machine directory was made. `AgentMeta.machine` now says what stays out of a model's view (the way in) and what does not (a path, as scp writes it), and the CONTEXT.md Machine, node, Roster and working-directory entries describe every kind and no longer name the page's add or `deliver_files`. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 48 +- raven/acp_client/remote.py | 117 +++- raven/agent/subagent/backends/__init__.py | 25 +- raven/agent/subagent/backends/cli_agent.py | 320 ++++++++++- raven/agent/subagent/backends/openai_api.py | 115 ++-- raven/agent/subagent/manager.py | 29 +- raven/agent/subagent/probe.py | 38 +- raven/agent/tools/raven_config.py | 22 +- raven/config/schema.py | 121 +++- raven/config/update_subagents.py | 9 + raven/node/client.py | 52 +- raven/node/protocol.py | 17 +- raven/node/server.py | 26 +- raven/rpc/methods/subagents.py | 34 +- raven/rpc/models.py | 16 +- rpc-schema/openrpc.json | 6 +- schemas/subagent.schema.json | 24 + tests/test_node_client.py | 45 ++ tests/test_node_server.py | 39 ++ tests/test_raven_config_tool.py | 11 +- tests/test_rpc_subagents.py | 27 +- tests/test_subagent_acp_remote.py | 7 + tests/test_subagent_cli_remote.py | 575 ++++++++++++++++++++ tests/test_subagent_direct_chat.py | 44 ++ ui-tui/src/rpc/generated.ts | 2 +- ui-web/src/rpc/generated.ts | 2 +- 26 files changed, 1600 insertions(+), 171 deletions(-) create mode 100644 tests/test_subagent_cli_remote.py diff --git a/CONTEXT.md b/CONTEXT.md index 524eeee65..5ef4ff5f5 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -381,11 +381,15 @@ keeping the registry and handing it over: a launcher points the agent's `RAVEN_CONNECTIONS` at the owner's store rather than copying rows into the agent's own home, and a sub-agent inherits `RAVEN_HOME` so it resolves the same file. Which machine a job lands on, and whether it can run at all, is -settled inside the agent that runs it, with one exception: an `acp` agent entry -that names a `machine` is started there by the machine's **Raven node**, reached over -the one ssh link the host keeps to the machine (`raven/node/client.py`) -- a link that -may go and come back while the agent runs on; the row's -address, port, key and user are read for that link alone (`raven/acp_client/remote.py`). +settled inside the agent that runs it, with one exception: an agent entry that +names a `machine`, of any kind. An `acp` or `cli` entry is started there by the +machine's **Raven node**, reached over the one ssh link the host keeps to the machine +(`raven/node/client.py`) -- a link that may go and come back while the agent runs on; +an `openai` entry's endpoint is reached from that machine through an ssh tunnel the +host opens for the turn and closes after it (`remote.tunnel`); a `builtin` entry cannot +name one -- Raven's own loop runs here -- and is loaded with a warning and runs here. +The row's address, port, key and user are read for that link or tunnel alone +(`raven/acp_client/remote.py`). Its dispatch reads the registry once (`remote.machine`), and the run fails before anything starts when the id is not registered, is listed twice, cannot be used, or is this computer; the `spawn` itself has already answered that it started. No other @@ -393,10 +397,11 @@ dispatch reads the registry, so no graph and no spawn is ever refused over the s of it. The address stays in the ssh line, in memory; the entry names only the machine's id, and the entry's `env` reaches the agent on the link, never on a command line. Such an entry is added the way any preset is, with `machine` (and -`remote_cwd`) on `subagents.add` — from the page or from a chat through `raven_config`, -which lists the registered machines while the lane is on — refused before anything -is spent when the lane is off or the id is not usable, and proved by the same -readiness ping as a local add, run on that machine through its node. +`remote_cwd`, for an `acp` or `cli` one) on `subagents.add` — from a chat through +`raven_config`, which lists the registered machines while the lane is on; the page's +add passes no machine yet — refused before anything is spent when the lane is off, +the id is not usable or the preset is a built-in agent, and proved by the same +readiness ping as a local add, run on that machine. _Avoid_: "host" / "server" / "node" (too broad, no link to the `ops connection` registry that supplies the rows); "GPU box" (only some are GPU hosts, and the term covers any registered compute destination); @@ -414,10 +419,12 @@ socket of the user's -- and leaves with the ssh; the daemon leaves on its own once idle). It does three things there and nothing else: answers file calls with Raven's own file tools (`read_file`, `list_dir`, `find`, `grep`, `write_file`, `edit_file`) fenced to the directories each call names, so a read of 50 lines of a large file moves -those 50 lines, and fetches one file whole (`fs/pull`); starts the `acp` agents whose -entries name the machine, each under a task id of the host's, in its own process group in -the machine user's login environment, forwards their ACP frames on numbered channels of -the connection that holds them and keeps those frames in the task's record +those 50 lines, and fetches one file whole (`fs/pull`); starts the `acp` and `cli` agents +whose entries name the machine, each under a task id of the host's, in its own process +group in the machine user's login environment, forwards what they write (an ACP agent's +frames, a CLI agent's output lines) on numbered channels of the connection that holds them +-- a CLI agent's prompt reaches its stdin the other way (`agent/stdin`), which also closes +it -- and keeps those lines in the task's record (`node_dir/tasks//`, owner-only like the host's journal), so a connection that goes leaves the agent running and the next one takes it over from the frame the host last read (`agent/attach`); and answers probes of @@ -453,10 +460,10 @@ The agent table rendered as the text spliced into `spawn`'s and `run_subagent_da descriptions — `name [stateful, local-files, live-progress] (description)`. The model's only account of which agents exist, so an agent absent from it cannot be chosen; the `enum` on the `subagent` parameter constrains the same set. All three capabilities render positive *or* -negative, because "no tag" and "the roster does not say" are indistinguishable otherwise. An -acp row that names a **Machine** renders a fourth tag after them, `on machine `: the one -conditional tag, since a local row has nothing to say in that position, and the id only, never -the address. +negative, because "no tag" and "the roster does not say" are indistinguishable otherwise. A +row of any kind that names a **Machine** renders a fourth tag after them, `on machine `: +the one conditional tag, since a local row has nothing to say in that position, and the id +only, never the address. Also spliced into the skill gate's prompt under push discovery, where it is the account of what can be delegated and so decides which candidate skills are dropped as already covered; that copy omits the generic `builtin` row, which claims no capability bias and would read as @@ -517,15 +524,14 @@ one the dispatching turn's own tools get. Every dispatch supplies it — `spawn` read), a DAG node takes the same, and a Direct Chat resolves `session_workdir` because its branch returns before `workdir.bind` wraps the turn body. Distinct from **Agent home** (`SubagentManager.workspace`, `~/.raven/workspace`), which holds raven's memory and skills -and is only the fallback for a dispatch that supplied nothing. An `acp` entry that names a -**Machine** is the exception: its session works on that machine, in +and is only the fallback for a dispatch that supplied nothing. An `acp` or `cli` entry that +names a **Machine** is the exception: its session works on that machine, in `/` (`remoteCwd` defaults to `~/raven-work`), made and resolved there by the machine's **Raven node** for each dispatch before the session opens (`session/dir`); this computer's working directory never reaches the machine. The files that come with the turn are named to the agent by address (`remote.files_note`); nothing is copied to the machine, and nothing comes back unasked: what the session made -stays there, named in the reply by address, and `deliver_files` fetches one file by its -remote path on request. +stays there, named in the reply by address. _Avoid_: treating the fallback as the default — a sub-agent working in Agent home inspects raven's own memory instead of the user's checkout, and says nothing about having done so. diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index 682dec0c3..3b57ce217 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -22,20 +22,27 @@ - one sentence for an ssh failure, naming the machine and never its address; ssh's own messages, which do name it, go to an owner-only log rather than to the stderr the connection journals and quotes; -- the one-shot commands the installer runs there (:func:`run`, :func:`push`). +- the one-shot commands the installer runs there (:func:`run`, :func:`push`); +- the tunnel an openai sub-agent's endpoint is reached through from the + machine, held for one turn (:func:`tunnel`). """ from __future__ import annotations +import asyncio +import contextlib import hashlib import os import re import shlex +import socket import subprocess -from collections.abc import Sequence +import time +from collections.abc import AsyncIterator, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any +from urllib.parse import urlsplit, urlunsplit from raven.home import MACHINE_LANE_ENV, machine_lane_enabled from raven.ops import connections, remote_path @@ -53,6 +60,14 @@ #: The node's own heartbeat is what notices first; these are the floor under it. KEEPALIVE = ("-T", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=3") +#: Options for a tunnel held for one turn: no remote command, the forward +#: refused rather than a session without it, and the keepalives a held session +#: has (``KEEPALIVE`` without ``-T``: there is no session to give a tty to). +TUNNEL = ("-N", "-o", "ExitOnForwardFailure=yes", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=3") + +#: How long past ssh's own connect timeout a tunnel's port may take to open. +TUNNEL_READY_S = 20.0 + #: The session directory every check of an agent reuses (Test, Connect, the #: capability record), so pressing Test leaves one directory on the machine #: rather than one per press. @@ -338,6 +353,97 @@ def push(target: Machine, command: str, data: bytes, *, timeout: float = 300.0) return done.returncode, out +@dataclass(frozen=True) +class Forwarded: + """``url`` as it is reached through a tunnel: the same path, on this computer's loopback.""" + + url: str + host: str + """The name the far server knows itself by, the netloc as written: for the ``Host`` header, since the + request reaches the server from its own machine and not under that name.""" + + +@contextlib.asynccontextmanager +async def tunnel( + target: Machine, url: str, *, ssh_log: str | os.PathLike[str], connect_timeout: int = 15 +) -> AsyncIterator[Forwarded]: + """``url``'s host and port, as ``target`` resolves them, reached from a port here for as long as the block runs. + + How an openai sub-agent on a machine is called: ssh forwards a port on + this computer's loopback to the endpoint as the machine sees it, the + request goes through it, and the ssh ends with the block, so nothing of + the machine is held between turns. ``http`` only: the tunnel ends at this + computer's loopback, which an ``https`` certificate would not name, and + that refusal is one sentence given before any ssh starts. ssh's own words, + which name the address, go to ``ssh_log`` (:func:`ssh_log_path`), and a + tunnel that does not open is explained from them (:func:`explain`). + """ + parts = urlsplit(url) + if parts.scheme != "http": + raise RemoteMachineError( + f"machine {target.label}: an endpoint reached through the machine has to be http, and this one is " + f"{parts.scheme or 'of no scheme'}: the tunnel ends at this computer's loopback, which a certificate " + "would not name" + ) + if not parts.hostname: + raise RemoteMachineError(f"machine {target.label}: the endpoint's URL names no host to reach from it") + host, port, key, user = _target(target) + local = _free_port() + forward = f"127.0.0.1:{local}:{parts.hostname}:{parts.port or 80}" + extra = TUNNEL + ("-L", forward, "-E", os.fspath(ssh_log)) + argv = ssh_argv(host, port, key, user=user, connect_timeout=connect_timeout, extra=extra) + try: + proc = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + except OSError as exc: + raise RemoteMachineError(f"machine {target.label}: ssh could not be started: {exc}") from None + try: + await _tunnel_up(target, proc, local, ssh_log, within=connect_timeout + TUNNEL_READY_S) + yield Forwarded(url=urlunsplit(parts._replace(netloc=f"127.0.0.1:{local}")), host=parts.netloc) + finally: + if proc.returncode is None: + with contextlib.suppress(ProcessLookupError): + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=5.0) + except asyncio.TimeoutError: + with contextlib.suppress(ProcessLookupError): + proc.kill() + await proc.wait() + + +async def _tunnel_up(target: Machine, proc: Any, local: int, log: str | os.PathLike[str], *, within: float) -> None: + """Wait for ssh to open the port here, or to end, whichever comes first; an end is explained.""" + deadline = time.monotonic() + within + while True: + if proc.returncode is not None: + said = explain(target, proc.returncode, read_ssh_log(log)) + raise RemoteMachineError(said or f"the tunnel to machine {target.label} ended before it was open") + try: + _reader, writer = await asyncio.wait_for(asyncio.open_connection("127.0.0.1", local), timeout=0.5) + except (OSError, asyncio.TimeoutError): + pass + else: + writer.close() + with contextlib.suppress(OSError): + await writer.wait_closed() + return + if time.monotonic() >= deadline: + raise RemoteMachineError(f"machine {target.label} did not open the tunnel within {within:.0f}s") + await asyncio.sleep(0.1) + + +def _free_port() -> int: + """A port on this computer's loopback that nothing listens on now.""" + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + def explain(target: Machine, rc: int, output: str) -> str | None: """One sentence for a failure ssh itself caused, or None when ssh got through. @@ -407,7 +513,9 @@ def shell_path(path: str) -> str: def shown(target: Machine, path: str) -> str: """``path`` on ``target`` the way scp writes it, with the registry's user, address and port. - What the file tools read back as a remote path (:mod:`raven.ops.remote_path`). + The one spelling a path on a machine has in a reply, a record or a note: + what names a file there, and what :mod:`raven.ops.remote_path` reads back + as one. """ return remote_path.RemotePath(machine=target.id, path=path, _row=target._row).display() @@ -422,9 +530,11 @@ def _target(target: Machine) -> tuple[str, int, str, str]: __all__ = [ "CHECK_HANDLE", "DEFAULT_ROOT", + "Forwarded", "KEEPALIVE", "Machine", "RemoteMachineError", + "TUNNEL", "agent_missing", "explain", "failure", @@ -440,4 +550,5 @@ def _target(target: Machine) -> tuple[str, int, str, str]: "shown", "ssh_command", "ssh_log_path", + "tunnel", ] diff --git a/raven/agent/subagent/backends/__init__.py b/raven/agent/subagent/backends/__init__.py index 03b55e63b..50a29d69b 100644 --- a/raven/agent/subagent/backends/__init__.py +++ b/raven/agent/subagent/backends/__init__.py @@ -98,9 +98,13 @@ class AgentMeta(NamedTuple): """The registered machine this agent runs on, by id, or ``""`` for this computer. Rendered as a tag so the model can send work to a named machine and knows - the files it gets back are there, not here. The id only: the registry keeps - the address out of anything a model reads. Appended last, for the reason - ``model_choices`` gives.""" + the files it gets back are there, not here. The id only, which is all the + roster says of the machine: the way onto it -- the address, port, account + and key the registry holds -- stays out of anything a model reads, while a + path on it is written to a model the way scp writes it, address included, + because that is how a file there is named (``remote.shown``). Read off + every kind: whichever kind runs there, the tag is the same. Appended last, + for the reason ``model_choices`` gives.""" def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: @@ -140,6 +144,12 @@ def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: preset states, or the operator of a custom endpoint writes by hand; the declaration is the only available carrier of a fact raven cannot discover. + ``machine`` is read off every kind the same way: an entry of any kind that + names one runs there -- a cli one through the machine's Raven node, an + openai one through a tunnel to it -- and a built-in row's is cleared by its + schema, so the roster's tag, the registry's lane gate and the dispatch + never disagree about where a row runs. + ``snapshot`` is the already-loaded snapshot for this config, for callers handling a batch. Omitted, it is read from the store -- so a caller that knows nothing about ACP (the spawn manager, the DAG tool) needs no change. @@ -179,7 +189,7 @@ def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: stateful = bool(getattr(cfg, "stateful", True)) else: stateful = bool(getattr(cfg, "resume_command", None)) - machine = str(getattr(cfg, "machine", None) or "") if kind == "acp" else "" + machine = str(getattr(cfg, "machine", None) or "") return AgentMeta( getattr(cfg, "name", "") or "", getattr(cfg, "description", "") or "", @@ -370,7 +380,9 @@ def build_third_party_backend( ) -> SubagentBackend: """Build a third-party backend from a config object (duck-typed on ``kind``). - Accepts ThirdPartyCliSubagentConfig / ThirdPartyOpenAISubagentConfig. + Accepts ThirdPartyCliSubagentConfig / ThirdPartyOpenAISubagentConfig / + ThirdPartyAcpSubagentConfig; each kind takes the machine its entry names, + and runs there its own way. ``registry``, ``timeout`` and ``ready_timeout_ms`` override the config for one call and exist for the availability test in :mod:`raven.agent.subagent.probe`, @@ -405,6 +417,8 @@ def build_third_party_backend( registry=registry, mcps=cfg.mcps, allow_mcp_secrets=cfg.allow_mcp_secrets, + machine=getattr(cfg, "machine", None), + remote_cwd=getattr(cfg, "remote_cwd", None), ) if kind == "acp": from raven.acp_client.acp_agent import AcpAgentBackend @@ -441,6 +455,7 @@ def build_third_party_backend( max_tokens=cfg.max_tokens, timeout=cfg.timeout if timeout is None else timeout, max_output_chars=cfg.max_output_chars, + machine=getattr(cfg, "machine", None), ) raise ValueError(f"unknown third-party subagent kind: {kind!r}") diff --git a/raven/agent/subagent/backends/cli_agent.py b/raven/agent/subagent/backends/cli_agent.py index fc1964eb4..7de3f4263 100644 --- a/raven/agent/subagent/backends/cli_agent.py +++ b/raven/agent/subagent/backends/cli_agent.py @@ -11,6 +11,16 @@ With ``resume_command`` set the agent is stateful: an instance handle is bound to the CLI's own session id in :mod:`raven.agent.subagent.instances`, so a later spawn with the same handle resumes that session. + +An entry that names a ``machine`` runs the same command on that registered +machine, started there by the machine's Raven node (:mod:`raven.node.client`) +in the registered user's login environment, in a directory made there for the +run (``/``). The prompt reaches it the same +three ways -- the file is written there, stdin travels on the link and is +closed after it -- and its output lines come back on the link and are read as +a local run's are. Nothing of this computer goes with it: not its login shell, +not its working directory, not the files that came with the task, which are +named to it by address. """ from __future__ import annotations @@ -19,12 +29,14 @@ import json import os import re +import shlex import signal import tempfile import time import uuid from collections.abc import Awaitable, Callable, Sequence from contextlib import nullcontext +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -167,6 +179,18 @@ class CliAgentReportedError(RuntimeError): """ +@dataclass(frozen=True) +class _OnMachine: + """One dispatch's place on its machine, resolved through the machine's Raven node.""" + + target: Any + """The registered machine (``raven.acp_client.remote.Machine``), named by id in every sentence.""" + session_cwd: str + """The run's directory under the entry's root, made there and resolved to an absolute path.""" + link: Any + """The machine's node pool: the launch, the prompt file, the frames and the end.""" + + class CliAgentBackend: kind = "cli" streams = False @@ -219,6 +243,8 @@ def __init__( registry: InstanceRegistry | None = None, mcps: list[str] | None = None, allow_mcp_secrets: bool = False, + machine: str | None = None, + remote_cwd: str | None = None, ) -> None: self.name = name self.command = command @@ -235,6 +261,10 @@ def __init__( self.mcps = mcps self.allow_mcp_secrets = allow_mcp_secrets self.mcp_source: McpSource | None = None + # The registered machine this agent runs on, or None for this computer; + # ``remote_cwd`` is where its runs work there (``remote.root_of``). + self.machine = (machine or "").strip() or None + self.remote_cwd = remote_cwd self.streams = self._can_stream(command, resume_command, transcript_format) def set_mcp_source(self, source: McpSource | None) -> None: @@ -300,6 +330,30 @@ async def _kill_process_group(proc: asyncio.subprocess.Process, pgid: int) -> No if proc.returncode is None: await proc.wait() + def _line_publisher(self, on_delta: Callable[[str], Awaitable[None]] | None) -> Callable[[str], Awaitable[None]]: + """How one stdout line of a run is observed as it lands, wherever the run is. + + The reply text of the line, where the transcript format carries partial + text (``delta_reader``), goes to the activity console and to + ``on_delta``; for the plain ``text`` format the raw line goes to the + console, since that is what a human watching the terminal would have + seen. Machine formats with no delta reader (codex_jsonl and kin) put + nothing readable on stdout mid-run, so only their stderr shows. + """ + read_delta = delta_reader(self.transcript_format) + raw_console = read_delta is None and (self.transcript_format or "text") == "text" + + async def publish_out_line(line: str) -> None: + if read_delta is not None: + if text := read_delta(line): + activity.note_console(text) + if on_delta is not None: + await on_delta(text) + elif raw_console: + activity.note_console(line + "\n") + + return publish_out_line + async def _communicate_streaming( self, proc: asyncio.subprocess.Process, @@ -335,8 +389,7 @@ async def _communicate_streaming( ``communicate`` does it: a child that fills the stdout pipe while raven is still writing its prompt deadlocks both ends. """ - read_delta = delta_reader(self.transcript_format) - raw_console = read_delta is None and (self.transcript_format or "text") == "text" + publish_out_line = self._line_publisher(on_delta) out_chunks: list[bytes] = [] err_chunks: list[bytes] = [] @@ -354,15 +407,6 @@ async def feed() -> None: finally: proc.stdin.close() - async def publish_out_line(line: str) -> None: - if read_delta is not None: - if text := read_delta(line): - activity.note_console(text) - if on_delta is not None: - await on_delta(text) - elif raw_console: - activity.note_console(line + "\n") - async def pump_out() -> None: if proc.stdout is None: return @@ -514,6 +558,148 @@ async def _exec( except OSError: pass + async def _exec_there( + self, + placed: _OnMachine, + template: str, + task: str, + task_id: str, + agent_id: str | None, + attempts: list[dict[str, Any]] | None, + *, + on_delta: Callable[[str], Awaitable[None]] | None, + ) -> tuple[str, str]: + """One invocation on the entry's machine, through its node: the same argv, run and read there. + + What differs from :meth:`_exec` is where each piece lives. The prompt + file is written into the run's directory on the machine, under its + ``.raven/``, since a path here means nothing there; stdin travels on + the link and is closed after it -- for every command, not only one + with no placeholder, because the node opens a stdin for each agent it + starts, where the local launch hands a placeholder command none, and a + command that reads to the end would wait for it. The MCP file, when + the command takes one, is written there naming no server: the host's + are reached through a socket on this computer, and the reply says so + (``_withheld_note``). The environment is the machine user's login one, + captured by the node, with the role mark and the entry's ``env`` over + it: this computer's login shell, its ``RAVEN_HOME`` and the parent's + model binding describe a process of raven's own here, and say nothing + on that machine. A cancel or a timeout ends the whole process tree + there (``agent/cancel``). The daemon forwards whole lines, so a blank + line of a plain-text answer does not come back; every format this + module parses is one frame per line. + """ + from raven.ops import remote_path + + cwd = placed.session_cwd + scratch = f"{cwd}/.raven" + prompt_path = f"{scratch}/prompt-{task_id}.txt" + mcp_there: str | None = None + try: + if "{prompt_file}" in template: + await placed.link.write_file(placed.target.id, prompt_path, task, roots=[cwd]) + if "{mcp_file}" in template: + mcp_there = f"{scratch}/mcp-{task_id}.json" + empty = json.dumps({"tools": McpGrant().for_child_config()}, ensure_ascii=False) + await placed.link.write_file(placed.target.id, mcp_there, empty, roots=[cwd]) + except remote_path.NodeCallError as exc: + raise RuntimeError(f"CLI agent {self.name!r}: {exc}") from None + argv, used_placeholder = self._build_argv(template, task, prompt_path, agent_id, mcp_there) + env = {**subagent_role_env(), **self.env} + logger.info("Subagent [{}] CLI agent {!r} on machine {}: {}", task_id, self.name, placed.target.id, argv[:1]) + try: + endpoint = await placed.link.open_channel( + placed.target.id, name=self.name, command=shlex.join(argv), cwd=cwd, env=env + ) + except remote_path.NodeCallError as exc: + raise RuntimeError(f"CLI agent {self.name!r}: {exc}") from None + out_lines: list[str] = [] + err_lines: list[str] = [] + publish_out_line = self._line_publisher(on_delta) + + async def read_out() -> None: + while raw := await endpoint.readline(): + line = raw.decode("utf-8", "replace") + out_lines.append(line) + await publish_out_line(line) + + async def read_err() -> None: + while raw := await endpoint.readerr(): + line = raw.decode("utf-8", "replace") + err_lines.append(line) + activity.note_console(line + "\n") + + async def drive() -> int | None: + # The prompt after the launch, and the end of stdin either way. + await endpoint.send_input("" if used_placeholder else task, eof=True) + await asyncio.gather(read_out(), read_err()) + return await endpoint.wait() + + try: + rc = await asyncio.wait_for(drive(), timeout=self.timeout) if self.timeout is not None else await drive() + except asyncio.TimeoutError: + await endpoint.end() + raise CliAgentTimeoutError(f"CLI agent {self.name!r} timed out after {self.timeout}s") from None + except BaseException: + # A cancel, or a delta sink that raised: either way nothing may + # be left running on the machine. See ``_kill_process_group``. + await endpoint.end() + raise + if (why := endpoint.ended()) is not None: + raise RuntimeError(f"CLI agent {self.name!r}: {why}") + stdout = "".join(f"{line}\n" for line in out_lines) + stderr = "".join(f"{line}\n" for line in err_lines) + if attempts is not None: + attempts.append( + { + "argv0": argv[0] if argv else "", + "agentId": agent_id, + "exitCode": rc, + "stdout": stdout, + "stderr": stderr, + } + ) + if rc != 0: + said = _human_failure(stdout, stderr) + raise RuntimeError(f"CLI agent {self.name!r} exited {rc}" + (f": {said}" if said else "")) + return stdout, stderr + + async def _on_machine(self, handle: str) -> _OnMachine: + """This dispatch's place on its machine, or one sentence that says why not, before anything starts. + + The run's directory is made fresh for every dispatch, never cached, + for the reason the acp lane gives: one removed on the machine between + two dispatches would otherwise be launched into and refused there. + """ + from raven.acp_client import remote + from raven.ops import remote_path + + try: + target = await asyncio.to_thread(remote.machine, self.machine or "") + except remote.RemoteMachineError as exc: + raise RuntimeError(f"CLI agent {self.name!r}: {exc}") from None + link = remote_path.nodes() + if link is None: + raise RuntimeError( + f"CLI agent {self.name!r}: this process does not reach registered machines, so it cannot run on " + f"machine {target.label}" + ) + try: + session_cwd, _since = await link.session_dir(target.id, remote.session_root(self.remote_cwd, handle)) + except remote_path.NodeCallError as exc: + raise RuntimeError(f"CLI agent {self.name!r}: {exc}") from None + return _OnMachine(target=target, session_cwd=session_cwd, link=link) + + def _withheld_note(self, grant: McpGrant) -> str: + """What the reply says of the servers a machine row was granted and not handed: there is no socket to hand.""" + names = sorted(server.name for server in grant.granted) + if not self.machine or not names: + return "" + return ( + f"MCP servers {names} were not lent to cli agent {self.name!r}: it runs on machine {self.machine!r}, " + "and the host's servers are reached through a socket on this computer" + ) + async def run( self, task: str, @@ -532,7 +718,10 @@ async def run( on_delta: Callable[[str], Awaitable[None]] | None = None, media: Sequence[Media] = (), ) -> str: - task = with_attachment_note(task, media) + # A path here means nothing on another machine: the files that came + # with the task are named to an agent there by address instead, once + # its machine is known (``_dispatch_there``). + task = task if self.machine else with_attachment_note(task, media) runtime_env: dict[str, str] = {} if model: runtime_env["RAVEN_PARENT_MODEL"] = model @@ -559,7 +748,12 @@ async def run( output = "" grant = mcp_grant if mcp_grant is not None else self.resolve_mcp_grant(mcps) mcp_path: str | None = None - if "{mcp_file}" in self.command or (self.resume_command and "{mcp_file}" in self.resume_command): + wants_mcp_file = "{mcp_file}" in self.command or bool( + self.resume_command and "{mcp_file}" in self.resume_command + ) + # On a machine the file is written there by the launch, naming no + # server (``_exec_there``); nothing here would reach the agent. + if wants_mcp_file and not self.machine: fd, mcp_path = tempfile.mkstemp(prefix=f"raven_subagent_{task_id}_", suffix=".mcp.json") with os.fdopen(fd, "w", encoding="utf-8") as stream: json.dump({"tools": grant.for_child_config()}, stream, ensure_ascii=False) @@ -581,8 +775,9 @@ async def run( notice_sink=on_delta, runtime_env=runtime_env, mcp_file=mcp_path, + media=media, ) - if note := grant.note_text(): + for note in filter(None, (self._withheld_note(grant), grant.note_text())): notice = f"\n\n[raven] {note}." if on_delta is not None and self.streams: await on_delta(notice) @@ -647,8 +842,21 @@ async def _dispatch( notice_sink: Callable[[str], Awaitable[None]] | None = None, runtime_env: dict[str, str] | None = None, mcp_file: str | None = None, + media: Sequence[Media] = (), ) -> str: skey = session_key or "default" + if self.machine: + return await self._dispatch_there( + task, + task_id, + skey=skey, + handle=handle, + resumable=resumable, + attempts=attempts, + on_delta=on_delta, + notice_sink=notice_sink, + media=media, + ) cwd = self.cwd or str(workspace) # Here rather than beside either launch: this lane sees nothing of what # the child did between them -- no tool results, no protocol -- so its @@ -673,6 +881,56 @@ async def _dispatch( finally: activity.record_snapshot_changes(before_files, take_snapshot(cwd), cwd) + async def _dispatch_there( + self, + task: str, + task_id: str, + *, + skey: str, + handle: str, + resumable: bool, + attempts: list[dict[str, Any]], + on_delta: Callable[[str], Awaitable[None]] | None, + notice_sink: Callable[[str], Awaitable[None]] | None, + media: Sequence[Media], + ) -> str: + """The run on the entry's machine: placed there first, with the task's files named by address. + + No account of this computer's files: the run's are on the machine, + and what it wrote there is named in the reply by address rather than + brought back. The parent's model binding and the MCP file stay here + too, for the reason ``_exec_there`` gives. + """ + from raven.acp_client import remote + + placed = await self._on_machine(handle) + # Nothing is carried to the machine: the files that come with the + # task are named to the agent by address, and it looks for them there. + if note := remote.files_note(media, placed.target): + task = f"{task}\n\n{note}" + output = await self._launch( + task, + task_id, + placed.session_cwd, + skey=skey, + handle=handle, + resumable=resumable, + attempts=attempts, + on_delta=on_delta, + notice_sink=notice_sink, + placed=placed, + ) + # Where to find what it made: the caller's workspace line names this + # computer, and what it wrote is there, by address -- nothing is + # brought back unasked. + notice = ( + f"\n\n[raven] Ran on machine {placed.target.label}, in " + f"{remote.shown(placed.target, placed.session_cwd)}; the files it wrote are there, not on this computer." + ) + if notice_sink is not None and self.streams: + await notice_sink(notice) + return output + notice + async def _launch( self, task: str, @@ -687,6 +945,7 @@ async def _launch( notice_sink: Callable[[str], Awaitable[None]] | None = None, runtime_env: dict[str, str] | None = None, mcp_file: str | None = None, + placed: _OnMachine | None = None, ) -> str: """Run the child, resuming this handle's session when it has one.""" if self.is_stateful: @@ -717,6 +976,7 @@ async def _launch( notice_sink=notice_sink, runtime_env=runtime_env, mcp_file=mcp_file, + placed=placed, ) return await self._attempt( @@ -733,6 +993,7 @@ async def _launch( notice_sink=notice_sink, runtime_env=runtime_env, mcp_file=mcp_file, + placed=placed, ) async def _run_stateful( @@ -749,6 +1010,7 @@ async def _run_stateful( notice_sink: Callable[[str], Awaitable[None]] | None = None, runtime_env: dict[str, str] | None = None, mcp_file: str | None = None, + placed: _OnMachine | None = None, ) -> str: """Resume this handle's session, or mint one. @@ -787,6 +1049,7 @@ async def _run_stateful( notice_sink=notice_sink, runtime_env=runtime_env, mcp_file=mcp_file, + placed=placed, ) except (CliAgentTimeoutError, CliAgentReportedError): # Neither is evidence the CLI's session store pruned this id: a @@ -824,6 +1087,7 @@ async def _run_stateful( notice_sink=notice_sink, runtime_env=runtime_env, mcp_file=mcp_file, + placed=placed, ) async def _attempt( @@ -842,19 +1106,25 @@ async def _attempt( notice_sink: Callable[[str], Awaitable[None]] | None = None, runtime_env: dict[str, str] | None = None, mcp_file: str | None = None, + placed: _OnMachine | None = None, ) -> str: """Run one CLI invocation (create or resume) and return its output, raising on failure.""" - stdout, stderr = await self._exec( - template, - task, - task_id, - cwd, - agent_id, - attempts, - on_delta=on_delta, - runtime_env=runtime_env, - mcp_file=mcp_file, - ) + if placed is not None: + stdout, stderr = await self._exec_there( + placed, template, task, task_id, agent_id, attempts, on_delta=on_delta + ) + else: + stdout, stderr = await self._exec( + template, + task, + task_id, + cwd, + agent_id, + attempts, + on_delta=on_delta, + runtime_env=runtime_env, + mcp_file=mcp_file, + ) jsonl_id: str | None = None jsonl_reply: str | None = None diff --git a/raven/agent/subagent/backends/openai_api.py b/raven/agent/subagent/backends/openai_api.py index f20462bb4..37a9aef2c 100644 --- a/raven/agent/subagent/backends/openai_api.py +++ b/raven/agent/subagent/backends/openai_api.py @@ -8,12 +8,20 @@ fields survive. ``reasoning_steps`` is one of those: it is how a deep-research endpoint reports the work behind an answer, and it reaches the run's own record through :mod:`raven.agent.subagent.openai_steps`. + +An entry that names a ``machine`` is posted to through that registered +machine: each turn opens an ssh tunnel to it (:func:`raven.acp_client.remote.tunnel`) +and the request goes to ``base_url`` as the machine resolves it, for an +endpoint that answers only on that machine's network, or only on the machine +itself. The tunnel ends with the turn. """ from __future__ import annotations +import asyncio +import contextlib import json as jsonlib -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any @@ -47,6 +55,7 @@ def __init__( max_tokens: int | None = None, timeout: int | None = None, max_output_chars: int = 128000, + machine: str | None = None, ) -> None: self.name = name self.base_url = base_url @@ -57,6 +66,45 @@ def __init__( self.max_tokens = max_tokens self.timeout = timeout self.max_output_chars = max_output_chars + # The registered machine the endpoint is reached from, or None for this computer. + self.machine = (machine or "").strip() or None + + @contextlib.asynccontextmanager + async def _reached(self, url: str, headers: dict[str, str]) -> AsyncIterator[tuple[str, dict[str, str]]]: + """``url`` and ``headers`` as this turn posts them. + + As they are from this computer; through a tunnel from the entry's + machine otherwise, opened for this turn and closed after it. The + request then arrives at a port on this computer's loopback: ``Host`` + carries the name the server knows itself by, and the proxy variables + of this computer's environment are not read for it (``_post_chat``), + since a proxy would take a loopback address somewhere else. + """ + if not self.machine: + yield url, headers + return + from raven.acp_client import remote + + try: + target = await asyncio.to_thread(remote.machine, self.machine) + log = await asyncio.to_thread(remote.ssh_log_path, f"{self.name}-tunnel") + except remote.RemoteMachineError as exc: + raise RuntimeError(f"OpenAI-API agent {self.name!r}: {exc}") from None + try: + # Emptied per turn, so a failure is read from this turn's lines. + log.write_text("", encoding="utf-8") + except OSError: + pass + try: + async with remote.tunnel(target, url, ssh_log=log) as reached: + yield reached.url, {**headers, "Host": reached.host} + except remote.RemoteMachineError as exc: + raise RuntimeError(f"OpenAI-API agent {self.name!r}: {exc}") from None + except aiohttp.ClientConnectionError as exc: + raise RuntimeError( + f"OpenAI-API agent {self.name!r}: its endpoint did not answer through the tunnel from machine " + f"{target.label}: {exc}" + ) from None async def _post_chat( self, @@ -74,8 +122,10 @@ async def _post_chat( # otherwise ignores -- unlike httpx (the rest of the codebase), which honors # them by default. On a host that can only reach the provider through a # proxy, a direct connection is not a connection error but whatever the - # provider says to an unexpected origin (mirothinker: HTTP 451). - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + # provider says to an unexpected origin (mirothinker: HTTP 451). Not + # for a row on a machine: its request goes to a port on this computer's + # loopback (``_reached``), which a proxy would take somewhere else. + async with aiohttp.ClientSession(timeout=timeout, trust_env=not self.machine) as session: async with session.post(url, json=json, headers=headers) as resp: await self._raise_for_status(resp) return await resp.json() @@ -120,7 +170,7 @@ async def _stream_chat( content: list[str] = [] reasoning: list[str] = [] usage: Any = None - async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with aiohttp.ClientSession(timeout=timeout, trust_env=not self.machine) as session: async with session.post(url, json=payload, headers=headers) as resp: await self._raise_for_status(resp) async for raw in resp.content: @@ -234,34 +284,35 @@ async def run( client_timeout = aiohttp.ClientTimeout(total=self.timeout, connect=30) reader = OpenAIStepReader() - if on_delta is not None: - content: Any = await self._stream_chat( - url, - payload=body, - headers=headers, - timeout=client_timeout, - on_delta=bounded_delta(on_delta, self.max_output_chars), - reader=reader, - task_id=task_id, - session_key=session_key, - ) - else: - data = await self._post_chat(url, json=body, headers=headers, timeout=client_timeout) - try: - message = data["choices"][0]["message"] - except (KeyError, IndexError, TypeError) as exc: - raise RuntimeError(f"OpenAI-API agent {self.name!r}: unexpected response shape") from exc - content = message.get("content") - if not content: - # Some reasoning models put the answer only in a reasoning field. - content = message.get("reasoning_content") or message.get("reasoning") or "" - reader.feed_steps(message.get("reasoning_steps")) - await activity.note_provider_usage( - data.get("usage"), - model=self.model, - session_key=session_key, - task_id=task_id, - ) + async with self._reached(url, headers) as (url, headers): + if on_delta is not None: + content: Any = await self._stream_chat( + url, + payload=body, + headers=headers, + timeout=client_timeout, + on_delta=bounded_delta(on_delta, self.max_output_chars), + reader=reader, + task_id=task_id, + session_key=session_key, + ) + else: + data = await self._post_chat(url, json=body, headers=headers, timeout=client_timeout) + try: + message = data["choices"][0]["message"] + except (KeyError, IndexError, TypeError) as exc: + raise RuntimeError(f"OpenAI-API agent {self.name!r}: unexpected response shape") from exc + content = message.get("content") + if not content: + # Some reasoning models put the answer only in a reasoning field. + content = message.get("reasoning_content") or message.get("reasoning") or "" + reader.feed_steps(message.get("reasoning_steps")) + await activity.note_provider_usage( + data.get("usage"), + model=self.model, + session_key=session_key, + task_id=task_id, + ) # The answer is deliberately not among these rows: the record keeps it and # its reader appends it as the closing message, the same contract the acp # lane follows. diff --git a/raven/agent/subagent/manager.py b/raven/agent/subagent/manager.py index 67f274682..cf602593d 100644 --- a/raven/agent/subagent/manager.py +++ b/raven/agent/subagent/manager.py @@ -1170,22 +1170,26 @@ async def chat( ``media`` is the turn's attachments, handed to the backend by path (see ``raven.agent.subagent.attachments``). An agent the roster tags [no-local-files] cannot open a path, so it is told the attachments stayed - behind rather than handed a spelling that means nothing where it runs. + behind rather than handed a spelling that means nothing where it runs -- + except a row on a registered machine, which carries the same tag: its + backend names each attachment to the agent by its address on this + computer and copies nothing (``remote.files_note``), so the attachments + are kept for it to name, and the page's bullets are spelled the same way. """ self.remember_session(session_key) self._require_addressable(agent, doing=f"chat with instance {handle!r}") row = self.registry.get(agent) - # A row on a registered machine cannot read a path here either, but its - # attachments are sent there by the backend, which tells the agent - # where they landed in its own terms; so they are kept, and the - # bullets are left as the page wrote them rather than pointed at this - # computer's disk. + # A row on a registered machine cannot read a path here either; what + # its backend sends is the address of each file on this computer, with + # the word that nothing was copied, so the agent looks for the file + # where its machine can read it or says it could not. The bullets are + # pointed at this computer's disk for the same reason as a local + # row's: relative to the session directory there they name nothing. on_machine = bool(row is not None and row.caps.machine) if media and row is not None and not row.caps.reads_local_files and not on_machine: text = with_undeliverable_note(text, media) media = () - if not on_machine: - text = retarget_note(text, media, self.workspace) + text = retarget_note(text, media, self.workspace) recorded = with_attachment_note(text, media) session_dir = self.session_dir_for(session_key) @@ -2111,10 +2115,13 @@ async def _announce_result( machine, where = self._machine_workdir(origin.get("agent") or "", origin.get("handle") or task_id) if machine: # Not this computer's directory: the agent worked on its machine, - # in a directory made there when the run started. Named here on + # in a directory made there as the run started. Named here on # every path, since a failed or cancelled run's result does not - # close with it the way a finished one's does. - workdir_line = f"Working directory: on machine {machine!r}, {where} (made there when the run started)\n" + # close with it the way a finished one's does -- but only a run + # that returned is known to have got as far as making it: one + # refused by the registry, the ssh or the node never did. + made = "made there when the run started" if status == "ok" else "made there only if the run got that far" + workdir_line = f"Working directory: on machine {machine!r}, {where} ({made})\n" record_line = f"\n\nRecord: {record_path}" if record_path else "" # How the run's calls went, which nothing on this path carried. Measured # on a real dispatch: two of three calls timed out, the run produced no diff --git a/raven/agent/subagent/probe.py b/raven/agent/subagent/probe.py index ba33849a8..79db14a4b 100644 --- a/raven/agent/subagent/probe.py +++ b/raven/agent/subagent/probe.py @@ -558,6 +558,9 @@ def done(status: ProbeStatus, detail: str, target: str = "") -> ProbeResult: exe = argv[0] if "{" in exe: return done("unknown", f"the command's first token is a placeholder ({exe})", exe) + if machine_id := (getattr(cfg, "machine", None) or "").strip(): + # A `which` here would answer for this computer, and the command runs there. + return _probe_remote(cfg, machine_id, done) # CliAgentBackend._exec builds the child's env as {**login_shell_env(), **self.env} # (cli_agent.py:139-140), so a configured PATH overrides the login shell's for a # real spawn; resolving against the login PATH alone here would report an agent @@ -602,7 +605,7 @@ def done(status: ProbeStatus, detail: str, target: str = "") -> ProbeResult: if not argv: return done("unknown", "command is empty") if machine_id := (getattr(cfg, "machine", None) or "").strip(): - return _probe_remote_acp(cfg, machine_id, done) + return _probe_remote(cfg, machine_id, done) exe = argv[0] cfg_path = (getattr(cfg, "env", None) or {}).get("PATH") resolved = shutil.which(exe, path=cfg_path or path) @@ -679,6 +682,10 @@ def done(status: ProbeStatus, detail: str) -> ProbeResult: # endpoint (a local vLLM) is legitimate, and short-circuiting would # report a working agent as broken. return done("attention", "api key not set") + if machine_id := (getattr(cfg, "machine", None) or "").strip(): + # The endpoint answers on the machine's network, through a tunnel a + # test opens; a request from here would ask the wrong network. + return _probe_remote(cfg, machine_id, lambda status, detail, target="": done(status, detail)) url = base_url.rstrip("/") + "/models" headers = {"Authorization": f"Bearer {cfg.api_key}"} if has_key else {} try: @@ -793,6 +800,7 @@ def elapsed() -> int: return int((time.monotonic() - started) * 1000) kind = getattr(cfg, "kind", None) + on_machine = bool((getattr(cfg, "machine", None) or "").strip()) if kind == "builtin": # Nothing to verify. A built-in agent is this process: there is no command # to launch, no endpoint to authenticate against, and no transcript parser @@ -825,9 +833,16 @@ def elapsed() -> int: if probe.status == "missing": return TestResult(cfg.name, source, "openai", False, probe.detail, None, elapsed()) answered = await ping_agent(cfg) - detail = probe.detail if answered.ok else f"{answered.detail}; {probe.detail}" + # On a machine the free probe said only where the endpoint is and to + # run this; the ping's own words are the whole verdict there. + if on_machine: + detail = answered.detail + else: + detail = probe.detail if answered.ok else f"{answered.detail}; {probe.detail}" return TestResult(cfg.name, source, "openai", answered.ok, detail, None, elapsed(), answered.remedy) - if probe.status != "ready": + # The free probe cannot say "ready" for a command on another machine, and + # this run through the machine's node is what can. + if probe.status != "ready" and not on_machine: return TestResult(cfg.name, source, "cli", False, probe.detail, None, elapsed()) try: @@ -936,13 +951,18 @@ class PingResult: remedy: Remedy | None = None -def _probe_remote_acp(cfg: Any, machine_id: str, done: Any) -> ProbeResult: - """The free check for an acp agent on a registered machine: still registered, and verified? +def _probe_remote(cfg: Any, machine_id: str, done: Any) -> ProbeResult: + """The free check for an agent on a registered machine: still registered, and verified? No ssh: this runs on every listing, and one machine slow to answer would hold the whole roster behind it. ``which`` here would answer for this computer, - which is not where the agent runs. What can be said for free is whether the - registry still names a usable machine, and what the last test recorded. + which is not where the agent runs, and a request from here would ask the + wrong network for an endpoint behind the machine. What can be said for free + is whether the registry still names a usable machine, the link's state, and + what the last test recorded: for an acp row, its handshake, kept as a + capability snapshot; a cli or openai row records nothing for free, so its + verdict is the test's alone, kept beside the row (``last_test_*``), and the + probe says to run one. """ from raven.acp_client import remote from raven.ops import remote_path @@ -956,6 +976,10 @@ def _probe_remote_acp(cfg: Any, machine_id: str, done: Any) -> ProbeResult: # how many agents it carries, idle, or the sentence of its last failure. if (link := remote_path.nodes()) is not None and (said := link.status(machine_id).said()): where += f" ({said})" + if getattr(cfg, "kind", None) != "acp": + return done( + "attention", f"{where}; nothing here can check it without reaching the machine -- run a test", machine_id + ) snapshot = acp_snapshot_for(cfg) if snapshot is None: return done("attention", f"{where}; its ACP capabilities have not been recorded yet -- run a test", machine_id) diff --git a/raven/agent/tools/raven_config.py b/raven/agent/tools/raven_config.py index 2759d41ad..354fa263c 100644 --- a/raven/agent/tools/raven_config.py +++ b/raven/agent/tools/raven_config.py @@ -251,16 +251,19 @@ def _what_it_needs( ) -> list[str]: """The next step for one agent that did not answer, read off its refusal's remedy. - On a registered machine there is no next step here: the agent, its install - and its sign-in are all on that machine, which exec does not reach. + On a registered machine the fix is made there: the agent, its install and + its sign-in are all on that machine. exec can look there -- it takes the + machine's id -- but a sign-in or an install is the user's, in a terminal of + their own on it. """ from raven.agent.subagent.presets import DIAGNOSE_HINTS if machine: lines = [ - f"- It runs on machine {machine!r}, so every fix is made there by the user, in a terminal on that " - "machine: install, sign-in or setup, as the refusal says. Nothing here reaches it (exec refuses " - "ssh to a registered machine). Tell the user what it said, then retry once they have." + f"- It runs on machine {machine!r}, so the fix is made there by the user, in a terminal on that " + f"machine: install, sign-in or setup, as the refusal says. exec(machine={machine!r}) can look there " + "first (command -v , its --version), capped at 60s; it is not where a sign-in or an install " + "happens. Tell the user what the refusal and the look said, then retry once they have." ] return lines + _menu_line(row, model) remedy = remedy or {} @@ -307,7 +310,7 @@ def _menu_line(row: dict[str, Any], model: str) -> list[str]: def _machines_line() -> str | None: - """The registered machines an acp preset can be added on, while the machine lane is on; else nothing.""" + """The registered machines a preset can be added on, while the machine lane is on; else nothing.""" from raven.home import machine_lane_enabled from raven.ops import connections @@ -325,9 +328,10 @@ def _machines_line() -> str | None: if not names: return None return ( - f" machines an acp preset can be added on: {', '.join(names)} -- add subagents " - '{"preset": "", "machine": "", "remote_cwd"?: "~/raven-work"}: it runs there, signed in ' - "there; the name defaults to @" + f" machines a preset can be added on: {', '.join(names)} -- add subagents " + '{"preset": "", "machine": "", "remote_cwd"?: "~/raven-work"}: an acp or cli preset runs ' + "there, signed in there; an openai preset's endpoint is reached from there through an ssh tunnel (no " + "remote_cwd); the name defaults to @" ) diff --git a/raven/config/schema.py b/raven/config/schema.py index 9829d9299..43cc2c8c8 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -1480,6 +1480,25 @@ def _malformed_reads_as_empty(cls, data: Any) -> Any: return data +def _settle_machine(entry: Any) -> str | None: + """``entry.machine`` as an id or ``None``, with the lane's state said once on load. + + Shared by every kind that can name a machine, so a row of any kind that + names one while the lane is off is said to stay off the table in the same + words; the registry's gate reads the same field for every kind. + """ + machine = (getattr(entry, "machine", None) or "").strip() or None + if machine and not machine_lane_enabled(): + logger.info( + "sub-agent {!r} names machine {!r}, and the machine lane is debug-only: the entry stays " + "off the table until raven is started with {}=1", + entry.name, + machine, + MACHINE_LANE_ENV, + ) + return machine + + class ThirdPartyCliSubagentConfig(Base): """A third-party CLI agent (claude code, codex, …) callable as a native subagent. @@ -1589,6 +1608,31 @@ class ThirdPartyCliSubagentConfig(Base): transcript_format: Literal["text", "codex_jsonl", "claude_stream_json", "openclaw_json", "opencode_json"] = "text" cwd: str | None = None env: dict[str, str] = Field(default_factory=dict) + machine: str | None = None + """The registered machine this agent runs on, by its ``connections.json`` id, or + ``None`` for this computer. + + Set, ``command`` is started on that machine by its Raven node, in the + registered user's login environment there with this entry's ``env`` over + it -- never this computer's login shell -- and each run works in + ``/`` there (``cwd`` names a directory on + this computer and is not read). The prompt reaches it the three ways it + does here: a ``{prompt}`` token as one argument, a ``{prompt_file}`` as a + file written into that directory, or on stdin, which the node closes after + it. The host's MCP servers are not lent to it, the files that come with a + task are named to it by address and not copied, and what it writes stays + on that machine. Raven reaches the machine over one ssh link with the + address, port and key the registry holds, none of which is written here. + See :mod:`raven.node.client` and + :class:`raven.agent.subagent.backends.cli_agent.CliAgentBackend`. + + Debug-only until the lane matures, as for an acp entry: unless raven was + started with ``RAVEN_DEBUG_MACHINES=1`` (:func:`raven.home.machine_lane_enabled`), + an entry that names a machine is left off the table and no machine is + reached for it.""" + remote_cwd: str | None = None + """Where on ``machine`` this agent works, ``~/raven-work`` when unset; ignored + without ``machine``. ``~`` is the registered user's home there.""" timeout: int | None = None max_output_chars: int = 30000 memory: SubagentMemoryConfig | None = Field(default=None, validation_alias=AliasChoices("memory", "everos")) @@ -1637,6 +1681,11 @@ def _check_mcp_file_placeholders(self) -> "ThirdPartyCliSubagentConfig": raise ValueError("mcps is set but command does not contain {mcp_file}") return self + @model_validator(mode="after") + def _settle_machine_row(self) -> "ThirdPartyCliSubagentConfig": + self.machine = _settle_machine(self) + return self + @model_validator(mode="after") def _resolve_preset(self) -> "ThirdPartyCliSubagentConfig": self.preset = _resolve_preset_provenance(self.name, self.preset) @@ -1737,12 +1786,37 @@ def _null_stateful_is_the_default(cls, value: Any) -> Any: memory: SubagentMemoryConfig | None = Field(default=None, validation_alias=AliasChoices("memory", "everos")) """How the host addresses this agent's memories, or ``None`` for an agent the host records none for.""" + machine: str | None = None + """The registered machine this agent's endpoint is reached from, by its + ``connections.json`` id, or ``None`` to reach ``baseUrl`` from this computer. + + Set, each turn opens an ssh tunnel to that machine and closes it after: a + port on this computer's loopback forwarded to ``baseUrl``'s host and port + as the machine resolves them, for an endpoint that answers only on that + machine's network, or only on the machine itself (``http://127.0.0.1:...``). + ``baseUrl`` has to be ``http``: the tunnel ends at this computer's + loopback, which an ``https`` certificate would not name. The address, port + and key the registry holds are read for the tunnel alone and written + nowhere here. Debug-only as for the other kinds: unless raven was started + with ``RAVEN_DEBUG_MACHINES=1``, the entry is left off the table and no + machine is reached for it. There is no ``remoteCwd``: an endpoint has no + working directory on the machine, and one written here is dropped with a + warning.""" @model_validator(mode="before") @classmethod def _drop_unsupported_mcp_fields(cls, data: Any) -> Any: if not isinstance(data, dict): return data + if "remoteCwd" in data or "remote_cwd" in data: + # Named here for the reason the mcp fields are: a field that loads + # and does nothing is the failure that takes longest to find. + logger.warning( + "remoteCwd is not supported for kind 'openai' (sub-agent {!r}); ignoring -- an endpoint reached " + "from a machine has no working directory there", + data.get("name") or "", + ) + data = {k: v for k, v in data.items() if k not in ("remoteCwd", "remote_cwd")} declared_mcps = data.get("mcps") allow_secrets = data.get("allowMcpSecrets", data.get("allow_mcp_secrets")) if declared_mcps or allow_secrets: @@ -1783,6 +1857,18 @@ def _drop_declared_local_file_access(self) -> "ThirdPartyOpenAISubagentConfig": self.reads_local_files = False return self + @model_validator(mode="after") + def _settle_machine_row(self) -> "ThirdPartyOpenAISubagentConfig": + self.machine = _settle_machine(self) + if self.machine and (self.base_url or "").strip().lower().startswith("https:"): + logger.warning( + "openai sub-agent {!r} reaches its endpoint from machine {!r}, and the tunnel there carries http " + "only: every dispatch will be refused until baseUrl is http or `machine` is removed", + self.name, + self.machine, + ) + return self + @model_validator(mode="after") def _resolve_preset(self) -> "ThirdPartyOpenAISubagentConfig": self.preset = _resolve_preset_provenance(self.name, self.preset) @@ -2126,15 +2212,7 @@ def _remote_rows_borrow_no_keys(self) -> "ThirdPartyAcpSubagentConfig": ``_warn_on_declared_cli_fields`` gives; the write path refuses it (``update_subagents.reject_unsupported_acp_fields``). """ - self.machine = (self.machine or "").strip() or None - if self.machine and not machine_lane_enabled(): - logger.info( - "sub-agent {!r} names machine {!r}, and the machine lane is debug-only: the entry stays " - "off the table until raven is started with {}=1", - self.name, - self.machine, - MACHINE_LANE_ENV, - ) + self.machine = _settle_machine(self) if self.machine and self.lend_keys: logger.warning( "sub-agent {!r} runs on machine {!r}, so Raven's keys {} are not lent to it; " @@ -2226,6 +2304,31 @@ class BuiltinAgentConfig(Base): row that says nothing about confinement cannot loosen it.""" timeout: int | None = None max_output_chars: int = 30000 + machine: str | None = None + """Declared so that a row naming one is told, not so that it runs there. + + A built-in agent is Raven's own loop in this process, where its model keys + and tools are, so it has no process to start on another machine; a value + here is dropped on load with a warning and the row runs on this computer, + reaching a machine the way the main loop does (``exec`` with a ``machine``, + paths written by address). Undeclared, the key would be dropped without a + word, which is how a row ends up running here while its author believes + it runs there.""" + + @model_validator(mode="after") + def _runs_here(self) -> "BuiltinAgentConfig": + machine = (self.machine or "").strip() + if machine: + logger.warning( + "built-in sub-agent {!r} names machine {!r}, which it cannot run on: Raven's own loop runs in " + "this process, where its model keys and tools are. The entry runs on this computer and reaches " + "the machine as the main loop does (exec with a machine, paths by address); remove `machine` " + "from it", + self.name, + machine, + ) + self.machine = None + return self AgentConfig = Annotated[ diff --git a/raven/config/update_subagents.py b/raven/config/update_subagents.py index a74ea8585..4bfe40b83 100644 --- a/raven/config/update_subagents.py +++ b/raven/config/update_subagents.py @@ -86,6 +86,15 @@ def reject_unsupported_openai_fields(entries: list[dict]) -> None: f"mcps/allowMcpSecrets/sessionMcp are not supported for kind 'openai' (sub-agent " f"{name!r}): the backend has no tool loop; remove the field(s)" ) + if "remoteCwd" in entry or "remote_cwd" in entry: + # The schema drops it with a warning on load; on a write the caller + # can be told. An endpoint reached from a machine works in no + # directory there, so the field would be written and do nothing. + name = entry.get("name") or "" + raise ValueError( + f"remoteCwd is not supported for kind 'openai' (sub-agent {name!r}): an endpoint reached " + "from a machine has no working directory there; remove the field" + ) def reject_unsupported_acp_fields(entries: list[dict]) -> None: diff --git a/raven/node/client.py b/raven/node/client.py index 1f6cbda41..2dd87b422 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -149,7 +149,9 @@ class ChannelEndpoint: The client reads and writes it as it would a child process's pipes; what it gets are the ``agent/frame`` and ``agent/stderr`` notifications of one - channel, and what it writes goes back the same way. ``end`` is + channel, and what it writes goes back the same way -- a frame per line + (``write``), or a CLI agent's whole prompt and the end of its stdin + (``send_input``, ``agent/stdin``). ``end`` is ``agent/cancel``: the node ends the agent's whole process tree, and the ``agent/exit`` that follows is what settles ``wait``. A link that goes before the agent does leaves the endpoint waiting (``detach``): the agent @@ -176,7 +178,8 @@ def __init__( self._rc: int | None = None self._lost: str | None = None self._detached: str | None = None - self._pending: list[str] = [] + self._pending: list[tuple[str, dict[str, Any]]] = [] + """What was written while the link was down, by method, to go out on the next link in order.""" self._cancel_wanted = False self._done = asyncio.Event() @@ -199,22 +202,38 @@ async def write(self, data: bytes) -> None: if self.returncode is not None: raise BrokenPipeError(f"the agent on {self.describe()} has ended") line = data.decode("utf-8", "replace").rstrip("\r\n") + await self._send(protocol.AGENT_FRAME, {"line": line}) + + async def send_input(self, text: str, *, eof: bool = False) -> None: + """``text`` to the agent's stdin as it is, and the end of its stdin when ``eof``. + + For an agent that reads a prompt rather than frames: a CLI agent takes + its task this way when its command names no placeholder for it, and + every CLI agent is told where its stdin ends, since the node opens one + for each agent it starts and a command that reads to the end would + otherwise wait for it. + """ + if self.returncode is not None: + raise BrokenPipeError(f"the agent on {self.describe()} has ended") + await self._send(protocol.AGENT_STDIN, {"data": text, "eof": eof}) + + async def _send(self, method: str, params: dict[str, Any]) -> None: if self._detached is not None or not self._link.alive: - # Down, or gone and not yet said so by the heartbeat: the frame waits for the next link. - self._hold(line) + # Down, or gone and not yet said so by the heartbeat: it waits for the next link. + self._hold(method, params) return try: - await self._link.notify(protocol.AGENT_FRAME, {"channel": self.channel, "line": line}) + await self._link.notify(method, {"channel": self.channel, **params}) except AcpConnectionError: - # The pool's heartbeat will say the link is lost; the frame waits for the next one. - self._hold(line) + # The pool's heartbeat will say the link is lost; it waits for the next one. + self._hold(method, params) - def _hold(self, line: str) -> None: + def _hold(self, method: str, params: dict[str, Any]) -> None: if len(self._pending) >= _PENDING_FRAMES: raise BrokenPipeError( f"the link to machine {self.target.label} has been down too long: {len(self._pending)} frames wait" ) - self._pending.append(line) + self._pending.append((method, params)) async def readline(self) -> bytes: item = await self._frames.get() @@ -277,8 +296,8 @@ async def reattach(self, link: AcpClient, channel: int) -> None: self._link, self.channel = link, channel self._detached = None held, self._pending = self._pending, [] - for line in held: - await self._link.notify(protocol.AGENT_FRAME, {"channel": channel, "line": line}) + for method, params in held: + await self._link.notify(method, {"channel": channel, **params}) if self._cancel_wanted: self._cancel_wanted = False await self.end() @@ -397,6 +416,17 @@ async def stat(self, machine_id: str, path: str, *, roots: Sequence[str], hash: """What is at ``path`` on the machine: exists, kind, size, mtime, and its digest when asked.""" return await self.call(machine_id, protocol.STAT, {"path": path, "roots": list(roots), "hash": hash}) + async def write_file(self, machine_id: str, path: str, content: str, *, roots: Sequence[str]) -> None: + """Write ``content`` to ``path`` on the machine, whole, inside ``roots``; the directories above it are made.""" + params = {"path": path, "content": content, "roots": list(roots)} + answer = await self.call(machine_id, protocol.WRITE, params) + said = str(answer.get("text") or "") + # A refusal is the tool's own sentence ("Error: ...", "Error writing file: ..."), never an exception. + if answer.get("ok") is False or said.startswith("Error"): + label = (await self._acquire(machine_id)).target.label + reason = said.split(": ", 1)[1] if ": " in said else said + raise NodeError(f"{path} could not be written on machine {label}: {reason.strip() or 'refused'}") + async def tasks(self, machine_id: str) -> list[dict[str, Any]]: """Every task the machine's daemon runs or remembers: id, state, exit code, times, frames.""" answer = await self.call(machine_id, protocol.TASK_LIST, {}) diff --git a/raven/node/protocol.py b/raven/node/protocol.py index 732cccbbc..1e7ddc8bf 100644 --- a/raven/node/protocol.py +++ b/raven/node/protocol.py @@ -11,12 +11,13 @@ One connection per machine carries everything the host asks of it. Agents the daemon starts for the host each get a *channel* of that connection, on which the agent's ACP frames travel in both directions as ``agent/frame`` -notifications -- and a *task* id of the host's choosing, which outlives the -connection: an agent keeps running when the ssh goes, its frames are kept on -the machine, and a new connection picks them up from where the host stopped -reading (``agent/attach``). Requests are answered as they finish, not in the -order they arrive: a search of a large tree does not hold a heartbeat behind -it. +notifications (a CLI agent's output lines travel the same way, and its prompt +reaches its stdin as ``agent/stdin``) -- and a *task* id of the host's +choosing, which outlives the connection: an agent keeps running when the ssh +goes, its frames are kept on the machine, and a new connection picks them up +from where the host stopped reading (``agent/attach``). Requests are answered +as they finish, not in the order they arrive: a search of a large tree does +not hold a heartbeat behind it. Every ``fs/*`` call carries ``roots``, the directories it may touch. The host decides them; the daemon checks them again, because only the machine can @@ -27,7 +28,7 @@ #: Bumped when a method or a field changes meaning. A daemon and a host that #: disagree refuse each other at ``node/hello`` instead of misreading a reply. -PROTOCOL = 3 +PROTOCOL = 4 HELLO = "node/hello" PING = "node/ping" @@ -40,6 +41,7 @@ AGENT_LAUNCH = "agent/launch" AGENT_ATTACH = "agent/attach" AGENT_FRAME = "agent/frame" +AGENT_STDIN = "agent/stdin" AGENT_STDERR = "agent/stderr" AGENT_EXIT = "agent/exit" AGENT_CANCEL = "agent/cancel" @@ -100,6 +102,7 @@ "AGENT_FRAME", "AGENT_LAUNCH", "AGENT_STDERR", + "AGENT_STDIN", "BUSY", "CHANGED", "CHUNK", diff --git a/raven/node/server.py b/raven/node/server.py index 905cc9d3c..d52fd0754 100644 --- a/raven/node/server.py +++ b/raven/node/server.py @@ -12,7 +12,9 @@ An agent the host starts here (``agent/launch``) runs in its own process group, in the login environment this daemon captures once at its first need, under a *task* id of the host's. Its ACP frames travel as ``agent/frame`` -notifications on a channel of the connection that asked, and are kept in the +notifications on a channel of the connection that asked (a CLI agent's output +lines the same way, and its prompt the other way as ``agent/stdin``, which +also closes its stdin), and are kept in the task's record on disk as they arrive -- so when that connection goes, the agent keeps running, and the next connection takes the task over from the frame the host last read (``agent/attach``). Ending a task ends its whole @@ -797,6 +799,19 @@ async def write(self, line: str) -> None: except (BrokenPipeError, ConnectionResetError): pass + async def feed(self, data: str, *, eof: bool) -> None: + """``data`` to the agent's stdin as it is, then the end of it when ``eof``: how a CLI agent's prompt arrives.""" + if self.proc.stdin is None or self.proc.stdin.is_closing(): + return + if data: + self.proc.stdin.write(data.encode("utf-8")) + try: + await self.proc.stdin.drain() + except (BrokenPipeError, ConnectionResetError): + return + if eof: + self.proc.stdin.close() + def end(self) -> None: """End the agent and everything it started, in its group or out of it.""" if self.proc.returncode is not None: @@ -1084,6 +1099,14 @@ async def frame(self, params: dict[str, Any]) -> None: if task is not None and isinstance(line, str): await task.write(line) + async def stdin(self, params: dict[str, Any]) -> None: + """A CLI agent's input: ``data`` written as it is, and its stdin closed after it when ``eof``.""" + channel = params.get("channel") + task = self.channels.get(channel) if isinstance(channel, int) else None + data = params.get("data", "") + if task is not None and isinstance(data, str): + await task.feed(data, eof=bool(params.get("eof"))) + async def cancel(self, params: dict[str, Any]) -> dict[str, Any]: self._task(params).end() return {} @@ -1150,6 +1173,7 @@ def _node_handlers(node: Node) -> dict[str, Callable[[dict[str, Any]], Awaitable protocol.AGENT_LAUNCH: node.launch, protocol.AGENT_ATTACH: node.attach, protocol.AGENT_FRAME: node.frame, + protocol.AGENT_STDIN: node.stdin, protocol.AGENT_CANCEL: node.cancel, protocol.AGENT_CLOSE: node.close, protocol.TASK_LIST: node.tasks, diff --git a/raven/rpc/methods/subagents.py b/raven/rpc/methods/subagents.py index 024697528..f850d6b10 100644 --- a/raven/rpc/methods/subagents.py +++ b/raven/rpc/methods/subagents.py @@ -460,10 +460,11 @@ async def subagents_add(params: dict, *, agent_loop_factory: "AgentLoopFactory | entry = third_party_subagent_preset(preset_name) machine = _clean_name(params.get("machine"), field="machine") if machine: - _refuse_unless_it_can_run_there(entry, machine) + remote_cwd = params["remote_cwd"].strip() if isinstance(params.get("remote_cwd"), str) else "" + _refuse_unless_it_can_run_there(entry, machine, remote_cwd=remote_cwd) entry["machine"] = machine - if isinstance(params.get("remote_cwd"), str) and params["remote_cwd"].strip(): - entry["remoteCwd"] = params["remote_cwd"].strip() + if remote_cwd: + entry["remoteCwd"] = remote_cwd name = _clean_name(params.get("name"), field="name") if name: entry["name"] = name @@ -523,22 +524,33 @@ async def subagents_add(params: dict, *, agent_loop_factory: "AgentLoopFactory | return {"added": True, "name": entry["name"]} -def _refuse_unless_it_can_run_there(entry: dict, machine: str) -> None: +def _refuse_unless_it_can_run_there(entry: dict, machine: str, *, remote_cwd: str = "") -> None: """Refuse an add on a registered machine before anything is spent on it. - Only an acp preset runs on a machine: its command is started there by the - machine's Raven node. The machine is judged the way a dispatch judges it - (:func:`raven.acp_client.remote.machine`): the lane on, the row registered - and usable -- each refusal one sentence, with no address in it. + An acp or cli preset's command is started there by the machine's Raven + node; an openai preset's endpoint is reached from the machine through an + ssh tunnel, so it has no directory there, and a ``remote_cwd`` is refused + with it rather than written and ignored. A built-in agent is Raven's own + loop in this process and runs on no other machine. The machine is judged + the way a dispatch judges it (:func:`raven.acp_client.remote.machine`): + the lane on, the row registered and usable -- each refusal one sentence, + with no address in it. """ from raven.acp_client import remote - if entry.get("kind") != "acp": + shown = str(entry.get("preset") or entry.get("name")) + if entry.get("kind") == "builtin": raise ConfigValidationError( - f"preset {str(entry.get('preset') or entry.get('name'))!r} is not an acp agent; only an acp agent " - "runs on a registered machine", + f"preset {shown!r} is a built-in agent: Raven's own loop in this process, which runs on no other " + "machine; reach the machine from it with exec and the file tools instead", data={"field": "machine", "machine": machine}, ) + if entry.get("kind") == "openai" and remote_cwd: + raise ConfigValidationError( + f"preset {shown!r} is an openai agent, reached from machine {machine!r} through a tunnel: it has no " + "working directory there, so leave remote_cwd out", + data={"field": "remote_cwd", "machine": machine}, + ) try: remote.machine(machine) except remote.RemoteMachineError as exc: diff --git a/raven/rpc/models.py b/raven/rpc/models.py index 81c39443d..d919eb4ef 100644 --- a/raven/rpc/models.py +++ b/raven/rpc/models.py @@ -108,8 +108,8 @@ class SubagentRow(_Strict): machine: str | None = Field( default=None, description=( - "The registered machine an acp row runs on, by its id in the machine registry; null for this " - "computer, and on a server that predates the field." + "The registered machine a row runs on -- an openai row: reaches its endpoint from -- by its id in " + "the machine registry; null for this computer, and on a server that predates the field." ), ) vendored: bool = Field( @@ -2659,14 +2659,18 @@ class SubagentsAddParams(_Strict): machine: str | None = Field( default=None, description=( - "A registered machine, by its id in the machine registry, for an acp preset: the preset's command is " - "started there by the machine's Raven node and signed in there, and the row's name defaults to " - "@. Refused unless the machine lane is on and the machine is registered and usable." + "A registered machine, by its id in the machine registry: an acp or cli preset's command is started " + "there by the machine's Raven node and signed in there; an openai preset's endpoint is reached from " + "that machine through an ssh tunnel. The row's name defaults to @. Refused for a " + "built-in agent, and unless the machine lane is on and the machine is registered and usable." ), ) remote_cwd: str | None = Field( default=None, - description="Where on `machine` the agent works (`~/raven-work` when unset); ignored without `machine`.", + description=( + "Where on `machine` an acp or cli agent works (`~/raven-work` when unset); ignored without `machine`, " + "refused with an openai preset, which has no working directory there." + ), ) api_key: str | None = None mcps: list[str] | None = None diff --git a/rpc-schema/openrpc.json b/rpc-schema/openrpc.json index 4221ac41d..34aa3c763 100644 --- a/rpc-schema/openrpc.json +++ b/rpc-schema/openrpc.json @@ -2144,7 +2144,7 @@ { "name": "machine", "required": false, - "description": "A registered machine, by its id in the machine registry, for an acp preset: the preset's command is started there by the machine's Raven node and signed in there, and the row's name defaults to @. Refused unless the machine lane is on and the machine is registered and usable.", + "description": "A registered machine, by its id in the machine registry: an acp or cli preset's command is started there by the machine's Raven node and signed in there; an openai preset's endpoint is reached from that machine through an ssh tunnel. The row's name defaults to @. Refused for a built-in agent, and unless the machine lane is on and the machine is registered and usable.", "schema": { "type": "string" } @@ -2152,7 +2152,7 @@ { "name": "remote_cwd", "required": false, - "description": "Where on `machine` the agent works (`~/raven-work` when unset); ignored without `machine`.", + "description": "Where on `machine` an acp or cli agent works (`~/raven-work` when unset); ignored without `machine`, refused with an openai preset, which has no working directory there.", "schema": { "type": "string" } @@ -12035,7 +12035,7 @@ }, "machine": { "type": "string", - "description": "The registered machine an acp row runs on, by its id in the machine registry; null for this computer, and on a server that predates the field." + "description": "The registered machine a row runs on -- an openai row: reaches its endpoint from -- by its id in the machine registry; null for this computer, and on a server that predates the field." }, "vendored": { "type": "boolean", diff --git a/schemas/subagent.schema.json b/schemas/subagent.schema.json index dc25c78dd..d39fb7bf7 100644 --- a/schemas/subagent.schema.json +++ b/schemas/subagent.schema.json @@ -373,6 +373,18 @@ "title": "Kind", "type": "string" }, + "machine": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Machine" + }, "maxOutputChars": { "default": 30000, "title": "Maxoutputchars", @@ -454,6 +466,18 @@ "title": "Readslocalfiles", "type": "boolean" }, + "remoteCwd": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Remotecwd" + }, "resumeCommand": { "anyOf": [ { diff --git a/tests/test_node_client.py b/tests/test_node_client.py index d600ba57f..1e51b8602 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -1001,6 +1001,51 @@ async def test_a_file_is_fetched_from_the_machine_whole_and_checked(box, pool, t assert len(box.launches()) == 1, "all of it on the one link" +@pytest.mark.slow +async def test_a_cli_agent_s_input_travels_on_the_link_and_its_stdin_ends_when_told(box, pool): + box.node() + + endpoint = await pool.open_channel("box", name="cat@box", command="cat", cwd="~", env={}) + await endpoint.send_input("first line\nsecond line\n", eof=True) + + lines = [] + while raw := await endpoint.readline(): + lines.append(raw.decode("utf-8")) + assert lines == ["first line", "second line"], "what went in came back, line by line" + assert await endpoint.wait() == 0, "its stdin closed, cat left on its own" + assert len(box.launches()) == 1, "on the node's ssh, like a frame" + + +async def test_input_written_while_the_link_is_down_goes_out_in_order_once_it_is_back(): + endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="cli@box") # type: ignore[arg-type] + endpoint.detach("the link went") + await endpoint.write(b'{"a": 1}\n') + await endpoint.send_input("the prompt", eof=True) + back = _Client() + + await endpoint.reattach(back, 7) # type: ignore[arg-type] + + assert back.said == [ + (protocol.AGENT_FRAME, {"channel": 7, "line": '{"a": 1}'}), + (protocol.AGENT_STDIN, {"channel": 7, "data": "the prompt", "eof": True}), + ], "on the new channel, in the order they were written" + + +async def test_input_for_an_agent_that_has_ended_or_whose_link_just_broke_is_refused_or_held(): + endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="cli@box") # type: ignore[arg-type] + endpoint.exited(0) + with pytest.raises(BrokenPipeError, match="has ended"): + await endpoint.send_input("late", eof=True) + + class _Gone(_Client): + async def notify(self, method: str, params: dict[str, Any]) -> None: + raise AcpConnectionError("the ssh ended") + + held = ChannelEndpoint(_Gone(), 1, target=_box(), agent="cli@box") # type: ignore[arg-type] + await held.send_input("the prompt", eof=True) + assert held._pending == [(protocol.AGENT_STDIN, {"data": "the prompt", "eof": True})], "kept for the next link" + + async def test_frames_held_for_a_link_that_is_down_are_capped(tmp_path): endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="echo@box") # type: ignore[arg-type] endpoint.detach("the link went") diff --git a/tests/test_node_server.py b/tests/test_node_server.py index 6f7b60630..b5e8b9dc5 100644 --- a/tests/test_node_server.py +++ b/tests/test_node_server.py @@ -624,6 +624,45 @@ async def test_an_agent_starts_in_its_root_in_the_login_environment_plus_the_ent assert host.node.agents == {} +@pytest.mark.slow +async def test_a_cli_agent_s_prompt_reaches_its_stdin_as_it_is_and_the_end_of_it_lets_it_leave(tmp_path, agent): + host = _Host() + await host.ask(protocol.AGENT_LAUNCH, {"channel": 1, "command": agent, "cwd": str(tmp_path)}) + + prompt = _frame("one") + "\n" + _frame("two", 2) + "\n" + await host.tell(protocol.AGENT_STDIN, {"channel": 1, "data": prompt, "eof": True}) + + await host.until(lambda: host.notified(protocol.AGENT_EXIT)) + answered = [json.loads(frame["line"])["id"] for frame in host.notified(protocol.AGENT_FRAME)] + assert answered == [1, 2], "both lines, in order, before the end of its stdin" + assert host.notified(protocol.AGENT_EXIT) == [{"channel": 1, "rc": 0}], "stdin closed, the agent left on its own" + assert host.node.agents == {} + + +@pytest.mark.slow +async def test_the_end_of_stdin_alone_is_a_frame_too_and_one_for_no_channel_is_dropped(tmp_path, agent): + host = _Host() + await host.ask(protocol.AGENT_LAUNCH, {"channel": 1, "command": agent, "cwd": str(tmp_path)}) + + await host.tell(protocol.AGENT_STDIN, {"channel": 9, "data": "nobody", "eof": True}) + await host.tell(protocol.AGENT_STDIN, {"channel": 1, "eof": True}) + + await host.until(lambda: host.notified(protocol.AGENT_EXIT)) + assert host.notified(protocol.AGENT_FRAME) == [] and host.notified(protocol.AGENT_EXIT) == [{"channel": 1, "rc": 0}] + + +@pytest.mark.slow +async def test_input_after_the_end_of_stdin_is_dropped_and_the_agent_is_left_alone(tmp_path): + host = _Host() + await host.ask(protocol.AGENT_LAUNCH, {"channel": 1, "command": "sleep 2", "cwd": str(tmp_path)}) + + await host.tell(protocol.AGENT_STDIN, {"channel": 1, "eof": True}) + await host.tell(protocol.AGENT_STDIN, {"channel": 1, "data": "late", "eof": True}) + + await host.until(lambda: host.notified(protocol.AGENT_EXIT)) + assert host.notified(protocol.AGENT_EXIT) == [{"channel": 1, "rc": 0}], "ended on its own, not by the late input" + + @pytest.mark.slow async def test_cancel_ends_the_agent_s_whole_process_group(tmp_path, agent): host = _Host() diff --git a/tests/test_raven_config_tool.py b/tests/test_raven_config_tool.py index dccc62eeb..86931e220 100644 --- a/tests/test_raven_config_tool.py +++ b/tests/test_raven_config_tool.py @@ -1750,8 +1750,9 @@ async def test_a_refused_add_on_a_machine_sends_the_user_to_that_machine_and_run reply = await _run(tool, action="add", path="subagents", value='{"preset": "claude_code", "machine": "ravenx2"}') assert reply.startswith("Not added:") and "on machine 'ravenx2': any fix is made there" in reply - assert "It runs on machine 'ravenx2', so every fix is made there by the user" in reply - assert "exec refuses ssh to a registered machine" in reply + assert "It runs on machine 'ravenx2', so the fix is made there by the user" in reply + # exec reaches the machine, so a look there is offered; a sign-in or an install is not. + assert "exec(machine='ravenx2') can look there first" in reply assert "Install it with exec" not in reply and "Run it on its own with exec" not in reply @@ -1783,10 +1784,12 @@ async def test_describe_names_the_registered_machines_only_while_the_lane_is_on( tool.set_rpc_caller(Calls({"subagents.list": {"rows": []}})) monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - assert "machines an acp preset" not in await _run(tool, action="describe", path="subagents") + assert "machines a preset can be added on" not in await _run(tool, action="describe", path="subagents") monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") said = await _run(tool, action="describe", path="subagents") - line = next(ln for ln in said.splitlines() if ln.strip().startswith("machines an acp preset")) + line = next(ln for ln in said.splitlines() if ln.strip().startswith("machines a preset can be added on")) assert line.split("on:", 1)[1].split("--", 1)[0].strip() == "ravenx2 (RavenX 2)", "the local row is no machine" assert '"machine": ""' in line and "203.0.113" not in said + # Every kind, each its own way: a command runs there, an endpoint is reached from there. + assert "an acp or cli preset runs there" in line and "openai preset's endpoint is reached from there" in line diff --git a/tests/test_rpc_subagents.py b/tests/test_rpc_subagents.py index 737a777a2..8fbdc5f3c 100644 --- a/tests/test_rpc_subagents.py +++ b/tests/test_rpc_subagents.py @@ -3223,16 +3223,35 @@ async def test_add_on_a_machine_nobody_registered_is_refused_by_name_without_an_ assert _stored(config_path) == before -async def test_add_on_a_machine_takes_only_an_acp_preset( +async def test_add_on_a_machine_takes_an_openai_preset_and_no_directory_for_it( config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + """An openai preset's endpoint is reached from the machine through a tunnel: it works in no directory there.""" monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _pings_ok) + out = await subagents_add({"preset": "mirothinker", "machine": "ravenx2"}) + + assert out == {"added": True, "name": "mirothinker@ravenx2"} + entry = next(e for e in _stored(config_path) if e["name"] == "mirothinker@ravenx2") + assert entry["machine"] == "ravenx2" and entry["kind"] == "openai" and "remoteCwd" not in entry + assert "203.0.113" not in json.dumps(_stored(config_path)) + with pytest.raises(ConfigValidationError) as caught: - await subagents_add({"preset": "mirothinker", "machine": "ravenx2"}) + await subagents_add({"preset": "mirothinker", "machine": "ravenx2", "name": "lab-llm", "remote_cwd": "~/x"}) - assert "not an acp agent" in str(caught.value) and caught.value.data["machine"] == "ravenx2" - assert all(e.get("machine") is None for e in _stored(config_path)) + assert "leave remote_cwd out" in str(caught.value) and caught.value.data["field"] == "remote_cwd" + assert all(e["name"] != "lab-llm" for e in _stored(config_path)) + + +def test_a_built_in_agent_is_refused_a_machine_in_one_sentence() -> None: + """Raven's own loop runs in this process: there is no process of it to start elsewhere.""" + from raven.rpc.methods.subagents import _refuse_unless_it_can_run_there + + with pytest.raises(ConfigValidationError) as caught: + _refuse_unless_it_can_run_there({"kind": "builtin", "name": "research-raven"}, "ravenx2") + + assert "built-in agent" in str(caught.value) and "runs on no other machine" in str(caught.value) + assert caught.value.data == {"field": "machine", "machine": "ravenx2"} async def test_add_on_a_machine_lends_no_key( diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index cbb74dd9f..1c2a8b3ba 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -611,6 +611,13 @@ def get_default_model(self) -> str: assert "Working directory: /tmp" not in text assert "Collected outputs:" not in text, "nothing came back from this run" + # A run refused by the registry, the ssh or the node never made it: the + # line names the directory without claiming it exists. + await mgr._announce_result("t1", "Look", "look", "Error: it could not be reached", origin, "error") + failed = submitted[-1].text + assert "Working directory: on machine 'box', ~/raven-work/t1 (made there only if the run got that far)" in failed + assert "made there when the run started" not in failed + record = tmp_path / "nodes" / "t1.out.md" await mgr._announce_result("t1", "Look", "look", "done", origin, "ok", record_path=str(record)) assert "Collected outputs:" not in submitted[1].text, "nothing is brought back from a machine unasked" diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py new file mode 100644 index 000000000..57d02d94b --- /dev/null +++ b/tests/test_subagent_cli_remote.py @@ -0,0 +1,575 @@ +"""Sub-agents of every kind on a registered machine: a cli entry runs there through the machine's node, an +openai entry's endpoint is reached from there through a tunnel, and a built-in entry is told it runs here. + +The stand-in machine is the one ``tests/test_subagent_acp_remote.py`` builds: a stand-in ``ssh`` on PATH hands +the one remote-command argument to a shell in the machine user's home, so the real ssh line, the real Raven node +and the real command all run, with only the network missing. A cli agent here is a shell command, and what it +prints is its reply. For the tunnel, a second stand-in forwards a port the way ``ssh -N -L`` does. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +import pytest +from loguru import logger + +from raven.agent.subagent import probe +from raven.agent.subagent.backends import agent_meta, build_third_party_backend, format_agent_listing +from raven.agent.subagent.backends.cli_agent import CliAgentTimeoutError +from raven.agent.subagent.instances import InstanceRegistry +from raven.agent.subagent.mcp_grant import GrantedServer, McpGrant +from raven.agent.subagent.registry import AgentRegistry, _row_for +from raven.config.schema import ( + BuiltinAgentConfig, + MCPServerConfig, + ThirdPartyCliSubagentConfig, + ThirdPartyOpenAISubagentConfig, +) +from raven.config.update_subagents import reject_unsupported_openai_fields +from raven.spine.message import Media +from tests.test_subagent_acp_remote import HOST, PORT, Machine, box # noqa: F401 - the fixture, by name + +# Every dispatch, test and ping here starts a real node over the stand-in ssh, +# and the command through it: real processes, real seconds. +pytestmark = pytest.mark.slow + +# The stand-in for ``ssh -N -L``: it logs the call, fails the way ssh fails in +# the named modes, and otherwise forwards the local port to the far host and +# port, as sshd would from the machine. +_FORWARDING_SSH = """#!{python} +import json, os, socket, sys, threading +here = os.path.dirname(os.path.abspath(__file__)) +args = sys.argv[1:] +with open(os.path.join(here, "calls"), "a") as f: + f.write(json.dumps(args) + "\\n") +mode = open(os.path.join(here, "mode")).read().strip() +log = args[args.index("-E") + 1] if "-E" in args else None +open(os.path.join(here, "tunnel.pid"), "w").write(str(os.getpid())) +if mode == "unreachable": + open(log, "a").write("ssh: connect to host {host} port {port}: Operation timed out\\n") + sys.exit(255) +if mode == "denied": + open(log, "a").write("worker@{host}: Permission denied (publickey).\\n") + sys.exit(255) +if mode == "silent": + import time + time.sleep(30) +if "-N" not in args or "-L" not in args: + sys.stderr.write("this stand-in only forwards\\n") + sys.exit(2) +bind_host, bind_port, far_host, far_port = args[args.index("-L") + 1].rsplit(":", 3) +listener = socket.socket() +listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +listener.bind((bind_host, int(bind_port))) +listener.listen() + +def pump(src, dst): + try: + while data := src.recv(65536): + dst.sendall(data) + except OSError: + pass + finally: + for s in (src, dst): + try: + s.shutdown(socket.SHUT_RDWR) + except OSError: + pass + +while True: + near, _ = listener.accept() + try: + far = socket.create_connection((far_host, int(far_port))) + except OSError: + near.close() + continue + threading.Thread(target=pump, args=(near, far), daemon=True).start() + threading.Thread(target=pump, args=(far, near), daemon=True).start() +""" + + +@pytest.fixture +def logs(): + lines: list[str] = [] + token = logger.add(lambda message: lines.append(str(message)), level="INFO") + yield lines + logger.remove(token) + + +@pytest.fixture +def forwarding(box: Machine) -> Machine: + """The machine with the stand-in that forwards a port, in place of the one that runs a command.""" + ssh = box.bin / "ssh" + ssh.write_text(_FORWARDING_SSH.format(python=sys.executable, host=HOST, port=PORT), encoding="utf-8") + ssh.chmod(0o755) + return box + + +def cli_config(command: str, **kw: Any) -> ThirdPartyCliSubagentConfig: + return ThirdPartyCliSubagentConfig( + name=kw.pop("name", "where@box"), + command=command, + machine=kw.pop("machine", "box"), + remote_cwd=kw.pop("remote_cwd", "~/raven-work"), + **kw, + ) + + +def http_config(base_url: str, **kw: Any) -> ThirdPartyOpenAISubagentConfig: + return ThirdPartyOpenAISubagentConfig( + name=kw.pop("name", "llm@box"), base_url=base_url, model="m", machine="box", **kw + ) + + +async def _run(cfg: Any, task: str, workspace: Path, **kw: Any) -> str: + return await build_third_party_backend(cfg, **{k: kw.pop(k) for k in ("registry",) if k in kw}).run( + task, task_id=kw.pop("task_id", "t1"), workspace=workspace, executor=None, **kw + ) + + +def _split(reply: str) -> tuple[str, str]: + """The agent's own words, and raven's notes after them.""" + head, _, notes = reply.partition("\n\n[raven] ") + return head, notes + + +async def _until(check: Any, *, timeout: float = 10.0) -> None: + deadline = time.monotonic() + timeout + while not check(): + assert time.monotonic() < deadline, "did not happen in time" + await asyncio.sleep(0.05) + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + try: + done, _ = os.waitpid(pid, os.WNOHANG) + return done == pid + except ChildProcessError: + return False + + +# ---- a cli agent on the machine -------------------------------------------------- + + +async def test_a_cli_dispatch_runs_on_the_machine_in_a_directory_made_there_and_says_where(box, tmp_path): + local = tmp_path / "local-workspace" + + reply = await _run(cli_config("sh -c pwd"), "hi", local, instance="reviewer") + + there = box.home / "raven-work" / "reviewer" + assert there.is_dir(), "the run's directory is made on the machine" + head, note = _split(reply) + assert Path(head.strip()).resolve() == there.resolve(), "the command ran in it" + assert note.startswith("Ran on machine 'Lab box' (box), in ") + assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in note + assert "the files it wrote are there, not on this computer" in note + assert not local.exists(), "the caller's workspace is never touched" + for argv in box.calls(): + assert str(local) not in " ".join(argv), "this computer's workspace path never reaches the machine" + + +async def test_the_prompt_reaches_a_cli_agent_there_the_three_ways_it_does_here(box, tmp_path): + prompt = "first line\n\nthird line, after a blank one" + + on_stdin, _ = _split(await _run(cli_config("cat"), prompt, tmp_path, task_id="t-stdin")) + as_argument, _ = _split(await _run(cli_config("printf %s {prompt}"), prompt, tmp_path, task_id="t-arg")) + from_file, _ = _split(await _run(cli_config("cat {prompt_file}"), prompt, tmp_path, task_id="t-file")) + + # The node forwards whole lines, so a blank line of a plain answer does not come back. + assert on_stdin == as_argument == from_file == "first line\nthird line, after a blank one" + written = box.home / "raven-work" / "t-file" / ".raven" / "prompt-t-file.txt" + assert written.read_text(encoding="utf-8") == prompt, "the prompt file is written into the run's directory there" + assert not list(tmp_path.glob("raven_subagent_*")), "no prompt file is made on this computer" + + +async def test_the_entry_s_env_reaches_the_agent_on_the_link_and_this_computer_s_shell_stays_here( + box, tmp_path, monkeypatch +): + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "raven-home")) + command = "sh -c 'printf %s \"$FROM_ENTRY:$RAVEN_SUBAGENT:${RAVEN_HOME:-unset}\"'" + + head, _ = _split(await _run(cli_config(command, env={"FROM_ENTRY": "sk-raven-test-not-real"}), "hi", tmp_path)) + + assert head == "sk-raven-test-not-real:1:unset", "the entry's env and the role mark, and not this host's home" + for argv in box.calls(): + assert "sk-raven-test-not-real" not in " ".join(argv), "a value in env is never an ssh argument" + + +async def test_a_cli_agent_that_exits_non_zero_there_fails_with_its_one_line(box, tmp_path): + with pytest.raises(RuntimeError, match=r"CLI agent 'where@box' exited 3: token expired$"): + await _run(cli_config("sh -c 'echo token expired; exit 3'"), "hi", tmp_path) + + +async def test_a_timeout_ends_the_agent_s_whole_process_tree_on_the_machine(box, tmp_path): + cfg = cli_config("sh -c 'echo $$ > pid.txt; exec sleep 30'", timeout=1) + + with pytest.raises(CliAgentTimeoutError, match="timed out after 1s"): + await _run(cfg, "hi", tmp_path, instance="slow") + + pid = int((box.home / "raven-work" / "slow" / "pid.txt").read_text(encoding="utf-8")) + await _until(lambda: _gone(pid)) + + +async def test_a_cancel_ends_the_agent_s_whole_process_tree_on_the_machine(box, tmp_path): + cfg = cli_config("sh -c 'echo $$ > pid.txt; exec sleep 30'") + pid_file = box.home / "raven-work" / "stopped" / "pid.txt" + turn = asyncio.ensure_future(_run(cfg, "hi", tmp_path, instance="stopped")) + await _until(pid_file.exists) + pid = int(pid_file.read_text(encoding="utf-8")) + assert not _gone(pid) + + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await turn + + await _until(lambda: _gone(pid)) + + +async def test_a_stateful_cli_agent_resumes_its_session_on_the_machine(box, tmp_path): + cfg = cli_config("sh -c 'echo created {agent_id}'", resume_command="sh -c 'echo resumed {agent_id}'") + registry = InstanceRegistry(path=tmp_path / "instances.json") + + first, _ = _split(await _run(cfg, "hi", tmp_path, registry=registry, instance="r1", task_id="t1")) + second, _ = _split(await _run(cfg, "again", tmp_path, registry=registry, instance="r1", task_id="t2")) + + assert first.startswith("created ") and second == f"resumed {first.removeprefix('created ')}" + assert (box.home / "raven-work" / "r1").is_dir(), "one directory per handle, both turns in it" + + +async def test_the_files_that_came_with_the_task_are_named_by_address_and_not_copied(box, tmp_path): + data = tmp_path / "sales.csv" + data.write_text("day,amount\n1,3\n", encoding="utf-8") + media = (Media(path=str(data), mime="text/csv", kind="file"),) + + head, _ = _split(await _run(cli_config("cat"), "count the rows", tmp_path, media=media, instance="counter")) + + # One line break, not the note's two: the node forwards whole lines, and a blank one does not come back. + assert head.startswith("count the rows\nFiles that came with this task, by their address on the computer") + assert "Nothing was copied here" in head and f"- {data}" in head + assert "[attachments, by absolute path]" not in head, "not the local note, whose paths mean nothing there" + assert not (box.home / "raven-work" / "counter" / "sales.csv").exists() + + +async def test_mcp_servers_are_not_lent_to_a_cli_agent_on_a_machine_and_the_reply_says_why(box, tmp_path): + cfg = cli_config("cat {mcp_file}", mcps=["pg"]) + grant = McpGrant(granted=(GrantedServer("pg", "stdio", MCPServerConfig(command="/bin/true")),)) + + reply = await _run(cfg, "hi", tmp_path, mcp_grant=grant) + + head, notes = _split(reply) + assert json.loads(head) == {"tools": {"mcpServers": {}, "disabledTools": []}}, "a file there, naming no server" + assert "MCP servers ['pg'] were not lent to cli agent 'where@box': it runs on machine 'box'" in notes + assert not list(tmp_path.glob("raven_subagent_*")), "no MCP file is made on this computer" + + +async def test_this_computer_s_files_are_not_accounted_for_a_remote_run(box, tmp_path, monkeypatch): + from raven.agent.subagent.backends import cli_agent + + looked: list[str] = [] + monkeypatch.setattr(cli_agent, "take_snapshot", lambda cwd: looked.append(cwd) or {}) + + await _run(cli_config("cat"), "hi", tmp_path) + + assert looked == [], "no snapshot of a directory here: the run's files are on the machine" + + +async def test_an_unreachable_machine_fails_a_cli_dispatch_in_one_sentence(box, tmp_path): + box.mode("unreachable") + + with pytest.raises(RuntimeError) as caught: + await _run(cli_config("cat"), "hi", tmp_path) + + said = str(caught.value) + assert said.startswith("CLI agent 'where@box': ") + assert "machine 'Lab box' (box) did not answer within the connection timeout" in said + assert HOST not in said and str(PORT) not in said + + +async def test_an_unregistered_machine_fails_a_cli_dispatch_before_anything_starts(box, tmp_path): + with pytest.raises(RuntimeError, match="machine 'elsewhere' is not registered"): + await _run(cli_config("cat", machine="elsewhere"), "hi", tmp_path) + assert box.calls() == [] + + +async def test_a_command_not_on_the_machine_and_a_prompt_file_it_cannot_write_each_say_so(box, tmp_path): + with pytest.raises(RuntimeError, match="command was not found on machine 'Lab box'"): + await _run(cli_config("no-such-agent-for-raven-tests {prompt}"), "hi", tmp_path) + + # The run's `.raven/` is a file there: the prompt file cannot be made under it. + blocked = box.home / "raven-work" / "blocked" + blocked.mkdir(parents=True) + (blocked / ".raven").write_text("not a directory", encoding="utf-8") + with pytest.raises(RuntimeError, match=r"prompt-t1\.txt could not be written on machine 'Lab box'"): + await _run(cli_config("cat {prompt_file}"), "hi", tmp_path, instance="blocked") + + +async def test_a_process_that_reaches_no_machine_says_so_before_anything_starts(box, tmp_path, monkeypatch): + from raven.ops import remote_path + + monkeypatch.setattr(remote_path, "_NODES", None) + + with pytest.raises(RuntimeError, match="this process does not reach registered machines"): + await _run(cli_config("cat"), "hi", tmp_path) + assert box.calls() == [] + + +# ---- checks -------------------------------------------------------------------------- + + +async def test_the_free_probe_of_a_cli_row_does_not_reach_the_machine_and_a_test_runs_there(box): + cfg = cli_config("sh -c 'echo PONG'") + + probed = await probe.probe_one(cfg, source="config") + + assert probed.status == "attention" and probed.target == "box" + assert "runs on machine 'Lab box' (box)" in probed.detail and "run a test" in probed.detail + assert box.calls() == [], "no ssh for a listing" + + tested = await probe.run_test(cfg, source="config") + + assert tested.ok is True and tested.detail == "the agent ran and replied" and tested.remedy is None + assert len(box.calls()) == 1, "the test went to the machine, over the node's one ssh" + + +async def test_a_ping_of_a_cli_row_answers_from_the_machine_and_leaves_one_directory(box): + pinged = await probe.ping_agent(cli_config("cat")) + + assert pinged.ok is True and pinged.detail == "it ran and replied" + assert sorted(p.name for p in (box.home / "raven-work").iterdir()) == ["raven-check"] + + +async def test_a_failed_ping_of_a_cli_row_says_the_fix_is_made_on_the_machine(box): + pinged = await probe.ping_agent(cli_config("sh -c 'echo not signed in >&2; exit 1'")) + + assert pinged.ok is False + assert "exited 1: not signed in" in pinged.detail and pinged.detail.endswith( + "(on machine 'box': any fix is made there)" + ) + assert pinged.remedy is None, "nothing offered to run here" + + +# ---- an openai agent reached from the machine ------------------------------------------- + + +async def _chat_completions_stub() -> tuple[Any, int, list[tuple[list[str], dict[str, Any]]]]: + """An endpoint on this computer's loopback: the far end of the stand-in tunnel.""" + seen: list[tuple[list[str], dict[str, Any]]] = [] + + async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + head = await reader.readuntil(b"\r\n\r\n") + lines = head.decode("utf-8", "replace").split("\r\n") + length = next((int(h.split(":", 1)[1]) for h in lines if h.lower().startswith("content-length:")), 0) + body = await reader.readexactly(length) if length else b"" + except (asyncio.IncompleteReadError, ConnectionError): + # The tunnel's own readiness check connects and leaves. + writer.close() + return + seen.append((lines, json.loads(body) if body else {})) + answer = {"choices": [{"message": {"content": "pong through the tunnel"}}], "usage": {"total_tokens": 3}} + payload = json.dumps(answer).encode("utf-8") + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n" + % len(payload) + + payload + ) + await writer.drain() + writer.close() + + server = await asyncio.start_server(handle, "127.0.0.1", 0) + return server, server.sockets[0].getsockname()[1], seen + + +async def test_an_openai_row_on_a_machine_is_reached_through_a_tunnel_from_it(forwarding, tmp_path, monkeypatch): + server, port, seen = await _chat_completions_stub() + # This computer's proxy is not where a port on its own loopback is. + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:9") + monkeypatch.delenv("NO_PROXY", raising=False) + try: + reply = await _run(http_config(f"http://localhost:{port}/v1"), "hi", tmp_path) + finally: + server.close() + await server.wait_closed() + + assert reply == "pong through the tunnel" + (headers, body) = seen[-1] + assert headers[0] == "POST /v1/chat/completions HTTP/1.1" + assert any(h.lower() == f"host: localhost:{port}" for h in headers), "the name the server knows itself by" + assert body["model"] == "m" and body["messages"][-1] == {"role": "user", "content": "hi"} + (argv,) = forwarding.calls() + assert "-N" in argv and "ExitOnForwardFailure=yes" in argv and "BatchMode=yes" in argv + forward = argv[argv.index("-L") + 1] + assert forward.startswith("127.0.0.1:") and forward.endswith(f":localhost:{port}") + pid = int((forwarding.bin / "tunnel.pid").read_text(encoding="utf-8")) + await _until(lambda: _gone(pid)) # the tunnel ends with the turn + + +async def test_the_free_probe_of_an_openai_row_asks_no_network_and_a_test_asks_through_the_tunnel(forwarding, tmp_path): + server, port, seen = await _chat_completions_stub() + cfg = http_config(f"http://localhost:{port}/v1", api_key="sk-raven-test-not-real") + try: + probed = await probe.probe_one(cfg, source="config") + assert probed.status == "attention" and "runs on machine 'Lab box' (box)" in probed.detail + assert "run a test" in probed.detail + assert forwarding.calls() == [] and seen == [], "nothing is asked of the network for a listing" + + tested = await probe.run_test(cfg, source="config") + finally: + server.close() + await server.wait_closed() + + assert tested.ok is True and tested.detail == "it ran and replied" + assert [argv for argv in forwarding.calls() if "-L" in argv], "the test went through the tunnel" + assert any("Bearer sk-raven-test-not-real" in h for h in seen[-1][0]), "the row's own key, through it" + + +async def test_a_tunnel_that_does_not_open_fails_the_turn_in_one_sentence_without_the_address(forwarding, tmp_path): + forwarding.mode("unreachable") + + with pytest.raises(RuntimeError) as caught: + await _run(http_config("http://localhost:1/v1"), "hi", tmp_path) + + said = str(caught.value) + assert said.startswith("OpenAI-API agent 'llm@box': ") + assert "machine 'Lab box' (box) did not answer within the connection timeout" in said + assert HOST not in said and str(PORT) not in said + + +async def test_an_endpoint_the_machine_cannot_reach_fails_the_turn_in_one_sentence(forwarding, tmp_path): + # The stand-in forwards, and the far port answers nobody: what the tunnel carries back is a closed connection. + with pytest.raises(RuntimeError, match="did not answer through the tunnel from machine 'Lab box'"): + await _run(http_config("http://localhost:1/v1"), "hi", tmp_path) + + +async def test_a_tunnel_that_never_opens_is_given_up_on_in_one_sentence(forwarding, tmp_path, monkeypatch): + from raven.acp_client import remote + + forwarding.mode("silent") + monkeypatch.setattr(remote, "TUNNEL_READY_S", 2.0) + target = remote.machine("box") + + with pytest.raises(remote.RemoteMachineError, match="machine 'Lab box' \\(box\\) did not open the tunnel within"): + async with remote.tunnel(target, "http://localhost:1/v1", ssh_log=tmp_path / "t.log", connect_timeout=0): + pass + pid = int((forwarding.bin / "tunnel.pid").read_text(encoding="utf-8")) + await _until(lambda: _gone(pid)) # the ssh that never opened it is ended + + +async def test_a_tunnel_names_what_it_cannot_do_before_any_ssh(box, tmp_path, monkeypatch): + from raven.acp_client import remote + + target = remote.machine("box") + with pytest.raises(remote.RemoteMachineError, match="names no host"): + async with remote.tunnel(target, "http:///v1", ssh_log=tmp_path / "t.log"): + pass + + async def no_ssh(*argv, **kw): + raise OSError("no ssh here") + + monkeypatch.setattr(remote.asyncio, "create_subprocess_exec", no_ssh) + with pytest.raises(remote.RemoteMachineError, match="ssh could not be started: no ssh here"): + async with remote.tunnel(target, "http://localhost:1/v1", ssh_log=tmp_path / "t.log"): + pass + with pytest.raises(RuntimeError, match="machine 'elsewhere' is not registered"): + await _run(http_config("http://localhost:1/v1").model_copy(update={"machine": "elsewhere"}), "hi", tmp_path) + assert box.calls() == [] + + +async def test_an_https_endpoint_on_a_machine_is_warned_on_load_and_refused_before_any_ssh(box, tmp_path, logs): + cfg = http_config("https://llm.lab.internal/v1") + + assert any("carries http only" in line and "'llm@box'" in line for line in logs) + with pytest.raises(RuntimeError, match="has to be http"): + await _run(cfg, "hi", tmp_path) + assert box.calls() == [] + + +# ---- the roster, the table and the schema, for every kind ---------------------------------- + + +def test_the_roster_names_the_machine_for_every_kind_and_hands_it_no_paths(): + cli = cli_config("cat") + http = http_config("http://llm.lab.internal:8000/v1") + + for meta in (agent_meta(cli), agent_meta(http)): + assert meta.machine == "box" and meta.reads_local_files is False + listing = format_agent_listing([agent_meta(cli), agent_meta(http)]) + assert listing.count("on machine box") == 2 and HOST not in listing + assert _row_for(cli).caps.machine == "box" and _row_for(http).caps.machine == "box" + + +def test_a_built_in_row_cannot_name_a_machine_and_is_told_so(logs): + cfg = BuiltinAgentConfig(name="research-raven", machine="box") + + assert cfg.machine is None and agent_meta(cfg).machine == "" + said = next(line for line in logs if "'research-raven'" in line) + assert "cannot run on" in said and "Raven's own loop runs in this process" in said and "remove `machine`" in said + assert BuiltinAgentConfig(name="research-raven").machine is None + + +def test_a_cli_row_s_machine_is_an_id_or_nothing(): + assert ThirdPartyCliSubagentConfig(name="x", command="cat", machine=" box ").machine == "box" + assert ThirdPartyCliSubagentConfig(name="x", command="cat", machine=" ").machine is None + assert ThirdPartyCliSubagentConfig(name="x", command="cat").remote_cwd is None + + +def test_an_openai_row_drops_a_remote_cwd_with_a_warning_and_the_write_door_refuses_it(logs): + row = { + "name": "llm", + "kind": "openai", + "baseUrl": "http://x/v1", + "model": "m", + "machine": "box", + "remoteCwd": "~/w", + } + + loaded = ThirdPartyOpenAISubagentConfig.model_validate(row) + + assert loaded.machine == "box" and not hasattr(loaded, "remote_cwd") + assert any("remoteCwd is not supported for kind 'openai'" in line for line in logs) + with pytest.raises(ValueError, match="remoteCwd is not supported for kind 'openai'"): + reject_unsupported_openai_fields([row]) + + +def test_with_the_lane_off_a_cli_machine_row_stays_off_the_table(box, monkeypatch): + registry = AgentRegistry(build_builtin=lambda row, narrowed: ("backend", row.name)) + local = ThirdPartyCliSubagentConfig(name="here", command="cat") + + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + registry.apply([cli_config("cat"), local]) + assert "where@box" not in registry.names() and "here" in registry.names() + + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + registry.apply([cli_config("cat"), local]) + assert "where@box" in registry.names() + + +async def test_with_the_lane_off_a_cli_dispatch_says_so_and_reaches_nothing(box, monkeypatch, tmp_path): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + + with pytest.raises(RuntimeError, match="the machine lane is debug-only and off"): + await _run(cli_config("cat"), "hi", tmp_path) + assert box.calls() == [] + + +async def test_the_announce_names_a_cli_row_s_directory_on_the_machine(box): + from raven.agent.subagent.manager import SubagentManager + + class _Provider: + def get_default_model(self) -> str: + return "m" + + mgr = SubagentManager(provider=_Provider(), workspace=Path("/tmp"), agents=[cli_config("cat")]) + + assert mgr._machine_workdir("where@box", "reviewer") == ("box", "~/raven-work/reviewer") diff --git a/tests/test_subagent_direct_chat.py b/tests/test_subagent_direct_chat.py index a1cc36952..1ce680117 100644 --- a/tests/test_subagent_direct_chat.py +++ b/tests/test_subagent_direct_chat.py @@ -880,6 +880,50 @@ async def spying_run(task, **kw): assert seen["task"] == f"use it\n\n{UNDELIVERABLE_NOTE}\n- house.pptx" +@pytest.mark.asyncio +async def test_chat_keeps_a_machine_row_s_attachments_for_its_backend_to_name_by_address(tmp_path, monkeypatch): + """A row on a registered machine carries [no-local-files] too, but its backend + names each attachment by its address on this computer and copies nothing + (``remote.files_note``), so the attachments stay for it and the page's + bullet is spelled the way the note will spell it.""" + import dataclasses + + from raven.agent.subagent.attachments import UNDELIVERABLE_NOTE + from raven.spine.message import Media + + manager = _direct_chat_manager(tmp_path, monkeypatch) + real_get = manager.registry.get + + def machine_row(name): + row = real_get(name) + if row is None: + return None + return dataclasses.replace(row, caps=dataclasses.replace(row.caps, reads_local_files=False, machine="box")) + + monkeypatch.setattr(manager.registry, "get", machine_row) + seen: dict[str, Any] = {} + real_run = manager.registry.backend("raven").run + + async def spying_run(task, **kw): + seen["media"] = kw.get("media") + seen["task"] = task + return await real_run(task, **kw) + + monkeypatch.setattr(manager.registry.backend("raven"), "run", spying_run) + deck = manager.workspace / "uploads" / "house.pptx" + deck.parent.mkdir(parents=True) + deck.write_bytes(b"deck") + attached = (Media(path=str(deck), mime="application/octet-stream", kind="file"),) + + await manager.chat( + session_key="s1", agent="raven", handle="deck", text="use it\n- uploads/house.pptx", media=attached + ) + + assert seen["media"] == attached, "kept, for the backend to name by address" + assert seen["task"] == f"use it\n- {deck.resolve()}", "the bullet names the file as the address note will" + assert UNDELIVERABLE_NOTE not in seen["task"] + + @pytest.mark.asyncio async def test_a_direct_chat_runs_in_the_session_workdir_not_agent_home(tmp_path, monkeypatch): """The same directory the turn's own tools get, and that ``spawn`` captures. diff --git a/ui-tui/src/rpc/generated.ts b/ui-tui/src/rpc/generated.ts index 48aaadd40..51f3f6159 100644 --- a/ui-tui/src/rpc/generated.ts +++ b/ui-tui/src/rpc/generated.ts @@ -1014,7 +1014,7 @@ export interface SubagentRow { enabled: boolean; configured: boolean; /** - * The registered machine an acp row runs on, by its id in the machine registry; null for this computer, and on a server that predates the field. + * The registered machine a row runs on -- an openai row: reaches its endpoint from -- by its id in the machine registry; null for this computer, and on a server that predates the field. */ machine?: string; /** diff --git a/ui-web/src/rpc/generated.ts b/ui-web/src/rpc/generated.ts index d8fbccca8..5acfe7224 100644 --- a/ui-web/src/rpc/generated.ts +++ b/ui-web/src/rpc/generated.ts @@ -827,7 +827,7 @@ export interface SubagentRow { enabled: boolean; configured: boolean; /** - * The registered machine an acp row runs on, by its id in the machine registry; null for this computer, and on a server that predates the field. + * The registered machine a row runs on -- an openai row: reaches its endpoint from -- by its id in the machine registry; null for this computer, and on a server that predates the field. */ machine?: string; /** From 906b92c9606aa9954b9e848cdcbf7a04d3a46a66 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:03:33 +0800 Subject: [PATCH 20/44] fix(node): read pages by chunks, keep reads out of writes, give uv pip's index, stay in node_dir Four notes of the review, each measured on this branch: - A page read decoded and split the whole file, two C calls that hold the interpreter for as long as the file takes (7.5 s on 1 GiB), so a node's heartbeat went unanswered and a host's turn stood still. `read_file` now reads the file a chunk at a time, counts the line breaks `splitlines` knows as the bytes come, holds back what could begin a break at a chunk's edge, and decodes only the page's own lines; the file is never whole in memory. Same lines, same footer, and a byte that is not UTF-8 refuses the page it is on as it refused the file. - In a worker thread a page read and the digest taken beside it could land inside a write of the same file and leave with the old page and the new digest, which a later write carrying that digest as `expect` would take as leave. The whole-file calls now go through one gate: readers together, a writer alone, a waiting writer before new readers. - uv reads no pip configuration, so a machine that reaches only the mirror pip is configured with could not be installed. The probe reads that index the way pip does (`PIP_INDEX_URL`, then `index-url` in pip's files, user over system, install over global), the plan names it, and `uv pip install` gets it as `--default-index`. - The uv fetched for a machine without one wrote a receipt under the home directory and was whichever version was current; it is now one pinned version, installed unmanaged, which writes no receipt and changes no PATH. And `node_dir` is made owner-only only when the install makes it: one that was there is the owner's. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/filesystem.py | 71 +++++++++++++++++++++--- raven/node/install.py | 95 ++++++++++++++++++++++++--------- raven/node/server.py | 76 +++++++++++++++++++++----- tests/test_filesystem_tools.py | 61 +++++++++++++++++++++ tests/test_node_install.py | 91 +++++++++++++++++++++++++++---- tests/test_node_server.py | 84 +++++++++++++++++++++++++++++ 6 files changed, 424 insertions(+), 54 deletions(-) diff --git a/raven/agent/tools/filesystem.py b/raven/agent/tools/filesystem.py index df4551829..67156ebe8 100644 --- a/raven/agent/tools/filesystem.py +++ b/raven/agent/tools/filesystem.py @@ -21,6 +21,63 @@ # line cap anyway) and the prompt shows the path alone. _PREVIEW_MAX_BYTES = 256 * 1024 +#: How a page of a text file is read: this many bytes at a time, the line +#: breaks counted as the bytes come and only the page's own lines decoded. +#: ``read_text().splitlines()`` held the interpreter for the whole file -- +#: 7.5 s on a 1 GiB file (decode 2.7 s, split 4.8 s, measured 2026-10-13), +#: during which nothing else ran: on a Raven node, no heartbeat answered; +#: here, no turn served. Each step is now one chunk's, and the file is never +#: whole in memory. +_PAGE_CHUNK = 4 * 1024 * 1024 + +# Where ``str.splitlines`` ends a line, as UTF-8 bytes: the two-byte ``\r\n`` +# first so it counts once, the one-byte breaks, then NEL, LS and PS. +_LINE_BREAK = re.compile(rb"\r\n|[\n\r\x0b\x0c\x1c\x1d\x1e]|\xc2\x85|\xe2\x80[\xa8\xa9]") +# The breaks other than ``\n``: a chunk with none splits on ``\n`` alone, the +# common case, in one C call. Looked for one by one with ``in``, a search at +# memchr speed; a regular expression over the same bytes took twenty times +# as long (measured 2026-10-13 on a 1 GiB file: 80 ms a chunk against 4). +_ODD_BREAKS = (b"\r", b"\x0b", b"\x0c", b"\x1c", b"\x1d", b"\x1e", b"\xc2\x85", b"\xe2\x80\xa8", b"\xe2\x80\xa9") + + +def _page_of_lines(handle: Any, start: int, limit: int) -> tuple[list[bytes], int]: + """Lines ``start`` to ``start + limit`` (0-based) of the bytes ``handle`` reads, and how many lines there are. + + Exactly the lines ``read_text(encoding="utf-8").splitlines()`` would give, + read a chunk at a time. A break may straddle two chunks, so the bytes + that could begin one are held back until the next chunk says; the tail + after the last break is a line only when it holds something, as + ``splitlines`` has it. + """ + page: list[bytes] = [] + total = 0 + stop = start + limit + + def take(lines: list[bytes]) -> None: + nonlocal total + if total < stop and total + len(lines) > start: + page.extend(lines[max(start - total, 0) : stop - total]) + total += len(lines) + + def cut(body: bytes) -> bytes: + """Count and keep the complete lines of ``body``; what follows the last break comes back.""" + odd = any(sign in body for sign in _ODD_BREAKS) + pieces = _LINE_BREAK.split(body) if odd else body.split(b"\n") + take(pieces[:-1]) + return pieces[-1] + + buffer = b"" + while chunk := handle.read(_PAGE_CHUNK): + buffer += chunk + hold = 2 if buffer.endswith(b"\xe2\x80") else (1 if buffer.endswith((b"\r", b"\xc2", b"\xe2")) else 0) + kept = buffer[len(buffer) - hold :] if hold else b"" + buffer = cut(buffer[: len(buffer) - hold] if hold else buffer) + kept + if buffer: + tail = cut(buffer) + if tail: + take([tail]) + return page, total + def _write_text_bytes(content: str) -> bytes: """The bytes ``Path.write_text`` would leave on disk for ``content``. @@ -274,19 +331,21 @@ async def execute(self, path: str, offset: int = 1, limit: int | None = None, ** return f"Error decoding image {path}: {e}" # An extension-only guess that would not decode: fall through. - all_lines = fp.read_text(encoding="utf-8").splitlines() - total = len(all_lines) - if offset < 1: offset = 1 + start = offset - 1 + with fp.open("rb") as fh: + raw_page, total = _page_of_lines(fh, start, limit or self._DEFAULT_LIMIT) if total == 0: return f"(Empty file: {path})" if offset > total: return f"Error: offset {offset} is beyond end of file ({total} lines)" - start = offset - 1 - end = min(start + (limit or self._DEFAULT_LIMIT), total) - numbered = [f"{start + i + 1}| {line}" for i, line in enumerate(all_lines[start:end])] + # Strict, as the whole-file decode was: a byte that is not UTF-8 + # on the page refuses the read with the codec's own sentence. + lines = [raw.decode("utf-8") for raw in raw_page] + end = start + len(lines) + numbered = [f"{start + i + 1}| {line}" for i, line in enumerate(lines)] result = "\n".join(numbered) if len(result) > self._MAX_CHARS: diff --git a/raven/node/install.py b/raven/node/install.py index 43e95c9d3..1612da8fa 100644 --- a/raven/node/install.py +++ b/raven/node/install.py @@ -5,14 +5,19 @@ virtualenv, the packages in :data:`raven.node.protocol.REQUIREMENTS` pinned to this Raven's own versions, and this Raven's own code (:mod:`raven.node.bundle`). Nothing else on the machine changes: no system package, no shell rc file, no -environment it already had. +environment it already had, no file outside ``node_dir``, and a ``node_dir`` +that was there before keeps its mode (only one this makes is made owner-only). Everything Python goes through ``uv``, as it does in this repository: the virtualenv is ``uv venv`` of the machine's own Python 3.12 when it has one, else of a Python uv keeps inside ``node_dir``; the packages are ``uv pip -install``. A machine without uv gets one under ``node_dir/.uv`` first, from -uv's own installer, fetched with the machine's curl or wget and told to -touch nothing else; with neither uv nor a way to fetch it, the install stops +install``, from the index pip is configured with on the machine when it is +(``PIP_INDEX_URL``, or ``index-url`` in pip's config files, which uv does +not read on its own), else from PyPI. A machine without uv gets one under +``node_dir/.uv`` first: uv's installer for one pinned version +(:data:`UV_VERSION`), fetched with the machine's curl or wget and run as an +unmanaged install, which writes no receipt under the home directory and +changes no PATH; with neither uv nor a way to fetch it, the install stops and says so. Each step is one ssh call through :func:`raven.acp_client.remote.run`; a step @@ -38,8 +43,13 @@ _PYTHON_KB = 80 * 1024 _UV_KB = 25 * 1024 -#: uv's own installer, which puts one binary where ``UV_INSTALL_DIR`` says. +#: uv's own installer, as a person is told to run it when the machine has no uv and no way to fetch one. UV_INSTALLER = "https://astral.sh/uv/install.sh" +#: The uv an install fetches when the machine has none: one version, as the +#: packages are one version each, so two installs a month apart put the same +#: uv on two machines. The installer for a version is served under its number. +UV_VERSION = "0.13.0" +UV_PINNED_INSTALLER = f"https://astral.sh/uv/{UV_VERSION}/install.sh" _PROBE_TIMEOUT_S = 60.0 _STEP_TIMEOUT_S = 900.0 @@ -62,6 +72,8 @@ class Plan: free_kb: int | None fetch: str = "" """The machine's curl or wget, with which uv can be fetched when the machine has none.""" + index: str = "" + """The package index pip is configured with on the machine, or ``""`` for PyPI.""" @property def uv_from(self) -> str: @@ -90,16 +102,22 @@ def describe(self) -> str: else "a Python 3.12 made by uv, kept beside it" ) uv = ( - f" uv itself first, under {self.node_dir}/.uv (the machine has none), fetched with its own {self.fetch};" + f" uv {UV_VERSION} itself first, under {self.node_dir}/.uv (the machine has none), fetched with its own " + f"{self.fetch};" if self.uv_from == "fetched" else "" ) + index = ( + f"{self.index} (the index pip is configured with there)" + if self.index + else "PyPI (pip is configured with no other index there)" + ) space = f"; {self.free_kb // (1024 * 1024)} GB free there" if self.free_kb is not None else "" return ( f"Install the Raven node {self.install_name} on machine {self.target.label} into {where}:{uv} {python}, " - f"{' '.join(bundle.pinned((*protocol.REQUIREMENTS, protocol.SEARCH_REQUIREMENT)))} from the " - "machine's package index, and this Raven's code, all through uv. " - f"About {self.needed_kb // 1024} MB{space}. Nothing else on the machine changes." + f"{' '.join(bundle.pinned((*protocol.REQUIREMENTS, protocol.SEARCH_REQUIREMENT)))} from {index}, " + f"and this Raven's code, all through uv. About {self.needed_kb // 1024} MB{space}. " + "Nothing else on the machine changes." ) @@ -131,12 +149,30 @@ def _vars(node_dir: str, install_name: str) -> str: return f'dir={remote.shell_path(node_dir)}; id={shlex.quote(install_name)}; node="$dir/$id"; ' +#: The index pip would install from on the machine, read the way pip reads it +#: and said as ``INDEX=``: ``PIP_INDEX_URL`` first (the shell checks it), +#: else ``index-url`` in pip's config files, the user's over the system's +#: and ``[install]`` over ``[global]``. One line of Python for the machine's +#: own interpreter, with no single quote in it, since the whole line travels +#: inside single quotes on the ssh command. +_PIP_INDEX = ( + "import configparser, os; c = configparser.ConfigParser(interpolation=None); " + 'h = os.path.expanduser("~"); x = os.environ.get("XDG_CONFIG_HOME") or os.path.join(h, ".config"); ' + 'c.read(["/etc/pip.conf", "/etc/xdg/pip/pip.conf", os.path.join(h, ".pip", "pip.conf"), ' + 'os.path.join(x, "pip", "pip.conf")]); ' + 'v = next((c.get(s, "index-url") for s in ("install", "global") if c.has_option(s, "index-url")), ""); ' + 'print("INDEX=" + v.strip().splitlines()[0]) if v.strip() else None' +) + + def probe_script(node_dir: str, install_name: str, digest: str) -> str: - """One look at the machine: is this node there, which Python, uv and fetcher it has, how much room. + """One look at the machine: is this node there, which Python, uv, fetcher and package index it has, how much room. uv is looked for on the PATH, in the two places its installer puts it that a non-login PATH misses, and under ``node_dir/.uv``, where an - earlier install of this Raven may have put it. + earlier install of this Raven may have put it. The index is read the way + pip reads it (:data:`_PIP_INDEX`), by whichever Python the loop found, + since uv reads no pip configuration of its own accord. """ return _vars(node_dir, install_name) + ( f'if [ -x "$node/bin/python" ] && "$node/bin/python" -I -m raven.node --version 2>/dev/null ' @@ -147,6 +183,8 @@ def probe_script(node_dir: str, install_name: str, digest: str) -> str: 'for u in "$(command -v uv 2>/dev/null)" "$HOME/.local/bin/uv" "$HOME/.cargo/bin/uv" "$dir/.uv/uv"; do ' 'if [ -n "$u" ] && [ -x "$u" ]; then echo "UV=$u"; break; fi; done; ' 'for f in curl wget; do c=$(command -v "$f" 2>/dev/null) || continue; echo "FETCH=$c"; break; done; ' + f'if [ -n "$PIP_INDEX_URL" ]; then echo "INDEX=$PIP_INDEX_URL"; elif [ -n "$q" ]; then "$q" -c {shlex.quote(_PIP_INDEX)} ' + "2>/dev/null; fi; " 'd="$dir"; while [ ! -d "$d" ]; do d=$(dirname "$d"); done; ' 'echo "FREE_KB=$(df -Pk "$d" 2>/dev/null | awk \'NR==2 {print $4}\')"' ) @@ -170,7 +208,7 @@ def plan(machine_id: str, *, node_dir: str | None = None, run: Runner = _run) -> seen: dict[str, str] = {} for line in out.splitlines(): key, sep, value = line.strip().partition("=") - if sep and key in {"PRESENT", "PYTHON", "UV", "FETCH", "FREE_KB"}: + if sep and key in {"PRESENT", "PYTHON", "UV", "FETCH", "FREE_KB", "INDEX"}: seen[key] = value.strip() free = seen.get("FREE_KB", "") return Plan( @@ -182,6 +220,7 @@ def plan(machine_id: str, *, node_dir: str | None = None, run: Runner = _run) -> uv=seen.get("UV", ""), free_kb=int(free) if free.isdigit() else None, fetch=seen.get("FETCH", ""), + index=seen.get("INDEX", ""), ) @@ -208,28 +247,34 @@ def steps(p: Plan) -> list[tuple[str, str]]: uv = shlex.quote(p.uv) if p.uv_from == "machine" else '"$dir/.uv/uv"' head = _vars(p.node_dir, p.install_name) + f"uv={uv}; " env = 'UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python"' + # node_dir is made owner-only only when this makes it: one that was there + # already is the owner's, and may hold what they keep beside the nodes. + made = '[ -d "$dir" ] || { mkdir -p "$dir" && chmod 700 "$dir"; }; ' out: list[tuple[str, str]] = [] if p.uv_from == "fetched": - # uv's own installer, told where to put the one binary and to leave - # the shell's rc files alone; a wget machine reads it the same way. - fetch = f"wget -qO- {UV_INSTALLER}" if p.fetch.endswith("wget") else f"curl -LsSf {UV_INSTALLER}" + # uv's installer for one version, as an unmanaged install: the one + # binary goes where UV_UNMANAGED_INSTALL says, with no receipt written + # under the home directory and no shell rc file changed; a wget + # machine reads the installer the same way. + fetch = f"wget -qO- {UV_PINNED_INSTALLER}" if p.fetch.endswith("wget") else f"curl -LsSf {UV_PINNED_INSTALLER}" out.append( ( "uv", - head - + f'mkdir -p "$dir/.uv" && chmod 700 "$dir" && {fetch} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' - 'INSTALLER_NO_MODIFY_PATH=1 sh >/dev/null && test -x "$dir/.uv/uv"', + head + made + f'mkdir -p "$dir/.uv" && {fetch} | UV_UNMANAGED_INSTALL="$dir/.uv" sh >/dev/null ' + '&& test -x "$dir/.uv/uv"', ) ) python = shlex.quote(p.python) if p.how == "python" else "3.12" - venv = f'mkdir -p "$dir" && chmod 700 "$dir" && {env} "$uv" venv --quiet --python {python} "$node"' - pip = f'{env} "$uv" pip install --quiet --python "$node/bin/python"' + venv = made + f'{env} "$uv" venv --quiet --python {python} "$node"' + # The index pip is configured with on the machine, which uv would not + # read for itself; without one, uv's default, PyPI. + index = f" --default-index {shlex.quote(p.index)}" if p.index else "" + pip = f'{env} "$uv" pip install --quiet --python "$node/bin/python"{index}' requirements = " ".join(shlex.quote(r) for r in bundle.pinned(protocol.REQUIREMENTS)) deps = f"{pip} {requirements}" - # Optional, and never fails the install: the index uv is configured with - # on the machine first (its owner's choice, often a nearby mirror); then - # the public index, for a mirror that does not carry it; else the node - # searches without rg. + # Optional, and never fails the install: the machine's index first (its + # owner's choice, often a nearby mirror); then the public index too, for + # a mirror that does not carry it; else the node searches without rg. (rg,) = bundle.pinned((protocol.SEARCH_REQUIREMENT,)) search = ( f"{{ {pip} {shlex.quote(rg)} || {pip} --extra-index-url {protocol.PUBLIC_INDEX} {shlex.quote(rg)}; }} " @@ -327,6 +372,8 @@ def _confirm_digest(p: Plan, out: str) -> None: __all__ = [ "UV_INSTALLER", + "UV_PINNED_INSTALLER", + "UV_VERSION", "InstallError", "Installed", "Plan", diff --git a/raven/node/server.py b/raven/node/server.py index d52fd0754..3ade90ccf 100644 --- a/raven/node/server.py +++ b/raven/node/server.py @@ -52,7 +52,7 @@ import threading import time import uuid -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Iterator from functools import partial from pathlib import Path from typing import Any, BinaryIO @@ -329,9 +329,10 @@ async def _write(params: dict[str, Any]) -> dict[str, Any]: ``expect`` is the file as the host last read it -- its digest, or ``absent`` -- and a file that is not so any more is left alone, with :data:`raven.node.protocol.CHANGED`. The check and the write run in one - worker thread under the daemon's write lock (:func:`_apart`), so no other - write comes between them: what was checked is what is written over, and - the loop is not held while a whole file is hashed. + worker thread, alone among the whole-file calls (:func:`_apart`, + :class:`_FileGate`), so no other write comes between them and no read + lands inside them: what was checked is what is written over, and the + loop is not held while a whole file is hashed. """ from raven.agent.tools.filesystem import WriteFileTool @@ -357,7 +358,7 @@ async def _write(params: dict[str, Any]) -> dict[str, Any]: async def _edit(params: dict[str, Any]) -> dict[str, Any]: - """The host's ``edit_file`` on this machine: one read, the replacement, one write, with nothing between (under the write lock, :func:`_apart`).""" + """The host's ``edit_file`` on this machine: one read, the replacement, one write, with nothing between (alone among the whole-file calls, :func:`_apart`).""" from raven.agent.tools.filesystem import EditFileTool tool = _tool(EditFileTool, params) @@ -1137,18 +1138,67 @@ async def shutdown(self, params: dict[str, Any]) -> dict[str, Any]: #: agent's frames is not held for as long as a large file takes (measured #: 2026-10-12: 3.5 s for a 1 GiB file, with no ping answered meanwhile). _WHOLE_FILE = frozenset({protocol.READ, protocol.STAT, protocol.WRITE, protocol.EDIT, protocol.PULL}) -#: The ones that change a file, one at a time: what a write checked is what it writes over. +#: The ones that change a file: alone, so what a write checked is what it writes over and what a read digests is one file. _WRITERS = frozenset({protocol.WRITE, protocol.EDIT}) -_WRITE_LOCK = threading.Lock() + + +class _FileGate: + """The whole-file calls, readers together and a writer alone, so what a read digests is the file it read. + + On the daemon's one loop nothing came between a read and a write; in a + worker thread a page read and the digest taken beside it could land + inside a write of the same file and leave with the old page and the new + digest (measured 2026-10-13: 2 to 4 of some 50 reads during one write of + a 400,000-line file), which a later ``fs/write`` carrying that digest as + ``expect`` would take as leave to replace content never read. A waiting + writer goes before new readers, so a stream of reads cannot keep it out. + """ + + def __init__(self) -> None: + self._changed = threading.Condition() + self._readers = 0 + self._writing = False + self._writers_waiting = 0 + + @contextlib.contextmanager + def reading(self) -> Iterator[None]: + with self._changed: + while self._writing or self._writers_waiting: + self._changed.wait() + self._readers += 1 + try: + yield + finally: + with self._changed: + self._readers -= 1 + self._changed.notify_all() + + @contextlib.contextmanager + def writing(self) -> Iterator[None]: + with self._changed: + self._writers_waiting += 1 + try: + while self._writing or self._readers: + self._changed.wait() + finally: + self._writers_waiting -= 1 + self._writing = True + try: + yield + finally: + with self._changed: + self._writing = False + self._changed.notify_all() + + +_FILES = _FileGate() def _apart( - handler: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]], params: dict[str, Any], *, serial: bool + handler: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]], params: dict[str, Any], *, writes: bool ) -> dict[str, Any]: - """``handler`` run to completion in this worker thread, on a loop of its own; a writer one at a time.""" - if not serial: - return asyncio.run(handler(params)) - with _WRITE_LOCK: + """``handler`` run to completion in this worker thread, on a loop of its own, through the file gate.""" + with _FILES.writing() if writes else _FILES.reading(): return asyncio.run(handler(params)) @@ -1210,7 +1260,7 @@ async def handle(frame: Any, node: Node | None = None) -> dict[str, Any] | None: if not isinstance(params, dict): raise _RefusedError(_INVALID_PARAMS, "params must be an object") if method in _WHOLE_FILE: - result = await asyncio.to_thread(_apart, handler, params, serial=method in _WRITERS) + result = await asyncio.to_thread(_apart, handler, params, writes=method in _WRITERS) else: result = await handler(params) return {"jsonrpc": "2.0", "id": request_id, "result": result} diff --git a/tests/test_filesystem_tools.py b/tests/test_filesystem_tools.py index 3e641339d..4736cb5c8 100644 --- a/tests/test_filesystem_tools.py +++ b/tests/test_filesystem_tools.py @@ -231,3 +231,64 @@ def test_a_file_too_large_to_preview_is_not_read_at_all(tmp_path: Path): view = WriteFileTool(workspace=tmp_path).approval_evidence({"path": "big.log", "content": "small"}) assert view == {"path": str(target.resolve()), "created": False} + + +# --- a page of a text file, read a chunk at a time ------------------------------------ + + +_EVERY_BREAK = ( + b"one\n" + b"two\r\n" + b"three\r" + b"four\x0b" + b"five\x0c" + b"six\x1c" + b"seven\x1d" + b"eight\x1e" + b"nine\xc2\x85" + b"ten\xe2\x80\xa8" + b"eleven\xe2\x80\xa9" + b"\n" + b"twelve \xc3\xa9 last, with no break after it" +) + + +@pytest.mark.parametrize("chunk", [1, 2, 3, 4, 7, 64]) +def test_a_page_is_the_lines_splitlines_gives_whatever_the_chunk_edge(monkeypatch, chunk: int): + """Every break ``str.splitlines`` knows, each landing on a chunk edge for some chunk size.""" + from io import BytesIO + + from raven.agent.tools import filesystem + + monkeypatch.setattr(filesystem, "_PAGE_CHUNK", chunk) + want = _EVERY_BREAK.decode("utf-8").splitlines() + assert len(want) == 13 and want[11] == "", "a blank line between two breaks counts; a trailing break does not" + + for start, limit in ((0, 2000), (0, 1), (1, 3), (8, 2), (12, 1), (13, 5)): + page, total = filesystem._page_of_lines(BytesIO(_EVERY_BREAK), start, limit) + assert total == 13 and [line.decode("utf-8") for line in page] == want[start : start + limit] + assert filesystem._page_of_lines(BytesIO(b""), 0, 5) == ([], 0) + assert filesystem._page_of_lines(BytesIO(b"\n"), 0, 5) == ([b""], 1) + + +@pytest.mark.asyncio +async def test_read_file_pages_a_file_without_holding_all_of_it(tmp_path: Path, monkeypatch): + from raven.agent.tools import filesystem + + target = tmp_path / "breaks.txt" + target.write_bytes(_EVERY_BREAK) + monkeypatch.setattr(filesystem, "_PAGE_CHUNK", 5) + read = ReadFileTool(workspace=tmp_path) + + first = await read.execute(path=str(target), limit=2) + assert first == "1| one\n2| two\n\n(Showing lines 1-2 of 13. Use offset=3 to continue.)" + last = await read.execute(path=str(target), offset=12) + assert last == "12| \n13| twelve \u00e9 last, with no break after it\n\n(End of file \u2014 13 lines total)" + assert await read.execute(path=str(target), offset=14) == "Error: offset 14 is beyond end of file (13 lines)" + + # A byte that is not UTF-8 refuses the page it is on, as the whole-file decode refused the file. + target.write_bytes(b"fine\n\xff broken\n") + assert ( + await read.execute(path=str(target), limit=1) + ) == "1| fine\n\n(Showing lines 1-1 of 2. Use offset=2 to continue.)" + assert str(await read.execute(path=str(target), offset=2)).startswith("Error reading file: 'utf-8' codec") diff --git a/tests/test_node_install.py b/tests/test_node_install.py index 806a3bf85..ce593167c 100644 --- a/tests/test_node_install.py +++ b/tests/test_node_install.py @@ -16,6 +16,7 @@ import dataclasses import json import shutil +import stat import subprocess import sys from pathlib import Path @@ -232,7 +233,8 @@ def test_a_machine_without_uv_gets_one_first_and_the_room_for_it_is_counted(): assert p.uv_from == "fetched" and p.how == "python" and p.needed_kb == 85 * 1024 said = p.describe() assert ( - "uv itself first, under ~/.raven-node/.uv (the machine has none), fetched with its own /usr/bin/curl;" in said + f"uv {install.UV_VERSION} itself first, under ~/.raven-node/.uv (the machine has none), fetched with its own " + "/usr/bin/curl;" in said ) assert "a virtualenv of the machine's own /usr/bin/python3.12" in said and "all through uv" in said assert _plan(uv="", fetch="").how == "" and _plan(uv="", fetch="").uv_from == "" @@ -279,6 +281,8 @@ def test_an_install_that_cannot_go_ahead_says_why(fields, said): _UV_ENV = 'UV_CACHE_DIR="$dir/.uv-cache" UV_PYTHON_INSTALL_DIR="$dir/.uv-python"' +# node_dir made owner-only only when the step makes it; one that is there is left as it is. +_MADE = '[ -d "$dir" ] || { mkdir -p "$dir" && chmod 700 "$dir"; }; ' def test_the_steps_build_a_virtualenv_with_uv_then_fill_it_with_uv(): @@ -289,11 +293,11 @@ def test_the_steps_build_a_virtualenv_with_uv_then_fill_it_with_uv(): assert all(command.startswith(head) for _, command in steps) commands = dict(steps) assert commands["virtualenv"] == ( - head - + f'mkdir -p "$dir" && chmod 700 "$dir" && {_UV_ENV} "$uv" venv --quiet --python \'/opt/py 3.12/bin/python3\' "$node"' + head + _MADE + f'{_UV_ENV} "$uv" venv --quiet --python \'/opt/py 3.12/bin/python3\' "$node"' ) pip = f'{_UV_ENV} "$uv" pip install --quiet --python "$node/bin/python"' assert commands["packages"] == head + f"{pip} " + " ".join(bundle.pinned(protocol.REQUIREMENTS)) + assert "--default-index" not in " ".join(commands.values()), "no index configured for pip: uv's own default" assert "python -m pip" not in " ".join(commands.values()) and " -m venv " not in " ".join(commands.values()) (rg,) = bundle.pinned((protocol.SEARCH_REQUIREMENT,)) assert commands["search"].count(rg) == 2 and commands["search"].count(pip) == 2 @@ -309,25 +313,50 @@ def test_uv_makes_its_python_inside_the_node_dir(): commands = dict(install.steps(_plan(python="", uv="/root/.local/bin/uv"))) assert commands["virtualenv"].endswith( - f'uv=/root/.local/bin/uv; mkdir -p "$dir" && chmod 700 "$dir" && {_UV_ENV} "$uv" venv --quiet --python 3.12 "$node"' + f'uv=/root/.local/bin/uv; {_MADE}{_UV_ENV} "$uv" venv --quiet --python 3.12 "$node"' ) +def test_the_index_pip_is_configured_with_is_handed_to_uv_and_named_in_the_plan(): + p = _plan(index="https://mirror.example/simple") + + commands = dict(install.steps(p)) + + pip = ( + f'{_UV_ENV} "$uv" pip install --quiet --python "$node/bin/python" --default-index https://mirror.example/simple' + ) + assert commands["packages"] == commands["packages"].split(pip)[0] + pip + " " + " ".join( + bundle.pinned(protocol.REQUIREMENTS) + ) + # The search package tries the machine's index first, then that index with the public one beside it. + assert commands["search"].count(pip) == 2 and f"--extra-index-url {protocol.PUBLIC_INDEX}" in commands["search"] + assert "from https://mirror.example/simple (the index pip is configured with there), and this Raven's code" in ( + p.describe() + ) + assert "from PyPI (pip is configured with no other index there), and this Raven's code" in _plan().describe() + + @pytest.mark.parametrize( ("fetch", "fetched_with"), - [("/usr/bin/curl", f"curl -LsSf {install.UV_INSTALLER}"), ("/usr/bin/wget", f"wget -qO- {install.UV_INSTALLER}")], + [ + ("/usr/bin/curl", f"curl -LsSf {install.UV_PINNED_INSTALLER}"), + ("/usr/bin/wget", f"wget -qO- {install.UV_PINNED_INSTALLER}"), + ], ) -def test_a_machine_without_uv_fetches_it_under_the_node_dir_first(fetch, fetched_with): +def test_a_machine_without_uv_fetches_one_version_of_it_under_the_node_dir_and_nowhere_else(fetch, fetched_with): steps = install.steps(_plan(uv="", fetch=fetch)) assert [name for name, _ in steps] == ["uv", "virtualenv", "packages", "search", "code", "path", "check"] head = "dir=~/.raven-node; id=" + bundle.install_name() + '; node="$dir/$id"; uv="$dir/.uv/uv"; ' assert all(command.startswith(head) for _, command in steps) + # One version, as the packages are pinned; an unmanaged install, which + # writes no receipt under the home directory and changes no PATH. + assert install.UV_VERSION in install.UV_PINNED_INSTALLER and install.UV_PINNED_INSTALLER != install.UV_INSTALLER assert dict(steps)["uv"] == ( - head - + f'mkdir -p "$dir/.uv" && chmod 700 "$dir" && {fetched_with} | UV_INSTALL_DIR="$dir/.uv" UV_NO_MODIFY_PATH=1 ' - 'INSTALLER_NO_MODIFY_PATH=1 sh >/dev/null && test -x "$dir/.uv/uv"' + head + _MADE + f'mkdir -p "$dir/.uv" && {fetched_with} | UV_UNMANAGED_INSTALL="$dir/.uv" sh >/dev/null ' + '&& test -x "$dir/.uv/uv"' ) + assert "UV_INSTALL_DIR" not in dict(steps)["uv"] and "MODIFY_PATH" not in dict(steps)["uv"] assert dict(steps)["virtualenv"].endswith('"$uv" venv --quiet --python /usr/bin/python3.12 "$node"') @@ -481,12 +510,12 @@ def test_a_prune_that_failed_says_where(): # --- the scripts, under a real shell ---------------------------------------------- -def _sh(command: str, home: Path, path: str) -> str: +def _sh(command: str, home: Path, path: str, **env: str) -> str: done = subprocess.run( ["/bin/sh", "-c", command], capture_output=True, text=True, - env={"HOME": str(home), "PATH": path}, + env={"HOME": str(home), "PATH": path, **env}, cwd=home, timeout=60, ) @@ -543,6 +572,46 @@ def test_the_probe_finds_the_node_a_new_enough_python_uv_and_the_room(shell): assert seen["FREE_KB"].isdigit() +@pytest.mark.slow +def test_the_probe_reads_the_index_pip_is_configured_with_as_pip_reads_it(shell): + home, tools = shell + _script(tools / "python3", f'exec {sys.executable} "$@"\n') + conf = home / ".config" / "pip" / "pip.conf" + conf.parent.mkdir(parents=True) + conf.write_text("[global]\nindex-url = https://mirror.example/simple\n", encoding="utf-8") + probe = install.probe_script("~/.raven-node", "0.1-x", "d") + + assert _lines(_sh(probe, home, str(tools)))["INDEX"] == "https://mirror.example/simple" + + # [install] over [global], as for pip's own install command; the environment over every file. + conf.write_text( + "[global]\nindex-url = https://a/simple\n[install]\nindex-url = https://b/simple\n", encoding="utf-8" + ) + assert _lines(_sh(probe, home, str(tools)))["INDEX"] == "https://b/simple" + assert _lines(_sh(probe, home, str(tools), PIP_INDEX_URL="https://c/simple"))["INDEX"] == "https://c/simple" + conf.unlink() + assert "INDEX" not in _lines(_sh(probe, home, str(tools))), "nothing configured: uv's default" + + +@pytest.mark.slow +def test_a_node_dir_that_was_there_keeps_its_mode_and_one_the_install_makes_is_owner_only(shell, tmp_path): + home, tools = shell + for name in ("mkdir", "chmod"): + (tools / name).symlink_to(shutil.which(name) or name) + uv = _script(tools / "uv", "exit 0\n") + theirs = tmp_path / "theirs" + theirs.mkdir(mode=0o755) + (theirs / "notes.txt").write_text("the owner's own\n", encoding="utf-8") + ours = tmp_path / "ours" + + for node_dir in (theirs, ours): + step = dict(install.steps(_plan(node_dir=str(node_dir), uv=str(uv), python=sys.executable)))["virtualenv"] + _sh(step, home, str(tools)) + + assert stat.S_IMODE(theirs.stat().st_mode) == 0o755 and (theirs / "notes.txt").is_file() + assert stat.S_IMODE(ours.stat().st_mode) == 0o700 + + @pytest.mark.slow def test_the_probe_finds_the_uv_an_earlier_install_put_under_the_node_dir(shell): home, tools = shell diff --git a/tests/test_node_server.py b/tests/test_node_server.py index b5e8b9dc5..db87d3408 100644 --- a/tests/test_node_server.py +++ b/tests/test_node_server.py @@ -17,6 +17,7 @@ import signal import subprocess import sys +import threading import time from pathlib import Path from typing import Any @@ -1288,6 +1289,89 @@ def slowly(path: Path) -> str: assert (await read)["sha256"] == hashlib.sha256(b"hay\n").hexdigest() +async def test_a_read_never_lands_inside_a_write_of_the_same_file(tree, monkeypatch): + """The page and the digest a read answers with are one state of the file, whatever a write does meanwhile.""" + digest = server._sha256 + + def slowly(path: Path) -> str: + time.sleep(0.3) + return digest(path) + + monkeypatch.setattr(server, "_sha256", slowly) + path = str(tree / "b.md") + read = asyncio.create_task(_result(protocol.READ, {"path": path, "roots": [str(tree)]})) + await asyncio.sleep(0.05) + write = asyncio.create_task(_result(protocol.WRITE, {"path": path, "content": "straw\n", "roots": [str(tree)]})) + + page, written = await asyncio.gather(read, write) + + assert page["text"].startswith("1| hay") and page["sha256"] == hashlib.sha256(b"hay\n").hexdigest() + assert written["ok"] and (tree / "b.md").read_text(encoding="utf-8") == "straw\n", "the write landed after" + + +def test_a_writer_that_waits_goes_before_new_readers(): + gate = server._FileGate() + first_in = threading.Event() + writer_waiting = threading.Event() + writer_done = threading.Event() + order: list[str] = [] + + def first_reader() -> None: + with gate.reading(): + first_in.set() + writer_waiting.wait(5) + time.sleep(0.1) + order.append("first reader out") + + def writer() -> None: + writer_waiting.set() + with gate.writing(): + order.append("writer") + writer_done.set() + + def second_reader() -> None: + with gate.reading(): + order.append("second reader") + + threads = [ + threading.Thread(target=first_reader), + threading.Thread(target=writer), + threading.Thread(target=second_reader), + ] + threads[0].start() + first_in.wait(5) + threads[1].start() + writer_waiting.wait(5) + time.sleep(0.02) + threads[2].start() + for thread in threads: + thread.join(5) + + assert order == ["first reader out", "writer", "second reader"] + + +@pytest.mark.slow +async def test_a_page_read_of_a_large_file_keeps_the_heartbeat_answered(tmp_path): + """The page is read a chunk at a time, so no one step holds the interpreter for the whole file.""" + big = tmp_path / "big.log" + with big.open("wb") as handle: + line = b"2026-10-13 12:00:00 one line of a log that goes on for a while, as log lines do\n" + for _ in range(120 * 1024 * 1024 // len(line)): + handle.write(line) + read = asyncio.create_task(_result(protocol.READ, {"path": str(big), "limit": 3, "roots": [str(tmp_path)]})) + gaps: list[float] = [] + last = time.monotonic() + while not read.done(): + assert await _result(protocol.PING, {"t": 1}) == {"t": 1} + now = time.monotonic() + gaps.append(now - last) + last = now + await asyncio.sleep(0.005) + + assert max(gaps) < 0.25, f"a ping waited {max(gaps):.2f}s on the read" + assert (await read)["text"].startswith("1| 2026-10-13 12:00:00 one line") + + @pytest.mark.parametrize("argv", [["--serve"], ["--attach"]]) def test_serve_and_attach_need_the_node_directory(argv, capsys): with pytest.raises(SystemExit): From 7bebb452a79bce2830f92cac98631d54bec92b3e Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:22:48 +0800 Subject: [PATCH 21/44] fix(node): the install plan shows the package index without the sign-in pip may keep inside it pip takes an index with a sign-in in the URL (`https://user:password@host/simple`), and the plan that names the index is printed before the confirmation, and under `--yes`, so the password reached the terminal and any transcript of it. The plan now shows the host and the path only, saying so when a sign-in was left out; uv still gets the full URL, and a failed step's sentence, which quotes uv's own last line, is shown the same way. Co-authored-by: Claude (claude-fable-5-1) --- raven/node/install.py | 33 ++++++++++++++++++++++++++++----- tests/test_node_install.py | 19 +++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/raven/node/install.py b/raven/node/install.py index 1612da8fa..38b7ee268 100644 --- a/raven/node/install.py +++ b/raven/node/install.py @@ -31,6 +31,7 @@ import shlex from collections.abc import Callable from dataclasses import dataclass +from urllib.parse import urlsplit, urlunsplit from raven.acp_client import remote from raven.node import bundle, protocol @@ -107,11 +108,17 @@ def describe(self) -> str: if self.uv_from == "fetched" else "" ) - index = ( - f"{self.index} (the index pip is configured with there)" - if self.index - else "PyPI (pip is configured with no other index there)" - ) + if self.index: + # The host and the path only: pip takes a sign-in inside the URL + # (``https://user:password@host/simple``), and this line is + # printed before the confirmation and under ``--yes``. + shown = _shown_index(self.index) + how = "the index pip is configured with there" + ( + ", shown without its sign-in" if shown != self.index else "" + ) + index = f"{shown} ({how})" + else: + index = "PyPI (pip is configured with no other index there)" space = f"; {self.free_kb // (1024 * 1024)} GB free there" if self.free_kb is not None else "" return ( f"Install the Raven node {self.install_name} on machine {self.target.label} into {where}:{uv} {python}, " @@ -348,12 +355,28 @@ def install(p: Plan, *, run: Runner = _run, push: Pusher = _push) -> Installed: return Installed(p.node_dir, p.install_name, int(size) if size.isdigit() else None, rg=rg) +def _shown_index(url: str) -> str: + """``url`` as a sentence shows it: the host and the path, with any sign-in in it left out.""" + parts = urlsplit(url) + if not (parts.username or parts.password): + return url + host = parts.hostname or "" + if ":" in host: + host = f"[{host}]" + if parts.port: + host = f"{host}:{parts.port}" + return urlunsplit(parts._replace(netloc=host)) + + def _failed(p: Plan, step: str, rc: int, out: str) -> str: """One sentence for a failed step: ssh's failure explained, else the machine's own last line.""" if (said := remote.explain(p.target, rc, out)) is not None and rc != remote.NOT_FOUND_RC: return f"installing the Raven node: {said}" lines = [line.strip() for line in out.splitlines() if line.strip()] last = lines[-1][:300] if lines else f"exit {rc}" + if p.index: + # uv names the index in its own refusals; a sign-in in it stays out of this sentence too. + last = last.replace(p.index, _shown_index(p.index)) return f"installing the Raven node on machine {p.target.label} failed at the {step} step: {last}" diff --git a/tests/test_node_install.py b/tests/test_node_install.py index ce593167c..3d8e93952 100644 --- a/tests/test_node_install.py +++ b/tests/test_node_install.py @@ -336,6 +336,25 @@ def test_the_index_pip_is_configured_with_is_handed_to_uv_and_named_in_the_plan( assert "from PyPI (pip is configured with no other index there), and this Raven's code" in _plan().describe() +def test_a_sign_in_inside_the_index_url_reaches_uv_and_never_a_sentence(): + """pip takes `https://user:password@host/simple`; the plan is printed before the confirmation, and under --yes.""" + p = _plan(index="https://user:s3cret-pw@mirror.example:8443/simple") + + said = p.describe() + assert ( + "from https://mirror.example:8443/simple (the index pip is configured with there, shown without its sign-in)" + in said + ) + assert "s3cret-pw" not in said and "user:" not in said + assert "--default-index https://user:s3cret-pw@mirror.example:8443/simple" in dict(install.steps(p))["packages"] + failed = install._failed( + p, "packages", 2, "error: Failed to fetch: `https://user:s3cret-pw@mirror.example:8443/simple/loguru/`" + ) + assert "s3cret-pw" not in failed and "https://mirror.example:8443/simple/loguru/" in failed + assert install._shown_index("https://mirror.example/simple") == "https://mirror.example/simple" + assert install._shown_index("http://u:p@[::1]:8080/simple") == "http://[::1]:8080/simple" + + @pytest.mark.parametrize( ("fetch", "fetched_with"), [ From a7ae4b9e2e5ff84f2084cb4e873c3b5da44624f2 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:05:33 +0800 Subject: [PATCH 22/44] fix(node): blank lines are frames too, and a command gone before its stdin still answers The daemon dropped every blank line an agent wrote, and the endpoint had no way to carry one (its EOF is the empty read), so a plain-text CLI answer came back with its paragraphs run together. A frame is now any line, blank ones included, forwarded, recorded and replayed alike, and the endpoint hands each line back with its line break as a pipe does, the empty read meaning only that the agent is gone; the ACP reader over it skips blank lines as it always did. Protocol 5, since a frame may now be empty. A quick command on a machine could leave before its prompt and the end of its stdin reached it, and that was read as a failure to deliver instead of the run it was: its own output and exit code are the report, as for a local run. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/subagent/backends/cli_agent.py | 21 ++++-- raven/node/client.py | 11 +-- raven/node/protocol.py | 7 +- raven/node/server.py | 10 +-- tests/test_node_client.py | 17 ++++- tests/test_node_server.py | 20 ++++++ tests/test_subagent_cli_remote.py | 78 ++++++++++++++++++++-- 7 files changed, 138 insertions(+), 26 deletions(-) diff --git a/raven/agent/subagent/backends/cli_agent.py b/raven/agent/subagent/backends/cli_agent.py index 7de3f4263..f9096118a 100644 --- a/raven/agent/subagent/backends/cli_agent.py +++ b/raven/agent/subagent/backends/cli_agent.py @@ -585,9 +585,10 @@ async def _exec_there( it: this computer's login shell, its ``RAVEN_HOME`` and the parent's model binding describe a process of raven's own here, and say nothing on that machine. A cancel or a timeout ends the whole process tree - there (``agent/cancel``). The daemon forwards whole lines, so a blank - line of a plain-text answer does not come back; every format this - module parses is one frame per line. + there (``agent/cancel``). The node forwards every line the command + writes, a blank one included, so a plain-text answer comes back as it + was printed; a quick command may leave before its stdin reaches it, + and then its own output and exit code are the report, as here. """ from raven.ops import remote_path @@ -619,19 +620,25 @@ async def _exec_there( async def read_out() -> None: while raw := await endpoint.readline(): - line = raw.decode("utf-8", "replace") + line = raw.decode("utf-8", "replace").rstrip("\n") out_lines.append(line) await publish_out_line(line) async def read_err() -> None: while raw := await endpoint.readerr(): - line = raw.decode("utf-8", "replace") + line = raw.decode("utf-8", "replace").rstrip("\n") err_lines.append(line) activity.note_console(line + "\n") async def drive() -> int | None: - # The prompt after the launch, and the end of stdin either way. - await endpoint.send_input("" if used_placeholder else task, eof=True) + # The prompt after the launch, and the end of stdin either way. A + # quick command may have left already, having read none of it: its + # own output and exit code are the report, as they are here. + try: + await endpoint.send_input("" if used_placeholder else task, eof=True) + except BrokenPipeError: + if endpoint.returncode is None: + raise await asyncio.gather(read_out(), read_err()) return await endpoint.wait() diff --git a/raven/node/client.py b/raven/node/client.py index 7ed5c7ee9..9041cf90e 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -149,7 +149,9 @@ class ChannelEndpoint: The client reads and writes it as it would a child process's pipes; what it gets are the ``agent/frame`` and ``agent/stderr`` notifications of one - channel, and what it writes goes back the same way -- a frame per line + channel, each read back as a pipe gives a line, with its line break (a + blank line is a frame too: a CLI agent's answer is its lines as printed), + and what it writes goes back the same way -- a frame per line (``write``), or a CLI agent's whole prompt and the end of its stdin (``send_input``, ``agent/stdin``). ``end`` is ``agent/cancel``: the node ends the agent's whole process tree, and the @@ -253,6 +255,7 @@ def _hold(self, method: str, params: dict[str, Any]) -> None: self._pending.append((method, params)) async def readline(self) -> bytes: + """The next line the agent wrote, with its line break, as a pipe gives it; ``b""`` once the agent is gone.""" item = await self._frames.get() if item is None: return b"" @@ -295,13 +298,13 @@ def release(self) -> None: # --- fed by the pool, off the link's notifications --------------------------- def feed_frame(self, line: Any, n: Any = None) -> None: - if isinstance(line, str) and line and not self._done.is_set(): + if isinstance(line, str) and not self._done.is_set(): self.received = n if isinstance(n, int) and not isinstance(n, bool) else self.received + 1 - self._frames.put_nowait((self.received, line.encode("utf-8"))) + self._frames.put_nowait((self.received, line.encode("utf-8") + b"\n")) def feed_stderr(self, text: Any) -> None: if isinstance(text, str) and not self._done.is_set(): - self._stderr.put_nowait(text.encode("utf-8")) + self._stderr.put_nowait(text.encode("utf-8") + b"\n") def exited(self, rc: Any) -> None: if self._done.is_set(): diff --git a/raven/node/protocol.py b/raven/node/protocol.py index 1e7ddc8bf..848ec572b 100644 --- a/raven/node/protocol.py +++ b/raven/node/protocol.py @@ -11,8 +11,9 @@ One connection per machine carries everything the host asks of it. Agents the daemon starts for the host each get a *channel* of that connection, on which the agent's ACP frames travel in both directions as ``agent/frame`` -notifications (a CLI agent's output lines travel the same way, and its prompt -reaches its stdin as ``agent/stdin``) -- and a *task* id of the host's +notifications (a CLI agent's output lines travel the same way, every one of +them, a blank line included, since they are its answer; its prompt reaches +its stdin as ``agent/stdin``) -- and a *task* id of the host's choosing, which outlives the connection: an agent keeps running when the ssh goes, its frames are kept on the machine, and a new connection picks them up from where the host stopped reading (``agent/attach``). Requests are answered @@ -28,7 +29,7 @@ #: Bumped when a method or a field changes meaning. A daemon and a host that #: disagree refuse each other at ``node/hello`` instead of misreading a reply. -PROTOCOL = 4 +PROTOCOL = 5 HELLO = "node/hello" PING = "node/ping" diff --git a/raven/node/server.py b/raven/node/server.py index 3ade90ccf..13f3f5d8a 100644 --- a/raven/node/server.py +++ b/raven/node/server.py @@ -740,8 +740,9 @@ async def _forward(self, stream: asyncio.StreamReader | None, *, frames: bool) - if not raw: return text = raw.decode("utf-8", "replace").rstrip("\r\n") - if not text: - continue + # Every line, a blank one included: a CLI agent's answer is its + # lines as it printed them, and a line numbered here is one an + # attach replays, so the two counts agree. async with self._send_lock: handle = self._frames_file if frames else self._stderr_file handle.write(text.encode("utf-8") + b"\n") @@ -852,9 +853,8 @@ async def _replay(record: Path, node: Node, channel: int, after: int) -> int: if n <= after: continue text = raw.decode("utf-8", "replace").rstrip("\r\n") - if text: - await node.notify(protocol.AGENT_FRAME, {"channel": channel, "line": text, "n": n}) - sent += 1 + await node.notify(protocol.AGENT_FRAME, {"channel": channel, "line": text, "n": n}) + sent += 1 return sent diff --git a/tests/test_node_client.py b/tests/test_node_client.py index be346f900..bcc9392f4 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -994,9 +994,9 @@ async def test_a_channel_endpoint_carries_frames_both_ways_and_ends_with_the_age endpoint.feed_frame("too late") assert link.said == [(protocol.AGENT_FRAME, {"channel": 3, "line": '{"jsonrpc":"2.0","id":1,"method":"x"}'})] - assert await endpoint.readline() == b'{"jsonrpc":"2.0","id":1,"result":{}}' + assert await endpoint.readline() == b'{"jsonrpc":"2.0","id":1,"result":{}}\n' assert await endpoint.readline() == b"" - assert await endpoint.readerr() == b"agent: hello" and await endpoint.readerr() == b"" + assert await endpoint.readerr() == b"agent: hello\n" and await endpoint.readerr() == b"" assert endpoint.returncode == 3 and await endpoint.wait() == 3 and endpoint.ended() is None with pytest.raises(BrokenPipeError): await endpoint.write(b"{}\n") @@ -1004,6 +1004,17 @@ async def test_a_channel_endpoint_carries_frames_both_ways_and_ends_with_the_age assert all(method != protocol.AGENT_CANCEL for method, *_ in link.asked), "nothing to cancel once it has exited" +async def test_a_blank_line_is_a_frame_the_endpoint_hands_on_as_a_pipe_would(): + endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="cli@box") # type: ignore[arg-type] + endpoint.feed_frame("a", 1) + endpoint.feed_frame("", 2) + endpoint.feed_frame("b", 3) + endpoint.exited(0) + + assert [await endpoint.readline() for _ in range(4)] == [b"a\n", b"\n", b"b\n", b""] + assert endpoint.received == 3, "the blank line counts, so an attach picks up where this one stopped" + + async def test_ending_a_running_channel_is_a_cancel_on_the_link(tmp_path): link = _Client() endpoint = ChannelEndpoint(link, 2, target=_box(), agent="echo@box") # type: ignore[arg-type] @@ -1069,7 +1080,7 @@ async def test_a_cli_agent_s_input_travels_on_the_link_and_its_stdin_ends_when_t lines = [] while raw := await endpoint.readline(): - lines.append(raw.decode("utf-8")) + lines.append(raw.decode("utf-8").rstrip("\n")) assert lines == ["first line", "second line"], "what went in came back, line by line" assert await endpoint.wait() == 0, "its stdin closed, cat left on its own" assert len(box.launches()) == 1, "on the node's ssh, like a frame" diff --git a/tests/test_node_server.py b/tests/test_node_server.py index db87d3408..b2cdd9cb7 100644 --- a/tests/test_node_server.py +++ b/tests/test_node_server.py @@ -987,6 +987,26 @@ def _record(daemon: server.Daemon, task: str) -> dict[str, Any]: return json.loads((daemon.record_dir(task) / "task.json").read_text(encoding="utf-8")) +@pytest.mark.slow +async def test_a_blank_line_an_agent_prints_is_forwarded_recorded_and_replayed(tmp_path): + """A CLI agent's answer is its lines as printed: a blank line is a frame, numbered, kept and replayed like any.""" + daemon = server.Daemon(tmp_path / "node") + first = _Conn(daemon) + command = f"{sys.executable} -c \"print('a'); print(); print('b')\"" + await first.ask( + protocol.AGENT_LAUNCH, {"channel": 1, "task": "t-blank", "command": command, "cwd": str(tmp_path / "w")} + ) + await first.until(lambda: first.notified(protocol.AGENT_EXIT)) + + assert [(f["line"], f["n"]) for f in first.notified(protocol.AGENT_FRAME)] == [("a", 1), ("", 2), ("b", 3)] + assert (daemon.record_dir("t-blank") / "frames.jsonl").read_bytes() == b"a\n\nb\n" + + second = _Conn(daemon) + await second.ask(protocol.AGENT_ATTACH, {"channel": 5, "task": "t-blank", "after": 1}, request_id=2) + await second.until(lambda: second.notified(protocol.AGENT_EXIT)) + assert [(f["line"], f["n"]) for f in second.notified(protocol.AGENT_FRAME)] == [("", 2), ("b", 3)] + + @pytest.mark.slow async def test_a_task_is_recorded_as_it_runs_and_goes_on_when_its_connection_ends(tmp_path, agent): daemon = server.Daemon(tmp_path / "node") diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py index 57d02d94b..291c9d369 100644 --- a/tests/test_subagent_cli_remote.py +++ b/tests/test_subagent_cli_remote.py @@ -186,8 +186,7 @@ async def test_the_prompt_reaches_a_cli_agent_there_the_three_ways_it_does_here( as_argument, _ = _split(await _run(cli_config("printf %s {prompt}"), prompt, tmp_path, task_id="t-arg")) from_file, _ = _split(await _run(cli_config("cat {prompt_file}"), prompt, tmp_path, task_id="t-file")) - # The node forwards whole lines, so a blank line of a plain answer does not come back. - assert on_stdin == as_argument == from_file == "first line\nthird line, after a blank one" + assert on_stdin == as_argument == from_file == prompt, "every line as printed, the blank one included" written = box.home / "raven-work" / "t-file" / ".raven" / "prompt-t-file.txt" assert written.read_text(encoding="utf-8") == prompt, "the prompt file is written into the run's directory there" assert not list(tmp_path.glob("raven_subagent_*")), "no prompt file is made on this computer" @@ -206,6 +205,78 @@ async def test_the_entry_s_env_reaches_the_agent_on_the_link_and_this_computer_s assert "sk-raven-test-not-real" not in " ".join(argv), "a value in env is never an ssh argument" +async def test_a_command_that_leaves_before_its_stdin_reaches_it_still_answers_with_every_line(box, tmp_path): + """A quick command is gone before the prompt is sent: its own output and exit code are the report.""" + head, _ = _split(await _run(cli_config("sh -c 'printf \"a\\n\\nb\\n\"'"), "hi", tmp_path)) + + assert head == "a\n\nb", "every line as printed, the blank one included" + + +class _Link: + """A node link that records what is said on it and answers every request with nothing.""" + + def __init__(self) -> None: + self.alive = True + self.said: list[tuple[str, dict[str, Any]]] = [] + + async def notify(self, method: str, params: dict[str, Any]) -> None: + self.said.append((method, params)) + + async def request(self, method: str, params: dict[str, Any], *, timeout: float) -> dict[str, Any]: + return {} + + +class _Placed: + """A node pool whose one agent is ``endpoint``, already started: the dispatch's place on the machine.""" + + def __init__(self, endpoint: Any) -> None: + self.endpoint = endpoint + + async def session_dir(self, machine_id: str, path: str) -> tuple[str, float]: + return f"/home/worker/raven-work/{path.rsplit('/', 1)[-1]}", 0.0 + + async def write_file(self, machine_id: str, path: str, content: str, *, roots: Any) -> None: + return None + + async def open_channel(self, machine_id: str, **kw: Any) -> Any: + return self.endpoint + + +async def test_an_agent_gone_before_its_stdin_is_sent_is_read_as_it_left(box, tmp_path, monkeypatch): + """The prompt reaches an agent that already left: nothing is sent, and what it printed and how it exited are the report.""" + from raven.acp_client import remote + from raven.node.client import ChannelEndpoint + from raven.ops import remote_path + + link = _Link() + endpoint = ChannelEndpoint(link, 1, target=remote.machine("box"), agent="where@box") # type: ignore[arg-type] + endpoint.feed_frame("PONG", 1) + endpoint.feed_frame("", 2) + endpoint.feed_stderr("done already") + endpoint.exited(0) + monkeypatch.setattr(remote_path, "_NODES", lambda: _Placed(endpoint)) + + head, _ = _split(await _run(cli_config("sh -c 'echo PONG'"), "hi", tmp_path)) + + assert head == "PONG" and link.said == [], "nothing is sent to an agent that is gone" + + +async def test_a_link_down_too_long_for_the_prompt_fails_the_run(box, tmp_path, monkeypatch): + from raven.acp_client import remote + from raven.node import client as node_client + from raven.node.client import ChannelEndpoint + from raven.ops import remote_path + + link = _Link() + endpoint = ChannelEndpoint(link, 1, target=remote.machine("box"), agent="where@box") # type: ignore[arg-type] + endpoint.detach("the link went") + endpoint._pending = [(node_client.protocol.AGENT_FRAME, {"line": "x"})] * node_client._PENDING_FRAMES + monkeypatch.setattr(remote_path, "_NODES", lambda: _Placed(endpoint)) + + with pytest.raises(BrokenPipeError, match="down too long"): + await _run(cli_config("cat"), "hi", tmp_path) + + async def test_a_cli_agent_that_exits_non_zero_there_fails_with_its_one_line(box, tmp_path): with pytest.raises(RuntimeError, match=r"CLI agent 'where@box' exited 3: token expired$"): await _run(cli_config("sh -c 'echo token expired; exit 3'"), "hi", tmp_path) @@ -254,8 +325,7 @@ async def test_the_files_that_came_with_the_task_are_named_by_address_and_not_co head, _ = _split(await _run(cli_config("cat"), "count the rows", tmp_path, media=media, instance="counter")) - # One line break, not the note's two: the node forwards whole lines, and a blank one does not come back. - assert head.startswith("count the rows\nFiles that came with this task, by their address on the computer") + assert head.startswith("count the rows\n\nFiles that came with this task, by their address on the computer") assert "Nothing was copied here" in head and f"- {data}" in head assert "[attachments, by absolute path]" not in head, "not the local note, whose paths mean nothing there" assert not (box.home / "raven-work" / "counter" / "sales.csv").exists() From 4778ada090d7ad2e29661f0d028b8d3aca63fb26 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:11:51 +0800 Subject: [PATCH 23/44] test(node): the watcher test reads a frame as a pipe gives it, with its line break Co-authored-by: Claude (claude-fable-5-1) --- tests/test_node_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_node_client.py b/tests/test_node_client.py index bcc9392f4..2d19f3d23 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -1198,7 +1198,7 @@ def bad(n: int) -> None: sink = logger.add(lambda message: records.append(message.record["message"]), level="WARNING") try: endpoint.feed_frame('{"jsonrpc":"2.0","id":1,"result":{}}', 1) - assert await endpoint.readline() == b'{"jsonrpc":"2.0","id":1,"result":{}}' + assert await endpoint.readline() == b'{"jsonrpc":"2.0","id":1,"result":{}}\n' endpoint.consumed() finally: logger.remove(sink) From 5d53dd5b82ffc88f069ce71f5e63de11b97a2c56 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:46:54 +0800 Subject: [PATCH 24/44] fix(*): a copy to a machine stays the shell's, a remote write asks, a read's digest is its reader's The shell refused scp, sftp and rsync to a registered machine and sent the model to the file tools. The refusal protected nothing: a copy starts nothing on the far side and passes exec's own question like any other command, the file tools show the same address the refusal objected to, and they cannot carry what a copy carries -- a directory, a binary file, a file over the fetch budget. With the lane off they would even have written the spelling here. The copy is the plain shell's again, as on main, and the ssh refusal names the file tools only while the lane is on. A write on a machine was allowed without a question inside the directories the agents work in there, where a write on this computer asks; it asks wherever it falls now, and its grant is for the one file, as here. The digest a whole-file write expected was one table for the process, so another agent's read of a changed file let a write based on an older read through, and refused a writer that never read the file. The digest is the reader's own -- the conversation, and the sub-agent run inside it -- and the table is bounded. The files deliver_files fetches are not reclaimed, and the landing says so: left out on purpose. The design ids in comments go; each site says in words what it means. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/deliver.py | 5 +- raven/agent/tools/machine_exec.py | 193 ++------------------------ raven/agent/tools/remote_files.py | 40 ++++-- raven/agent/tools/shell.py | 33 ++--- raven/ops/remote_path.py | 55 +++++--- raven/permissions/builtin.py | 14 +- raven/permissions/rules.py | 11 +- tests/test_deliver_files_tool.py | 2 +- tests/test_ops_remote_path.py | 45 ++++-- tests/test_permissions_remote_read.py | 47 ++++--- tests/test_remote_file_tools.py | 39 ++++++ tests/test_shell_machine_channel.py | 107 ++++++-------- 12 files changed, 251 insertions(+), 340 deletions(-) diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index 77fe5db4c..e8db466ff 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -147,7 +147,10 @@ def _landing(workspace: Path | None, machine: str, place: str) -> Path: keeps (:func:`_local_name`), stays readable beside it. Checked before anything is made or written: a landing that resolves outside its machine's delivery directory is refused, whatever the name did to get - there. + there. Nothing reclaims what lands here: a delivery is opened from its + landing for as long as its record lasts, and a bound by size or age was + left out of this change on purpose; clearing ``pulled/`` is the owner's + to do. """ pulled = (workspace or Path.cwd()) / "pulled" / _local_name(machine) key = hashlib.sha256(f"{machine}\0{place}".encode()).hexdigest()[:16] diff --git a/raven/agent/tools/machine_exec.py b/raven/agent/tools/machine_exec.py index 775b5f793..b50a8a4d2 100644 --- a/raven/agent/tools/machine_exec.py +++ b/raven/agent/tools/machine_exec.py @@ -104,7 +104,7 @@ def machines_registered() -> bool: # next command and not to this ssh. _COMMAND_SEPARATORS = frozenset({";", "&", "&&", "|", "||", "|&", "(", ")"}) #: On a Windows host the shell that runs a command reads a backslash as a path's -#: own, not as an escape: the words are read that way too, so ``C:\...\scp.exe`` +#: own, not as an escape: the words are read that way too, so ``C:\...\ssh.exe`` #: keeps its name. _WINDOWS = os.name == "nt" @@ -186,13 +186,12 @@ def _read_option_group(word: str, tokens: list[str], index: int) -> tuple[int, s def _program(word: str) -> str: - """The program a command word runs, as the refusals name one: ``ssh``, ``scp``, ``sftp``, ``rsync``. + """The program a command word runs, as the refusal names one: ``ssh``. Written ``ssh``, ``/usr/bin/ssh`` or ``\\ssh`` (a backslash suppresses alias - lookup and still runs the client), and on Windows ``scp.exe``, ``SCP.EXE`` - or ``C:\\Windows\\System32\\OpenSSH\\scp.exe``, all of which reach the far - side: the directory, the case and the suffix are taken off (before - this, the three Windows spellings went past the copy refusal). + lookup and still runs the client), and on Windows ``ssh.exe``, ``SSH.EXE`` + or ``C:\\Windows\\System32\\OpenSSH\\ssh.exe``, all of which reach the far + side: the directory, the case and the suffix are taken off. """ name = PureWindowsPath(PurePosixPath(word.strip("\"'").lstrip("\\")).name).name.lower() for suffix in _EXE_SUFFIXES: @@ -313,178 +312,6 @@ def _ssh_destinations(command: str) -> list[tuple[str, int]]: return found -#: The programs that copy files to or from a machine over ssh, and so carry a -#: destination written ``[user@]host:path``, ``host::module`` or as a URI. -_COPIERS = frozenset({"scp", "sftp", "rsync"}) -# The short options of each that take a value, so the value is never read as -# an operand: ``scp -P 58717 a.txt host:b`` names a port, not a file. rsync's -# ``-P`` takes none (it is ``--partial --progress``). -_COPY_VALUE_FLAGS = { - "scp": frozenset("iFclSJDoP"), - "sftp": frozenset("iFclSJDoPbsBRX"), - "rsync": frozenset("eBTMf"), -} -# rsync's long options that take a value the next word may carry. -_RSYNC_VALUE_OPTIONS = frozenset( - { - "--rsh", - "--port", - "--files-from", - "--exclude", - "--include", - "--exclude-from", - "--include-from", - "--log-file", - "--timeout", - "--contimeout", - "--bwlimit", - "--chmod", - "--chown", - "--rsync-path", - "--password-file", - "--out-format", - "--temp-dir", - "--compare-dest", - "--copy-dest", - "--link-dest", - "--backup-dir", - "--suffix", - "--max-size", - "--min-size", - "--filter", - "--partial-dir", - "--iconv", - "--block-size", - "--max-delete", - "--modify-window", - "--address", - "--sockopts", - "--checksum-choice", - } -) -_URI_DESTINATION = re.compile( - r"^(?:scp|sftp|rsync)://(?:[^/@]+@)?(?P\[[^\]]+\]|[^/:]+)(?::(?P\d+))?(?:/|$)", re.IGNORECASE -) -# The port an rsync ``-e``/``--rsh`` value hands its ssh: ``ssh -p 58717``, ``ssh -o Port=58717``. -_RSH_PORT = re.compile(r"(?:^|\s)-p\s*(\d+)|(?:^|\s)-o\s*[\"']?Port[=\s](\d+)", re.IGNORECASE) - - -def _copy_host(word: str) -> tuple[str, int | None] | None: - """The host (and the port a URI carries) a scp, sftp or rsync operand names, or ``None`` for a local path. - - scp's own rule: a colon before any slash makes a remote spelling, so - ``./a:b`` and ``/tmp/a:b`` are files here. ``host::module`` and - ``rsync://`` are rsync's daemon forms; the host still counts. - """ - if match := _URI_DESTINATION.match(word): - port = match.group("port") - return match.group("host").strip("[]").lower(), (int(port) if port else None) - if "::" in word: - head = word.split("::", 1)[0] - elif ":" in word: - head = word.split(":", 1)[0] - else: - return None - if not head or "/" in head: - return None - host = head.rsplit("@", 1)[-1].strip("[]").lower() - return (host, None) if host else None - - -def _copy_destinations(command: str) -> list[tuple[str, int]]: - """Every host and port a ``scp``, ``sftp`` or ``rsync`` word in a shell command would reach. - - Read the way :func:`_ssh_destinations` reads ssh, program by program: the - port from scp's and sftp's ``-P`` or ``-o Port``, from rsync's ``--port`` - or the ssh its ``-e``/``--rsh`` names, else 22; every operand written as a - remote path names the host, and all of one command's share its port. - """ - tokens = _words(command) - if not tokens: - return [] - found: list[tuple[str, int]] = [] - index = 0 - while index < len(tokens): - program = _program(tokens[index]) - if program not in _COPIERS: - index += 1 - continue - index += 1 - port: int | None = None - operands: list[str] = [] - options_ended = False - while index < len(tokens): - word = tokens[index] - if word in _COMMAND_SEPARATORS or _program(word) in _COPIERS | {"ssh"}: - break - following = tokens[index + 1] if index + 1 < len(tokens) else None - if word in _REDIRECTIONS: - index += 1 - if following is not None and following not in _COMMAND_SEPARATORS and following not in _REDIRECTIONS: - index += 1 - continue - if word.isdigit() and following in _REDIRECTIONS: - index += 1 - continue - index += 1 - if word == "--" and not options_ended: - options_ended = True - continue - if options_ended or not word.startswith("-") or len(word) == 1: - operands.append(word) - continue - if word.startswith("--"): - name, _, value = word.partition("=") - if not _ and program == "rsync" and name in _RSYNC_VALUE_OPTIONS and index < len(tokens): - value = tokens[index] - index += 1 - if name == "--port" and value.isdigit(): - port = int(value) - elif name == "--rsh" and (match := _RSH_PORT.search(value)): - port = int(match.group(1) or match.group(2)) - continue - for offset, letter in enumerate(word[1:]): - if letter not in _COPY_VALUE_FLAGS[program]: - continue - value = word[offset + 2 :] - if not value and index < len(tokens): - value = tokens[index] - index += 1 - if letter == "P" and value.isdigit(): - port = int(value) - elif letter == "o": - pair = _OPTION_SPLIT.split(value.strip(), maxsplit=1) - if len(pair) == 2 and pair[0].lower() == "port" and pair[1].strip().isdigit(): - port = int(pair[1].strip()) - elif letter == "e" and (match := _RSH_PORT.search(value)): - port = int(match.group(1) or match.group(2)) - break - for word in operands: - named = _copy_host(word) - if named is None and program == "sftp": - # sftp's one operand is the machine, with or without a path: - # ``sftp root@h`` opens a session there. - head = word.split(":", 1)[0] - host = head.rsplit("@", 1)[-1].strip("[]").lower() - named = (host, None) if host and "/" not in head else None - if named is not None: - host, explicit = named - found.append((host, explicit or port or 22)) - return found - - -def raw_copy_target(command: str) -> dict[str, Any] | None: - """The registered machine a plain-shell ``scp``, ``sftp`` or ``rsync`` reaches, or ``None``. - - The sibling of :func:`raw_ssh_target` for the copying clients. Refused by - ``exec`` not because a copy starts anything there -- it does not -- but - because the address it carries would then stand in the model's text where - a path shown by the file tools belongs, and the copy would go around the - budgets and the read check those tools keep. - """ - return _registered(_copy_destinations(command)) - - def raw_ssh_target(command: str) -> dict[str, Any] | None: """The registered machine a plain-shell command reaches over its own ssh. @@ -501,14 +328,10 @@ def raw_ssh_target(command: str) -> dict[str, Any] | None: typed ``ssh -p root@ '... &'`` from the local shell 58 times to start GPU work: the look here is capped at 60 s and on-call's job runner was not theirs to call, so the address was the path of least resistance, - and the ledger never saw the runs. ``scp``, ``sftp`` and ``rsync`` are - :func:`raw_copy_target`'s, for a reason of their own. + and the ledger never saw the runs. Only ssh is matched -- ``scp`` and + ``rsync`` move files and start nothing on the far side. """ - return _registered(_ssh_destinations(command)) - - -def _registered(destinations: list[tuple[str, int]]) -> dict[str, Any] | None: - """The registry row at the first of ``destinations`` that is registered, host and port both.""" + destinations = _ssh_destinations(command) if not destinations: return None try: diff --git a/raven/agent/tools/remote_files.py b/raven/agent/tools/remote_files.py index 830151b26..22cfebb15 100644 --- a/raven/agent/tools/remote_files.py +++ b/raven/agent/tools/remote_files.py @@ -7,11 +7,14 @@ fenced to the directory the permission check allowed, and sent to the machine's node (:mod:`raven.node.client`), which runs the very tool the call named there. Only the answer comes back, and every path in it is shown the -way scp writes it, with the registry's user, address and port (C28). +way scp writes it, with the registry's user, address and port. -A read remembers the file's digest; a write that replaces the whole file -then tells the node what it expects to replace, and a file changed in -between is left alone, with a sentence that says to read it again. +A read remembers the file's digest for the agent that read it -- the +conversation, and the sub-agent run inside it; a write by that agent that +replaces the whole file then tells the node what it expects to replace, and +a file changed in between is left alone, with a sentence that says to read +it again. Another agent's read does not vouch for it, and an agent that +never read the file replaces it, as a write here does. Every failure is one sentence the tool returns: a path that names no registered machine, a machine without this Raven's node, one that does not @@ -57,6 +60,7 @@ async def answer(tool: str, path: Any, params: dict[str, Any]) -> str | ToolResu pool = remote_path.nodes() if pool is None: return f"Error: {path} is on another machine, and this process does not read registered machines' files" + reader = _reader() try: home = await pool.home(where.machine) place = where.absolute(home) @@ -65,27 +69,45 @@ async def answer(tool: str, path: Any, params: dict[str, Any]) -> str | ToolResu if ( tool == "write_file" and params.get("mode") != "append" - and (read := remote_path.last_read(where.machine, place)) + and (read := remote_path.last_read(where.machine, place, reader=reader)) ): - # What this process last read there, so a file changed since is + # What this agent last read there, so a file changed since is # not written over unseen (``fs/write``'s ``expect``). sent["expect"] = read got = await pool.call(where.machine, _METHODS[tool], sent) except remote_path.NodeCallError as exc: if exc.reason == "changed": - remote_path.forget_read(where.machine, place) + remote_path.forget_read(where.machine, place, reader=reader) return ( f"Error: {where.display(place)} on machine {remote_path._label(where._row)} changed after it was " "read; nothing was written. Read it again before writing." ) return f"Error: {exc}" if tool == "read_file" and isinstance(got.get("sha256"), str): - remote_path.remember_read(where.machine, place, got["sha256"]) + remote_path.remember_read(where.machine, place, got["sha256"], reader=reader) elif tool in ("write_file", "edit_file"): - remote_path.forget_read(where.machine, place) + remote_path.forget_read(where.machine, place, reader=reader) return _shown(got, where, [place, *roots], tool) +def _reader() -> str: + """Who is reading: the conversation, and the sub-agent run inside it, whose later write expects what it read. + + The permission turn names the conversation; a delegated run inside one is + told apart by its activity's uid, since the sub-agents of a turn read and + write on their own. Empty outside both, where every call is one reader. + """ + # Imported here, both: a machine's node imports the file tools and runs + # them with only its own requirements importable, and neither the + # permission turn nor the sub-agent package is among them. This runs on + # the host alone, once a path has named a machine. + from raven.agent.subagent import activity + from raven.permissions.turn import current_turn + + run = activity.current() + return "\x00".join(part for part in (current_turn().session_key, run.uid if run is not None else "") if part) + + #: The lines of a tool's answer that are the file's own bytes, which no path #: display may rewrite: a read's numbered lines, a search's matches and #: context (``path:line:text`` and ``path-line-text``). What the tool says diff --git a/raven/agent/tools/shell.py b/raven/agent/tools/shell.py index d7a5f3022..59f52f1fd 100644 --- a/raven/agent/tools/shell.py +++ b/raven/agent/tools/shell.py @@ -303,31 +303,26 @@ async def execute( # Typed ssh to a registered machine is the registry bypassed: the # cap, the process-group sweep and the ledger all live on the # other two paths. Refused by name so the model learns the path, - # not just that this one closed. + # not just that this one closed. A copy there with scp, sftp or + # rsync starts nothing and passes this tool's own question like + # any other command, so it is left alone; and the file tools are + # named only while they read a remote path, with the machine + # lane on -- with it off they would write that spelling here. + from raven.home import machine_lane_enabled + conn_id = str(row.get("id") or "").strip() + files = ( + " Its files are read and written with read_file, write_file and edit_file on the path the way " + "scp writes it." + if machine_lane_enabled() + else "" + ) return ( f"Error: this command reaches {row.get('display_name') or conn_id} over raw ssh from " f"this computer, and that machine is registered as {conn_id!r}. Look at it with " f"exec(machine={conn_id!r}) (capped at 60s, nothing left running), and hand anything " "longer, or anything that must keep running, to the on-call agent's ops_submit " - "(budget, dedup, ledger). Its files are read and written with read_file, write_file " - "and edit_file on the path the way scp writes it. Nothing was run." - ) - from raven.agent.tools.machine_exec import raw_copy_target - - if (row := raw_copy_target(command)) is not None: - # A copy over scp, sftp or rsync starts nothing there, but it - # carries the address into the model's text and goes around the - # file tools' budgets and read check. The tools take the same - # spelling, so the model learns the path, not just that this - # one closed. - conn_id = str(row.get("id") or "").strip() - return ( - f"Error: this command copies files to or from {row.get('display_name') or conn_id} with scp, " - f"sftp or rsync from this computer, and that machine is registered as {conn_id!r}. Its files " - "are reached through its Raven node instead: give read_file, list_dir, find, grep, write_file " - "and edit_file the path the way scp writes it (user@host:/path, or scp://user@host:port//path), " - "and deliver_files brings a file back. Nothing was run." + f"(budget, dedup, ledger).{files} scp and rsync to it are still fine here. Nothing was run." ) cwd = self._cwd_for(working_dir) diff --git a/raven/ops/remote_path.py b/raven/ops/remote_path.py index f35ffb67b..89d5b5b09 100644 --- a/raven/ops/remote_path.py +++ b/raven/ops/remote_path.py @@ -51,8 +51,8 @@ #: check is about. READING_TOOLS = frozenset({"read_file", "list_dir", "find", "grep"}) -#: The writing tools a remote path reaches. Freed by the same roots as a read, -#: asked about on their own account elsewhere. +#: The writing tools a remote path reaches. Asked about wherever they fall, as +#: a write here is; the roots place their fence, not their answer. WRITING_TOOLS = frozenset({"write_file", "edit_file"}) # The tools whose path names a file, so the directory a call reaches is the file's. @@ -184,26 +184,34 @@ def known_home(machine: str) -> str: return _HOMES.get(machine, "") -# The digest of each remote file as it was last read in this process, by -# machine and absolute path (``raven.agent.tools.remote_files``), so a write -# that replaces the whole file can say what it expects to replace. -_READ: dict[tuple[str, str], str] = {} +# The digest of each remote file as its reader last read it in this process, +# by reader, machine and absolute path (``raven.agent.tools.remote_files``), so +# a write that replaces the whole file can say what it expects to replace. By +# reader because the digest describes what one agent saw: another agent's +# later read must not vouch for a write based on an older one. Bounded, since +# readers come and go; past the bound the oldest entry goes, and the write it +# would have checked is then one by an agent that never read the file. +_READ: dict[tuple[str, str, str], str] = {} +_READ_LIMIT = 4096 -def remember_read(machine: str, path: str, digest: str) -> None: - """Record ``digest`` as what ``path`` on ``machine`` held when it was read.""" +def remember_read(machine: str, path: str, digest: str, *, reader: str = "") -> None: + """Record ``digest`` as what ``path`` on ``machine`` held when ``reader`` read it.""" if machine and path.startswith("/") and digest: - _READ[(machine, path)] = digest + _READ.pop((reader, machine, path), None) + _READ[(reader, machine, path)] = digest + while len(_READ) > _READ_LIMIT: + del _READ[next(iter(_READ))] -def last_read(machine: str, path: str) -> str | None: - """The digest ``path`` on ``machine`` had when this process last read it, if it did.""" - return _READ.get((machine, path)) +def last_read(machine: str, path: str, *, reader: str = "") -> str | None: + """The digest ``path`` on ``machine`` had when ``reader`` last read it in this process, if it did.""" + return _READ.get((reader, machine, path)) -def forget_read(machine: str, path: str) -> None: - """Drop what was read of ``path`` on ``machine``: it was written, or it changed.""" - _READ.pop((machine, path), None) +def forget_read(machine: str, path: str, *, reader: str = "") -> None: + """Drop what ``reader`` read of ``path`` on ``machine``: it wrote the file, or found it changed.""" + _READ.pop((reader, machine, path), None) def target(text: str) -> RemotePath | None: @@ -216,7 +224,7 @@ def target(text: str) -> RemotePath | None: def reading_roots(where: RemotePath) -> list[str]: - """The directories on ``where``'s machine a read needs no question for (C30). + """The directories on ``where``'s machine a read needs no question for. The default remote root, the ``remoteCwd`` of every acp agent entry that names this machine, and the registry row's ``paths``: where this Raven's @@ -263,10 +271,12 @@ class Reading: """What one call on a remote path reaches, for its permission check.""" target: RemotePath + place: str + """The path the call names, placed on the machine (:func:`_placed`).""" directory: str """The directory the call reaches: the file's for a tool that names a file, else the path itself.""" inside: bool - """Whether that is under a :func:`reading_roots` directory, so no question is asked.""" + """Whether that is under a :func:`reading_roots` directory, so a read asks no question.""" def reading(tool: str, params: dict[str, Any] | None) -> Reading | None: @@ -283,10 +293,11 @@ def reading(tool: str, params: dict[str, Any] | None) -> Reading | None: def writing(tool: str, params: dict[str, Any] | None) -> Reading | None: """What ``tool``'s writing call reaches on a registered machine, or ``None``. - The roots that free a read free a write too: the directories this - Raven's agents work in there, and the row's own paths. Elsewhere a write - asks on its own account, apart from any read that was allowed - (``raven.permissions.builtin.session_keys``). + A write there asks wherever it falls, as a write here does + (``raven.permissions.rules.default_tier``), and a grant is for the one + file, apart from any read that was allowed + (``raven.permissions.builtin.session_keys``); the roots decide only the + fence the node is given (:func:`fence`). """ return _reach(tool, params) if tool in WRITING_TOOLS else None @@ -305,7 +316,7 @@ def _reach(tool: str, params: dict[str, Any] | None) -> Reading | None: place = _placed(where.path, home) directory = posixpath.dirname(place) or place if tool in _FILE_TOOLS else place inside = any(_under(place, _placed(root, home)) for root in reading_roots(where)) - return Reading(where, directory, inside) + return Reading(where, place, directory, inside) def fence(tool: str, where: RemotePath, place: str, home: str) -> str: diff --git a/raven/permissions/builtin.py b/raven/permissions/builtin.py index 08283a2c5..1b9baec59 100644 --- a/raven/permissions/builtin.py +++ b/raven/permissions/builtin.py @@ -117,8 +117,9 @@ def session_keys(tool_name: str, params: dict[str, Any], ask_segments: tuple[str call, so its grant is never presented by a call to a real site. A read on a registered machine is keyed by its machine and the directory it reaches, placed on that machine, so the grant covers that directory however the path - is written. Every other tool keys the exact call -- a `path` field on an - unknown tool says nothing about what its other fields do. + is written; a write there by its machine and the one file, as a write here + is. Every other tool keys the exact call -- a `path` field on an unknown + tool says nothing about what its other fields do. """ if tool_name == "exec" and ask_segments: working_dir = params.get("working_dir") @@ -130,13 +131,14 @@ def session_keys(tool_name: str, params: dict[str, Any], ask_segments: tuple[str if (reach := remote_path.reading(tool_name, params)) is not None: # A read on a registered machine is granted for its machine and # directory, placed on the machine, so the same file written another - # way presents the same grant (C30). + # way presents the same grant. material = f"remote-read\x00{reach.target.machine}\x00{reach.directory}" return (sha256(material.encode()).hexdigest(),) if (reach := remote_path.writing(tool_name, params)) is not None: - # A write there likewise, and apart from the read: allowing a look at - # a directory is not allowing changes to it. - material = f"remote-write\x00{reach.target.machine}\x00{reach.directory}" + # A write there is granted for the one file, as a write here is, and + # apart from the read: allowing a look at a directory is not allowing + # changes in it. + material = f"remote-write\x00{reach.target.machine}\x00{reach.place}" return (sha256(material.encode()).hexdigest(),) path = params.get("path") if tool_name in PATH_KEYED_TOOLS and isinstance(path, str) and path.strip(): diff --git a/raven/permissions/rules.py b/raven/permissions/rules.py index 91986fa1f..c04b64d3c 100644 --- a/raven/permissions/rules.py +++ b/raven/permissions/rules.py @@ -682,13 +682,16 @@ def self_config_tier(tool_name: str, params: dict[str, Any] | None = None) -> Ti def default_tier(tool_name: str, params: dict[str, Any] | None = None) -> Tier: - if (reach := remote_path.reading(tool_name, params) or remote_path.writing(tool_name, params)) is not None: + if (reach := remote_path.reading(tool_name, params)) is not None: # A registered machine's disk is not this computer's: its login is # often root and its disks shared. The directories this Raven's agents - # work in there and the row's own paths read and write freely; - # anywhere else asks, once per machine and directory, a write apart - # from a read (C30; ``builtin.session_keys``). + # work in there and the row's own paths read freely; anywhere else + # asks, once per machine and directory (``builtin.session_keys``). return Tier.ALLOW if reach.inside else Tier.ASK + if remote_path.writing(tool_name, params) is not None: + # A write there asks wherever it falls, as a write here does: the + # roots free a look, not a change, and a grant is for the one file. + return Tier.ASK if tool_name in DEFAULT_ALLOW_TOOLS: return Tier.ALLOW if (params or {}).get("action") in DEFAULT_ALLOW_ACTIONS.get(tool_name, ()): diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 43cded64f..f15f4bfbd 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -546,7 +546,7 @@ async def test_what_was_not_delivered_says_why(tool, machines) -> None: assert "is written as a path on another machine" in nobody assert "not found or not a regular file on machine 'Lab box' (box)" in folder # The machine is named by its label in every sentence; its address appears - # only inside a path shown the way scp writes it (C28). + # only inside a path shown the way scp writes it. assert ( outside.count("203.0.113.7") == 1 and "scp://worker@203.0.113.7:58717//root/.ssh/id_rsa is outside" in outside ) diff --git a/tests/test_ops_remote_path.py b/tests/test_ops_remote_path.py index f03e7c24d..00997b300 100644 --- a/tests/test_ops_remote_path.py +++ b/tests/test_ops_remote_path.py @@ -45,18 +45,18 @@ def _where(text: str): @pytest.mark.parametrize( ("text", "machine", "path"), [ - # A16: the machine's id, the id with its user, and the URI with the address. + # The machine's id, the id with its user, and the URI with the address. ("box2:/root/x", "box2", "/root/x"), ("root@box2:/root/x", "box2", "/root/x"), ("scp://root@203.0.113.7:49273//root/x", "box2", "/root/x"), - # C25: the first slash after host:port only separates. + # The first slash after host:port only separates. ("scp://203.0.113.7:49273/root/x", "box2", "root/x"), ("scp://box2", "box2", ""), - # C26: after the colon, a path not starting with / is under the home. + # After the colon, a path not starting with / is under the home. ("box2:", "box2", ""), ("box2:raven-work/a", "box2", "raven-work/a"), ("box2:~/raven-work/a", "box2", "~/raven-work/a"), - # C21: an address with no port is port 22; the registry decides the rest. + # An address with no port is port 22; the registry decides the rest. ("alice@198.51.100.9:/data/x", "std", "/data/x"), ("198.51.100.9:/data/x", "std", "/data/x"), ("sftp://alice@198.51.100.9//data", "std", "/data"), @@ -80,9 +80,9 @@ def test_a_user_and_a_machine_id_are_compared_as_written(): @pytest.mark.parametrize( ("text", "said"), [ - # A17: no port means 22, and no registered machine is on 22 there. + # No port means 22, and no registered machine is on 22 there. ("root@203.0.113.7:/etc/hostname", "no registered machine matches it"), - # C24: the short form has no port field -- `49273:/a` is a file name on 22. + # The short form has no port field -- `49273:/a` is a file name on 22. ("root@203.0.113.7:49273:/a", "no registered machine matches it"), ("bob@std:/x", "logs in as 'bob', and machine 'std' is registered for another user"), ("scp://box2:41941//x", "names machine 'Box 2' (box2) on a port it is not registered on"), @@ -93,7 +93,7 @@ def test_a_user_and_a_machine_id_are_compared_as_written(): ("scp://[2001:db8::5//x", "opens an address with '['"), ("scp://[2001:db8::5]x//x", "something after the address that is not a port"), ("nobox:/x", "write it as ./name or as an absolute path"), - # C27: a local name holding a colon reads as remote unless written as a path + # A local name holding a colon reads as remote unless written as a path # (one letter before the colon is a drive on a Windows host: the drive-relative test). ("ab:b.txt", "write it as ./name or as an absolute path"), ], @@ -148,7 +148,7 @@ def test_a_drive_relative_path_is_this_computer_s_on_a_windows_host_and_scp_s_el @pytest.mark.parametrize( ("text", "shown"), [ - # C28: not port 22 -- the URI form, absolute paths with two slashes. + # Not port 22 -- the URI form, absolute paths with two slashes. ("box2:/root/raven-work/t1/report.md", "scp://root@203.0.113.7:49273//root/raven-work/t1/report.md"), ("box2:raven-work/a", "scp://root@203.0.113.7:49273/raven-work/a"), # Port 22 -- the short form. @@ -339,3 +339,32 @@ def test_the_node_pool_is_whatever_assembly_handed_over(): remote_path.use_nodes(lambda: pool) assert remote_path.nodes() is pool + + +# --- what a reader read ---------------------------------------------------------------- + + +def test_what_was_read_is_the_reader_s_own_and_the_table_is_bounded(monkeypatch): + monkeypatch.setattr(remote_path, "_READ", {}) + monkeypatch.setattr(remote_path, "_READ_LIMIT", 3) + + remote_path.remember_read("box2", "/root/a", "d1", reader="A") + remote_path.remember_read("box2", "/root/a", "d2", reader="B") + assert remote_path.last_read("box2", "/root/a", reader="A") == "d1" + assert remote_path.last_read("box2", "/root/a", reader="B") == "d2" + assert remote_path.last_read("box2", "/root/a") is None, "no name is one more reader, not every reader" + + remote_path.forget_read("box2", "/root/a", reader="A") + assert remote_path.last_read("box2", "/root/a", reader="A") is None + assert remote_path.last_read("box2", "/root/a", reader="B") == "d2", "A wrote: B's next overwrite is checked" + + remote_path.remember_read("box2", "/root/b", "d3", reader="B") + remote_path.remember_read("box2", "/root/a", "d4", reader="B") # read again: the newest entry now + remote_path.remember_read("box2", "/root/c", "d5", reader="B") + remote_path.remember_read("box2", "/root/d", "d6", reader="B") + assert remote_path.last_read("box2", "/root/b", reader="B") is None, "past the bound the oldest goes" + assert [remote_path.last_read("box2", p, reader="B") for p in ("/root/a", "/root/c", "/root/d")] == [ + "d4", + "d5", + "d6", + ] diff --git a/tests/test_permissions_remote_read.py b/tests/test_permissions_remote_read.py index c6681b8fe..9c3159b6f 100644 --- a/tests/test_permissions_remote_read.py +++ b/tests/test_permissions_remote_read.py @@ -1,10 +1,11 @@ -"""The permission check for a read on a registered machine (C30). +"""The permission check for a read, and a write, on a registered machine. A registered machine's login is often root and its disks shared, so a read there is not waved through the way a read here is: the directories this Raven's agents work in there and the row's own paths read freely; anywhere else asks, and a grant for the session covers that machine and directory -however the path is written. +however the path is written. A write there asks wherever it falls, as a +write here does, and its grant is for the one file. """ from __future__ import annotations @@ -115,36 +116,38 @@ async def test_the_gate_asks_once_and_the_grant_holds_for_the_directory(): assert isinstance(inside, Allow) and inside.source is DecisionSource.DEFAULT -# --- writes ask on their own account -------------------------------------------------------- +# --- writes ask wherever they fall, as a write here does ------------------------------------ @pytest.mark.parametrize( - ("tool", "path", "tier"), + ("tool", "path"), [ - ("write_file", "box:raven-work/t1/out.md", Tier.ALLOW), - ("edit_file", "box:/srv/agents/t1/a.py", Tier.ALLOW), - ("edit_file", "nas:/data/shared/runs/notes.md", Tier.ALLOW), - ("write_file", "box:.bashrc", Tier.ASK), - ("write_file", "nas:/etc/hosts", Tier.ASK), - ("edit_file", "box:raven-work/../.ssh/authorized_keys", Tier.ASK), + # Inside the roots that free a read: a change is still asked about. + ("write_file", "box:raven-work/t1/out.md"), + ("edit_file", "box:/srv/agents/t1/a.py"), + ("edit_file", "nas:/data/shared/runs/notes.md"), + ("write_file", "box:.bashrc"), + ("write_file", "nas:/etc/hosts"), + ("edit_file", "box:raven-work/../.ssh/authorized_keys"), # A write here asks as it always did. - ("write_file", "/tmp/notes.md", Tier.ASK), + ("write_file", "/tmp/notes.md"), ], ) -def test_a_remote_write_is_free_in_the_roots_and_asks_elsewhere(tool, path, tier): - assert default_tier(tool, {"path": path}) is tier +def test_a_remote_write_asks_wherever_it_falls(tool, path): + assert default_tier(tool, {"path": path}) is Tier.ASK -def test_a_write_grant_is_kept_apart_from_a_read_grant_and_covers_its_directory(): +def test_a_write_grant_is_kept_apart_from_a_read_grant_and_covers_the_one_file(): remote_path.remember_home("box", "/root") read = session_keys("read_file", {"path": "box:/root/.ssh/id_rsa"}) write = session_keys("write_file", {"path": "box:/root/.ssh/id_rsa"}) - same_dir = session_keys("edit_file", {"path": "scp://root@203.0.113.7:58717//root/.ssh/config"}) - other_dir = session_keys("write_file", {"path": "box:/root/.config/x"}) + same_file = session_keys("edit_file", {"path": "scp://root@203.0.113.7:58717//root/.ssh/id_rsa"}) + relative = session_keys("write_file", {"path": "box:.ssh/id_rsa"}) + neighbour = session_keys("write_file", {"path": "box:/root/.ssh/config"}) - assert write == same_dir - assert len({read, write, other_dir}) == 3 + assert write == same_file == relative + assert len({read, write, neighbour}) == 3 assert session_keys("write_file", {"path": "/tmp/a"}) != session_keys("write_file", {"path": "/tmp/b"}) @@ -162,7 +165,11 @@ async def test_a_read_allowed_for_the_session_does_not_allow_a_write_there(): assert isinstance(write, NeedsApproval), "a look is not a change" remember_allowed("conv-1", write.session_keys) - again = await gate.check("edit_file", {"path": "box:.config/raven/other.json", "old_text": "a", "new_text": "b"}) + again = await gate.check("edit_file", {"path": "box:.config/raven/state.json", "old_text": "a", "new_text": "b"}) assert isinstance(again, Allow) and again.source is DecisionSource.SESSION + neighbour = await gate.check( + "edit_file", {"path": "box:.config/raven/other.json", "old_text": "a", "new_text": "b"} + ) + assert isinstance(neighbour, NeedsApproval), "the grant is for the one file, as it is here" inside = await gate.check("write_file", {"path": "box:raven-work/t1/out.md", "content": "x"}) - assert isinstance(inside, Allow) and inside.source is DecisionSource.DEFAULT + assert isinstance(inside, NeedsApproval), "the roots free a look, not a change" diff --git a/tests/test_remote_file_tools.py b/tests/test_remote_file_tools.py index d14d1597d..c4cc57753 100644 --- a/tests/test_remote_file_tools.py +++ b/tests/test_remote_file_tools.py @@ -32,6 +32,7 @@ def _fresh_state(monkeypatch): monkeypatch.setattr(remote_path, "_HOMES", {}) monkeypatch.setattr(remote_path, "_NODES", None) + monkeypatch.setattr(remote_path, "_READ", {}) monkeypatch.setattr( "raven.config.loader.load_config", lambda *a, **k: SimpleNamespace(subagents=SimpleNamespace(agents=[])) ) @@ -433,6 +434,44 @@ async def call(self, machine_id: str, method: str, params: dict[str, Any]) -> di assert remote_path.last_read("box", f"{HOME}/raven-work/t1/a.md") is None, "the next write reads first" +async def test_a_read_s_digest_is_the_reader_s_own_so_another_agent_s_read_vouches_for_no_write( + registry, pool, tmp_path +): + """A read v1; the file became v2 and B read it: A's overwrite still expects v1. C, who never read it, expects nothing. + + The conversation is the reader, and a sub-agent run inside one is a reader + of its own: B's sub-agent reads and writes on its own account, and B's + next overwrite is checked against B's read, not its sub-agent's write. + """ + from raven.agent.subagent import activity + from raven.permissions import turn + + place = "box:raven-work/t1/a.md" + reader, writer = _tools(tmp_path)["read_file"], _writers(tmp_path)["write_file"] + token = turn._TURN.set(turn.PermissionTurn(conversation_id="conv-a")) + try: + pool.answer = {"text": "1| one", "sha256": "a1" * 32} + await reader.execute(place) + turn._TURN.set(turn.PermissionTurn(conversation_id="conv-b")) + pool.answer = {"text": "1| two", "sha256": "b2" * 32} + await reader.execute(place) + with activity.collecting(): + await reader.execute(place) + pool.answer = {"text": "ok"} + await writer.execute(place, "by the sub-agent of b\n") + turn._TURN.set(turn.PermissionTurn(conversation_id="conv-a")) + await writer.execute(place, "by a\n") + turn._TURN.set(turn.PermissionTurn(conversation_id="conv-c")) + await writer.execute(place, "by c\n") + turn._TURN.set(turn.PermissionTurn(conversation_id="conv-b")) + await writer.execute(place, "by b\n") + finally: + turn._TURN.reset(token) + + expected = [params.get("expect") for _, method, params in pool.calls if method == "fs/write"] + assert expected == ["b2" * 32, "a1" * 32, None, "b2" * 32] + + def test_the_approval_card_for_a_write_on_a_machine_names_the_machine_and_the_path_as_written(registry, tmp_path): tools = _writers(tmp_path) diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index 17cdcfb29..b21bd39c4 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -810,64 +810,37 @@ def served() -> dict: assert "pass 'machine'" in after["description"] -# ---- a copy to a registered machine goes through the file tools, not around them ---- +# ---- a copy to a registered machine is the plain shell's, as on main ---- +@pytest.mark.asyncio @pytest.mark.parametrize( "command", [ "scp -P 58717 a.txt root@203.0.113.7:/tmp/", - "scp -rP 58717 root@203.0.113.7:/tmp/a ./here", - "scp -o Port=58717 a.txt 203.0.113.7:a", - "scp -P58717 a.txt root@203.0.113.7:", "sftp -P 58717 root@203.0.113.7", "rsync -av -e 'ssh -p 58717' ./dir root@203.0.113.7:/data/", - "rsync -avP --rsh='ssh -o Port=58717' root@203.0.113.7:/data/ ./dir", - "rsync -av --port 58717 ./dir rsync://203.0.113.7/module", - "scp scp://root@203.0.113.7:58717//tmp/a .", - "true && \\scp -P 58717 a root@203.0.113.7:b", "scp.exe -P 58717 a.txt root@203.0.113.7:/tmp/", - "SCP.EXE -P 58717 a.txt root@203.0.113.7:/tmp/", - "sftp.exe -P 58717 root@203.0.113.7", ], ) -async def test_a_copy_to_a_registered_machine_is_refused_and_points_at_the_file_tools(registry, tmp_path, command): +async def test_a_copy_to_a_registered_machine_is_not_refused(registry, tmp_path, command): + """A copy starts nothing on the far side, and passes exec's own question like any command. + + The file tools cannot carry what a copy carries -- a directory, a binary + file, a file over the fetch budget -- so a refusal left those with no + route, and taught the very spelling it objected to. ``true`` stands for + the client: nothing here connects anywhere. + """ tool = ExecTool(working_dir=str(tmp_path)) marker = tmp_path / "ran" - out = await tool.execute(command=f"{command}; touch {marker}") + out = await tool.execute(command=f"true {command}; touch {marker}") - assert out.startswith("Error: this command copies files to or from GPU box") - assert "conn_gpu" in out and "read_file" in out and "deliver_files" in out and "Raven node" in out - assert "203.0.113.7" not in out.replace("user@host", "") - assert not marker.exists() + assert "Error:" not in out + assert marker.exists() -@pytest.mark.parametrize( - ("command", "expected"), - [ - ("scp a.txt root@203.0.113.7:/tmp/", [("203.0.113.7", 22)]), - ("scp -P 58717 a.txt root@203.0.113.7:/tmp/ b.txt h2:x", [("203.0.113.7", 58717), ("h2", 58717)]), - ("scp ./a:b /tmp/", []), - ("scp /tmp/a:b /tmp/c", []), - ("rsync -av ./dir /tmp/dir", []), - ("rsync -avP root@h:/data/ ./dir", [("h", 22)]), - ("rsync -av --port=873 ./dir h::module", [("h", 873)]), - ("rsync -e 'ssh -p 2222' ./x h:y", [("h", 2222)]), - ("sftp -P 2222 -b batch.txt h", [("h", 2222)]), - ("sftp root@h:uploads", [("h", 22)]), - # Read wherever the word stands, as the ssh guard reads `ssh`: a false - # refusal of an echo costs a retry, a copy let through costs the address. - ("echo scp root@203.0.113.7:/tmp/a", [("203.0.113.7", 22)]), - ("scp -i key -P 58717 a root@203.0.113.7:b", [("203.0.113.7", 58717)]), - ("scp 'a b:c' root@h:d", [("a b", 22), ("h", 22)]), - ("scp a root@h:b 2>&1", [("h", 22)]), - ("scp -- -weird:name root@h:b", [("-weird", 22), ("h", 22)]), - ("scp 'unbalanced", []), - ], -) -def test_copy_destinations_are_read_the_way_the_programs_read_them(command, expected): - assert machine_exec._copy_destinations(command) == expected +# ---- the ssh client is read in its Windows spellings too ---- @pytest.mark.parametrize( @@ -876,24 +849,25 @@ def test_copy_destinations_are_read_the_way_the_programs_read_them(command, expe ("ssh", "ssh"), ("/usr/bin/ssh", "ssh"), ("\\ssh", "ssh"), - ("scp.exe", "scp"), - ("SCP.EXE", "scp"), - ("C:\\Windows\\System32\\OpenSSH\\scp.exe", "scp"), - ("rsync.cmd", "rsync"), - ("scpx", "scpx"), - ("my-scp.exe.bak", "my-scp.exe.bak"), + ("ssh.exe", "ssh"), + ("SSH.EXE", "ssh"), + ("C:\\Windows\\System32\\OpenSSH\\ssh.exe", "ssh"), + ("ssh.cmd", "ssh"), + ("sshx", "sshx"), + ("my-ssh.exe.bak", "my-ssh.exe.bak"), ], ) def test_the_program_a_word_runs_is_read_with_windows_spellings_too(word, program): assert machine_exec._program(word) == program +@pytest.mark.asyncio @pytest.mark.parametrize( "command", [ - "C:\\Windows\\System32\\OpenSSH\\scp.exe -P 58717 a.txt root@203.0.113.7:/tmp/", - '"C:\\Program Files\\OpenSSH\\scp.exe" -P 58717 a.txt root@203.0.113.7:/tmp/', "C:\\Windows\\System32\\OpenSSH\\ssh.exe -p 58717 root@203.0.113.7 ls", + '"C:\\Program Files\\OpenSSH\\ssh.exe" -p 58717 root@203.0.113.7 ls', + "SSH.EXE -p 58717 root@203.0.113.7 ls", ], ) async def test_on_a_windows_host_a_program_s_whole_path_keeps_its_backslashes_and_its_name( @@ -906,22 +880,20 @@ async def test_on_a_windows_host_a_program_s_whole_path_keeps_its_backslashes_an out = await tool.execute(command=f"{command}; touch {marker}") - assert out.startswith("Error:") and "the way scp writes it" in out and "Nothing was run." in out - assert "203.0.113.7" not in out.replace("user@host", "") + assert out.startswith("Error:") and "conn_gpu" in out and "Nothing was run." in out + assert "203.0.113.7" not in out assert not marker.exists() +@pytest.mark.asyncio @pytest.mark.parametrize( "command", [ 'ssh -p 58717 "root@203.0.113.7" true', - 'scp -P "58717" a.txt root@203.0.113.7:/tmp/a', - 'sftp -P "58717" "root@203.0.113.7"', - "rsync -av -e 'ssh -p 58717' ./dir \"root@203.0.113.7:/data/\"", + 'ssh -p "58717" root@203.0.113.7 true', 'ssh -p"58717" root@203.0.113.7 true', 'ssh -p 58717 "root@"203.0.113.7 true', 'ssh -p "587"17 root@203.0.113.7 true', - 'scp -P 58717 a.txt "root@203.0.113.7:"/tmp/a', ], ) async def test_on_a_windows_host_a_quoted_destination_or_port_is_matched_like_a_bare_one( @@ -935,8 +907,8 @@ async def test_on_a_windows_host_a_quoted_destination_or_port_is_matched_like_a_ out = await tool.execute(command=f"{command}; touch {marker}") - assert out.startswith("Error:") and "the way scp writes it" in out and "Nothing was run." in out - assert "203.0.113.7" not in out.replace("user@host", "") + assert out.startswith("Error:") and "conn_gpu" in out and "Nothing was run." in out + assert "203.0.113.7" not in out assert not marker.exists() @@ -971,14 +943,19 @@ def test_on_a_windows_host_the_words_are_read_without_their_quotes(monkeypatch): assert machine_exec._words('scp -P "58717" a.txt "root@h:/x"') == ["scp", "-P", "58717", "a.txt", "root@h:/x"] -async def test_a_copy_to_the_same_address_on_another_port_is_another_machine(registry): - assert machine_exec.raw_copy_target("scp a.txt root@203.0.113.7:/tmp/") is None - assert machine_exec.raw_copy_target("scp -P 58717 a.txt root@203.0.113.7:/tmp/") == dict(ROW) - - -async def test_typed_ssh_to_a_registered_machine_names_the_file_tools_too(registry, tmp_path): +@pytest.mark.asyncio +async def test_typed_ssh_s_refusal_leaves_copies_alone_and_names_the_file_tools_only_with_the_lane_on( + registry, tmp_path, monkeypatch +): + # With the lane off the file tools read a remote spelling as a path here, + # so naming them would send the model to write on this computer. tool = ExecTool(working_dir=str(tmp_path)) + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - out = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") + off = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + on = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") - assert "write_file and edit_file on the path the way scp writes it" in out + assert "scp and rsync to it are still fine here" in off and "read_file" not in off + assert "scp and rsync to it are still fine here" in on + assert "read_file, write_file and edit_file on the path the way scp writes it" in on From c8ee68ba2e208dd307dc4a8b3422d318f070c1b4 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:58:58 +0800 Subject: [PATCH 25/44] fix(agent): a paged read looks at each chunk once, so a line longer than a chunk costs its length The paged read held the line in progress whole and split it again with every chunk, so a file with no line break in it cost time square in its size: a 256 MiB single-line file took half a minute where the whole-file read took under a second, and one near 1 GiB would have outlasted the node's call timeout. Minified bundles, single-line JSON and machine-written CSV are that shape. Each chunk is split once now, and the bytes of the line in progress are kept only while that line may be on the page; off the page it is counted, not held. The same 256 MiB single-line file reads in a third of a second, on the page or off it, and the page is still exactly what splitlines gives. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/filesystem.py | 84 ++++++++++++++++++++++----------- tests/test_filesystem_tools.py | 47 ++++++++++++++++++ 2 files changed, 104 insertions(+), 27 deletions(-) diff --git a/raven/agent/tools/filesystem.py b/raven/agent/tools/filesystem.py index 67156ebe8..3f87146e9 100644 --- a/raven/agent/tools/filesystem.py +++ b/raven/agent/tools/filesystem.py @@ -40,42 +40,72 @@ _ODD_BREAKS = (b"\r", b"\x0b", b"\x0c", b"\x1c", b"\x1d", b"\x1e", b"\xc2\x85", b"\xe2\x80\xa8", b"\xe2\x80\xa9") +def _split_lines(body: bytes) -> list[bytes]: + """``body`` cut at every break ``str.splitlines`` knows; the last piece is what follows the last break.""" + odd = any(sign in body for sign in _ODD_BREAKS) + return _LINE_BREAK.split(body) if odd else body.split(b"\n") + + def _page_of_lines(handle: Any, start: int, limit: int) -> tuple[list[bytes], int]: """Lines ``start`` to ``start + limit`` (0-based) of the bytes ``handle`` reads, and how many lines there are. Exactly the lines ``read_text(encoding="utf-8").splitlines()`` would give, - read a chunk at a time. A break may straddle two chunks, so the bytes - that could begin one are held back until the next chunk says; the tail - after the last break is a line only when it holds something, as - ``splitlines`` has it. + read a chunk at a time and each chunk looked at once: the bytes of the + line in progress are kept only while that line may be on the page, so a + line longer than a chunk costs its length and nothing more, wherever it + falls. (Held whole and looked at again with every chunk, a file with no + break in it cost time square in its size: 256 MiB took half a minute.) + A break may straddle two chunks, so the bytes that could begin one are + held back until the next chunk says; the tail after the last break is a + line only when it holds something, as ``splitlines`` has it. """ page: list[bytes] = [] total = 0 stop = start + limit - - def take(lines: list[bytes]) -> None: - nonlocal total - if total < stop and total + len(lines) > start: - page.extend(lines[max(start - total, 0) : stop - total]) - total += len(lines) - - def cut(body: bytes) -> bytes: - """Count and keep the complete lines of ``body``; what follows the last break comes back.""" - odd = any(sign in body for sign in _ODD_BREAKS) - pieces = _LINE_BREAK.split(body) if odd else body.split(b"\n") - take(pieces[:-1]) - return pieces[-1] - - buffer = b"" + # The line in progress is line ``total``: its bytes, kept while it may be + # on the page, and whether it holds anything at all. + line: list[bytes] = [] + begun = False + + def wanted() -> bool: + return start <= total < stop + + def feed(body: bytes) -> None: + nonlocal total, line, begun + head, *rest = _split_lines(body) + if head: + begun = True + if wanted(): + line.append(head) + if not rest: + return + # The first break ends the line in progress, the pieces between + # breaks are whole lines, and the last piece begins the next line. + if wanted(): + page.append(b"".join(line)) + total += 1 + line = [] + *middle, last = rest + if middle: + if total < stop and total + len(middle) > start: + page.extend(middle[max(start - total, 0) : stop - total]) + total += len(middle) + begun = bool(last) + if last and wanted(): + line = [last] + + kept = b"" while chunk := handle.read(_PAGE_CHUNK): - buffer += chunk - hold = 2 if buffer.endswith(b"\xe2\x80") else (1 if buffer.endswith((b"\r", b"\xc2", b"\xe2")) else 0) - kept = buffer[len(buffer) - hold :] if hold else b"" - buffer = cut(buffer[: len(buffer) - hold] if hold else buffer) + kept - if buffer: - tail = cut(buffer) - if tail: - take([tail]) + body = kept + chunk + hold = 2 if body.endswith(b"\xe2\x80") else (1 if body.endswith((b"\r", b"\xc2", b"\xe2")) else 0) + kept = body[len(body) - hold :] if hold else b"" + feed(body[: len(body) - hold] if hold else body) + if kept: + feed(kept) + if begun: + if wanted(): + page.append(b"".join(line)) + total += 1 return page, total diff --git a/tests/test_filesystem_tools.py b/tests/test_filesystem_tools.py index 4736cb5c8..c26c1a66a 100644 --- a/tests/test_filesystem_tools.py +++ b/tests/test_filesystem_tools.py @@ -271,6 +271,53 @@ def test_a_page_is_the_lines_splitlines_gives_whatever_the_chunk_edge(monkeypatc assert filesystem._page_of_lines(BytesIO(b"\n"), 0, 5) == ([b""], 1) +@pytest.mark.parametrize("chunk", [1, 3, 8, 64]) +def test_a_line_longer_than_a_chunk_is_whole_on_its_page_and_only_counted_off_it(monkeypatch, chunk: int): + from io import BytesIO + + from raven.agent.tools import filesystem + + monkeypatch.setattr(filesystem, "_PAGE_CHUNK", chunk) + text = b"first\n" + b"L" * 50 + b"\r\nthird\n\n" + b"M" * 30 + want = text.decode("utf-8").splitlines() + assert len(want) == 5 + + for start, limit in ((0, 1), (0, 2), (1, 1), (1, 2), (2, 5), (4, 1), (5, 1)): + page, total = filesystem._page_of_lines(BytesIO(text), start, limit) + assert total == 5 and [line.decode("utf-8") for line in page] == want[start : start + limit], (start, limit) + + +def test_a_long_line_is_looked_at_once_whatever_its_length(monkeypatch): + """Each chunk is split once, never the whole of the line again: a long line costs its length, not its square. + + Held whole and split again with every chunk, a file with no break in it + took half a minute at 256 MiB where the whole-file read took under a + second (measured 2026-10-13). + """ + from io import BytesIO + + from raven.agent.tools import filesystem + + seen: list[int] = [] + split = filesystem._split_lines + monkeypatch.setattr(filesystem, "_split_lines", lambda body: (seen.append(len(body)), split(body))[1]) + monkeypatch.setattr(filesystem, "_PAGE_CHUNK", 16) + one_line = b"x" * 1000 + b"\xe2\x80" + b"y" * 500 # a held-back pair inside it, and no break at all + + page, total = filesystem._page_of_lines(BytesIO(one_line), 0, 5) + assert (page, total) == ([one_line], 1) + assert max(seen) <= 16 + 2 and sum(seen) == len(one_line), "every byte looked at once, plus a held-back pair" + + seen.clear() + assert filesystem._page_of_lines(BytesIO(one_line + b"\nnext"), 1, 5) == ([b"next"], 2) + assert max(seen) <= 16 + 2, "the line that is not on the page is counted, not kept" + + # A byte held back at the very end: a break that ends the last line, or part of it. + assert filesystem._page_of_lines(BytesIO(b"abc\r"), 0, 5) == ([b"abc"], 1) + assert filesystem._page_of_lines(BytesIO(b"abc\xe2"), 0, 5) == ([b"abc\xe2"], 1) + assert filesystem._page_of_lines(BytesIO(b"\r"), 0, 5) == ([b""], 1) + + @pytest.mark.asyncio async def test_read_file_pages_a_file_without_holding_all_of_it(tmp_path: Path, monkeypatch): from raven.agent.tools import filesystem From 81eb70777b50ecb25f0d381df4ebf0caebcb743f Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:47:07 +0800 Subject: [PATCH 26/44] fix(node): parked agents ride any new link, a cancel waits for delivery, a launch reserves its place A full review of the link's recovery and the daemon's state found five states they got wrong. Each is reproduced by a test of its own before the fix, and three of them on a real machine as well: a dead link replaced by a plain call left its agent parked for good; a cancel asked while the link was dead never arrived and the task ran on; eleven launches at once started nine against a limit of eight. A link the heartbeat had not yet seen die was replaced by the next call without its agents: the pool now attaches whatever waits for the machine on every link it makes, and the call hears why when that link goes too. A replay the new link could not carry escaped the reconnect with the frames already dropped and the endpoints after it in no list: a frame leaves the held list only once the link took it, the endpoint being replayed and every one not yet reached are parked again with what they hold, and the link is tried again. A cancel is wanted from the moment it is asked until the node takes it or says there is nothing to take; a link that dies under the request, or had died unnoticed, keeps it for the next link. The daemon's long-path socket is named for the install as well as the directory, as the socket under run/ is, so two installs on one long directory listen apart. A launch takes its place against the agent limit before its first await, so launches served together cannot pass the check as one. Co-authored-by: Claude (claude-fable-5-1) --- raven/node/client.py | 62 ++++++++++++++++++----- raven/node/server.py | 87 ++++++++++++++++++--------------- tests/test_node_client.py | 100 +++++++++++++++++++++++++++++++++++++- tests/test_node_server.py | 38 +++++++++++++++ 4 files changed, 234 insertions(+), 53 deletions(-) diff --git a/raven/node/client.py b/raven/node/client.py index 9041cf90e..d27e99487 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -31,7 +31,6 @@ import asyncio import base64 -import contextlib import hashlib import os import time @@ -272,6 +271,11 @@ async def end(self) -> None: Not while this process is stopping (:func:`raven.acp_client.remote.leave_agents_running`): then the agent stays with the machine's daemon, this end is only the endpoint's, and the run is picked up by the next process. + + Wanted until delivered: a link that dies under the request, or that + has died and not been said so yet, keeps the cancel for the next + link, and only the node's answer settles it -- the cancel taken, or + nothing there to cancel. """ if self.returncode is not None: return @@ -279,11 +283,19 @@ async def end(self) -> None: self._lost = f"left running on {self.describe()}: this computer stopped" self._settle() return - if self._detached is not None: - self._cancel_wanted = True + self._cancel_wanted = True + if self._detached is not None or not self._link.alive: return - with contextlib.suppress(AcpError): + try: await self._link.request(protocol.AGENT_CANCEL, {"channel": self.channel}, timeout=_CANCEL_TIMEOUT_S) + except AcpRemoteError: + # The node answered: no such channel or task there, so nothing is left to cancel. + self._cancel_wanted = False + except AcpError: + # The link is going (the pool parks this endpoint) or slow: the cancel goes with the next link. + return + else: + self._cancel_wanted = False async def wait(self) -> int | None: await self._done.wait() @@ -324,14 +336,19 @@ def detach(self, reason: str) -> None: self._detached = reason async def reattach(self, link: AcpClient, channel: int) -> None: - """The task is on ``channel`` of ``link`` again: send what waited, and the cancel if one was asked.""" + """The task is on ``channel`` of ``link`` again: send what waited, and the cancel if one was asked. + + A frame leaves the held list only once the link took it: a link that + goes under the replay leaves the rest held for the next one, and the + pool parks the endpoint again with them. + """ self._link, self.channel = link, channel self._detached = None - held, self._pending = self._pending, [] - for method, params in held: + while self._pending: + method, params = self._pending[0] await self._link.notify(method, {"channel": channel, **params}) + self._pending.pop(0) if self._cancel_wanted: - self._cancel_wanted = False await self.end() def _settle(self) -> None: @@ -630,6 +647,8 @@ async def _acquire(self, machine_id: str) -> _Node: if node is not None and node.client.alive: return node if node is not None: + # Gone before the heartbeat said so: its agents wait, as they + # would have for the heartbeat, and the new link brings them. await self._drop(machine_id, f"the Raven node on machine {node.target.label} stopped") try: node = await self._launch(machine_id) @@ -639,6 +658,14 @@ async def _acquire(self, machine_id: str) -> _Node: self._nodes[machine_id] = node self._errors.pop(machine_id, None) node.heartbeat = asyncio.create_task(self._heartbeat(machine_id, node), name=f"node-heartbeat-{machine_id}") + if self._parked.get(machine_id): + # Whichever path made this link, the agents parked for the + # machine ride on it; a link that goes under the attach is + # dropped again, and the call that asked for it hears why. + await self._reattach(machine_id) + if self._nodes.get(machine_id) is not node: + said = self._errors.get(machine_id, "").removesuffix("; reconnecting") + raise NodeError(said or f"the link to machine {node.target.label} was lost") return node def _prepare(self, machine_id: str) -> tuple[remote.Machine, Path, list[str], str]: @@ -789,7 +816,13 @@ async def _reconnect(self, machine_id: str) -> None: self._reconnects.pop(machine_id, None) async def _reattach(self, machine_id: str) -> None: - """Every endpoint parked for the machine onto a channel of its new link, from the frame it last read.""" + """Every endpoint parked for the machine onto a channel of its new link, from the frame it last read. + + A link that goes while this runs -- under the attach, or under the + replay of what an endpoint held -- parks that endpoint and every one + not yet reached again, with everything they still hold, and the link + is tried again: nothing that waited is dropped on the way. + """ node = self._nodes.get(machine_id) waiting = self._parked.pop(machine_id, []) if node is None or not node.client.alive: @@ -805,6 +838,8 @@ async def _reattach(self, machine_id: str) -> None: params = {"channel": channel, "task": endpoint.task, "after": endpoint.received} try: await node.client.request(protocol.AGENT_ATTACH, params, timeout=_ATTACH_TIMEOUT_S) + if endpoint.returncode is None: + await endpoint.reattach(node.client, channel) except AcpRemoteError as exc: node.channels.pop(channel, None) endpoint.lost( @@ -813,18 +848,19 @@ async def _reattach(self, machine_id: str) -> None: continue except (AcpTimeoutError, AcpConnectionError) as exc: # This link is gone too: this endpoint and every one not yet - # reached wait for the next one. + # reached wait for the next one, with what they still hold. node.channels.pop(channel, None) - self._parked.setdefault(machine_id, []).extend(e for e in waiting[i:] if e.returncode is None) said = ( self._ended(node.target, node.log, exc) if isinstance(exc, AcpConnectionError) else f"the Raven node on machine {node.target.label} did not answer {protocol.AGENT_ATTACH}" ) + rest = [e for e in waiting[i:] if e.returncode is None] + for e in rest: + e.detach(said) + self._parked.setdefault(machine_id, []).extend(rest) await self._lost(machine_id, node, said) return - if endpoint.returncode is None: - await endpoint.reattach(node.client, channel) logger.info("acp agent {!r}: attached again on {}", endpoint.agent, endpoint.describe()) def _stop_reconnect(self, machine_id: str) -> None: diff --git a/raven/node/server.py b/raven/node/server.py index 13f3f5d8a..98a59f373 100644 --- a/raven/node/server.py +++ b/raven/node/server.py @@ -551,15 +551,18 @@ def socket_path(root: Path) -> Path: """Where the daemon of this install listens for ``root``: under ``run/``, or when that path is too long for a socket, in a directory of this user's own in the temp directory. A unix socket's path is short by the system's rule, and a node directory - is wherever the owner put it. The fallback is named after ``root`` so a - relay and a daemon given the same directory meet, and it lies in a - directory only this user may open (:func:`own_dir`), as ``run/`` is, so - what listens there is this user's own daemon and nothing else. + is wherever the owner put it. The fallback is named after ``root`` and + this install, as the path under ``run/`` is, so a relay and a daemon of + one install given the same directory meet and another install's daemon + on that directory listens elsewhere; and it lies in a directory only this + user may open (:func:`own_dir`), as ``run/`` is, so what listens there is + this user's own daemon and nothing else. """ - under = root / RUN_DIR / f"{bundle.install_name()}.sock" + install = bundle.install_name() + under = root / RUN_DIR / f"{install}.sock" if len(os.fsencode(under)) <= _SOCKET_PATH_MAX: return under - tag = hashlib.sha256(os.fsencode(root.resolve())).hexdigest()[:16] + tag = hashlib.sha256(os.fsencode(root.resolve()) + b"\0" + install.encode("utf-8")).hexdigest()[:16] return Path(tempfile.gettempdir()) / f"raven-node-{os.getuid()}" / f"{tag}.sock" @@ -869,6 +872,8 @@ def __init__(self, root: Path, *, ephemeral: bool = False) -> None: self.root = root self.ephemeral = ephemeral self.tasks: dict[str, _Task] = {} + self._starting: set[str] = set() + """The task ids whose launch is under way: they take a place before their task is registered.""" self.connections: set[Node] = set() self._env: dict[str, str] | None = None self._env_lock = asyncio.Lock() @@ -894,41 +899,47 @@ def record_dir(self, task_id: str) -> Path: async def launch(self, node: Node, channel: int, params: dict[str, Any]) -> _Task: task_id = _task_id(params) known = self.tasks.get(task_id) - if known is not None and known.rc is None: + if (known is not None and known.rc is None) or task_id in self._starting: raise _RefusedError(_INVALID_PARAMS, f"task {task_id!r} is already running; attach to it") - if len(self.running()) >= protocol.MAX_AGENTS: + if len(self.running()) + len(self._starting) >= protocol.MAX_AGENTS: raise _RefusedError(protocol.BUSY, f"{protocol.MAX_AGENTS} agents are already running on this machine") - argv = _argv(params) - env = {**await self.env(), **_env(params)} - found = shutil.which(argv[0], path=env.get("PATH")) - if found is None: - raise _RefusedError(protocol.NOT_FOUND, f"{argv[0]!r} is not on this machine's PATH") - cwd = Path(_text(params, "cwd")).expanduser() - record = self.record_dir(task_id) - try: - cwd.mkdir(parents=True, exist_ok=True) - if record.exists(): - shutil.rmtree(record) - record.mkdir(mode=0o700, parents=True) - except OSError as exc: - raise _RefusedError(_INTERNAL_ERROR, f"could not make {cwd}: {exc.strerror or exc}") from None + # The place is taken before the first await: requests are served + # together, and a launch still preparing has not registered its task. + self._starting.add(task_id) try: - proc = await asyncio.create_subprocess_exec( - found, - *argv[1:], - cwd=str(cwd), - env=env, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - start_new_session=True, - limit=_AGENT_LINE_LIMIT, - ) - except OSError as exc: - raise _RefusedError(_INTERNAL_ERROR, f"could not start {argv[0]!r}: {exc.strerror or exc}") from None - task = _Task(task_id, proc, self, record, command=shlex.join(argv), cwd=str(cwd)) - task.subscriber = (node, channel) - self.tasks[task_id] = task + argv = _argv(params) + env = {**await self.env(), **_env(params)} + found = shutil.which(argv[0], path=env.get("PATH")) + if found is None: + raise _RefusedError(protocol.NOT_FOUND, f"{argv[0]!r} is not on this machine's PATH") + cwd = Path(_text(params, "cwd")).expanduser() + record = self.record_dir(task_id) + try: + cwd.mkdir(parents=True, exist_ok=True) + if record.exists(): + shutil.rmtree(record) + record.mkdir(mode=0o700, parents=True) + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not make {cwd}: {exc.strerror or exc}") from None + try: + proc = await asyncio.create_subprocess_exec( + found, + *argv[1:], + cwd=str(cwd), + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + limit=_AGENT_LINE_LIMIT, + ) + except OSError as exc: + raise _RefusedError(_INTERNAL_ERROR, f"could not start {argv[0]!r}: {exc.strerror or exc}") from None + task = _Task(task_id, proc, self, record, command=shlex.join(argv), cwd=str(cwd)) + task.subscriber = (node, channel) + self.tasks[task_id] = task + finally: + self._starting.discard(task_id) self.touch() return task diff --git a/tests/test_node_client.py b/tests/test_node_client.py index 2d19f3d23..3ce34cdd5 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -751,8 +751,8 @@ def _box(machine_id: str = "box") -> remote.Machine: return remote.Machine(id=machine_id, display_name="Lab box", _row=row(id=machine_id)) -def _live(pool: NodePool, tmp_path: Path, machine_id: str = "box", **kw: Any) -> _Client: - client = _Client(**kw) +def _live(pool: NodePool, tmp_path: Path, machine_id: str = "box", client: _Client | None = None, **kw: Any) -> _Client: + client = client or _Client(**kw) log = tmp_path / f"{machine_id}.log" log.write_text(f"ssh: connect to host {HOST} port {PORT}: Connection refused\n", encoding="utf-8") hello = {"protocol": protocol.PROTOCOL, "home": "/home/worker", "version": "0.9", "limits": {"agents": 8}} @@ -1204,3 +1204,99 @@ def bad(n: int) -> None: logger.remove(sink) assert heard == [1] and any("a watcher of task" in message and "no disk" in message for message in records) + + +# --- a link replaced by an ordinary call brings its agents along ------------------------ + + +async def test_a_call_that_replaces_a_dead_link_attaches_the_parked_agents_again(tmp_path, monkeypatch): + """The transport died before the heartbeat saw it: the next call relaunches the link, and the agents come with it.""" + pool = NodePool(heartbeat_s=60.0) + client = _live(pool, tmp_path) + endpoint = ChannelEndpoint(client, 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + pool._nodes["box"].channels[1] = endpoint + again = _Client() + + async def launch(machine_id: str) -> _Node: + return _Node(again, _box(), tmp_path / "box.log", {"home": "/h"}, {}) # type: ignore[arg-type] + + monkeypatch.setattr(pool, "_launch", launch) + client.alive = False + + assert await pool.call("box", protocol.PING, {}) == {"text": "ok"} + + assert pool._parked.get("box", []) == [] and endpoint.detached is None + assert (protocol.AGENT_ATTACH, {"channel": 1, "task": endpoint.task, "after": 0}) in [a[:2] for a in again.asked] + assert pool._nodes["box"].client is again and pool._nodes["box"].channels[1] is endpoint + await pool.close_all() + + +class _Gone(_Client): + """A connection that answers requests but can carry no notification: it died right after the attach.""" + + async def notify(self, method: str, params: dict[str, Any]) -> None: + self.said.append((method, params)) + raise AcpConnectionError("the ssh ended", returncode=255, stderr="") + + +async def test_a_replay_the_link_cannot_carry_keeps_the_frames_and_every_waiting_endpoint(tmp_path): + pool = NodePool(backoff_s=()) + client = _live(pool, tmp_path, client=_Gone()) + first = ChannelEndpoint(client, 1, target=_box(), agent="a@box") # type: ignore[arg-type] + second = ChannelEndpoint(client, 2, target=_box(), agent="b@box") # type: ignore[arg-type] + for endpoint in (first, second): + endpoint.detach("the link went") + await first.write(b"one\n") + await first.write(b"two\n") + pool._parked["box"] = [first, second] + + await pool._reattach("box") + + assert [params["line"] for _, params in first._pending] == ["one", "two"], ( + "nothing the link did not carry is dropped" + ) + assert pool._parked["box"] == [first, second], "the one being replayed and the one not yet reached both wait" + assert first.detached and second.detached and first.returncode is None and second.returncode is None + assert not client.alive and "box" not in pool._nodes + assert len(client.said) == 1, "the second frame was not tried on a link that had just failed" + assert "box" in pool._reconnects, "and the link is tried again" + assert pool.status("box").state == "lost" and pool.status("box").agents == 2 + await pool.close_all() + + +async def test_a_cancel_the_link_cannot_carry_is_kept_for_the_next_link(tmp_path): + dead = _Client(raises=AcpConnectionError("the ssh ended", returncode=255, stderr="")) + endpoint = ChannelEndpoint(dead, 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + + await endpoint.end() # the link died; the heartbeat has not said so yet + + assert endpoint._cancel_wanted and endpoint.returncode is None + again = _Client() + await endpoint.reattach(again, 3) + assert (protocol.AGENT_CANCEL, {"channel": 3}, node_client._CANCEL_TIMEOUT_S) in again.asked + assert not endpoint._cancel_wanted, "delivered" + + +async def test_a_cancel_that_fails_during_the_reattach_is_still_wanted_after_it(tmp_path): + endpoint = ChannelEndpoint(_Client(), 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + endpoint.detach("the link went") + await endpoint.end() + assert endpoint._cancel_wanted + dying = _Client(raises=AcpConnectionError("the ssh ended", returncode=255, stderr="")) + + await endpoint.reattach(dying, 2) # the cancel's request fails on the new link too + + assert endpoint._cancel_wanted, "not delivered: still wanted" + again = _Client() + await endpoint.reattach(again, 4) + assert (protocol.AGENT_CANCEL, {"channel": 4}, node_client._CANCEL_TIMEOUT_S) in again.asked + assert not endpoint._cancel_wanted + + +async def test_a_cancel_the_node_refuses_is_not_wanted_any_more(tmp_path): + refusing = _Client(raises=AcpRemoteError(protocol.AGENT_CANCEL, protocol.UNKNOWN_TASK, "no agent on channel 1")) + endpoint = ChannelEndpoint(refusing, 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + + await endpoint.end() + + assert not endpoint._cancel_wanted, "the node answered: there is nothing there to cancel" diff --git a/tests/test_node_server.py b/tests/test_node_server.py index b2cdd9cb7..152132472 100644 --- a/tests/test_node_server.py +++ b/tests/test_node_server.py @@ -1397,3 +1397,41 @@ def test_serve_and_attach_need_the_node_directory(argv, capsys): with pytest.raises(SystemExit): node_main.main(argv) assert "--dir" in capsys.readouterr().err + + +def test_the_long_path_socket_is_this_install_s_as_the_short_one_is(tmp_path, monkeypatch): + monkeypatch.setattr(server.tempfile, "gettempdir", lambda: str(tmp_path / "tmp")) + root = tmp_path / ("n" * 120) + monkeypatch.setattr(bundle, "install_name", lambda: "0.9.0-aaaaaaaaaaaa") + one = server.socket_path(root) + monkeypatch.setattr(bundle, "install_name", lambda: "0.9.0-bbbbbbbbbbbb") + other = server.socket_path(root) + + assert one.parent == other.parent and one != other, "two installs on one long directory listen apart" + assert server.socket_path(root) == other, "and a relay of the same install finds its daemon" + + +@pytest.mark.slow +async def test_a_launch_in_progress_counts_toward_the_limit(tmp_path, agent, monkeypatch): + """Two launches that pass the check before either has registered its task: only one may spawn.""" + host = _Host() + monkeypatch.setattr(protocol, "MAX_AGENTS", 1) + hold = threading.Event() + monkeypatch.setattr(server, "capture_login_env", lambda: (hold.wait(10), dict(os.environ))[1]) + + async def launch(channel: int) -> dict[str, Any]: + params = {"channel": channel, "task": f"t{channel}", "command": agent, "cwd": str(tmp_path)} + return await host.ask(protocol.AGENT_LAUNCH, params, request_id=channel) + + first = asyncio.create_task(launch(1)) + await asyncio.sleep(0.05) # past the check, waiting for the login environment + second = asyncio.create_task(launch(2)) + await asyncio.sleep(0.05) + hold.set() + one, two = await asyncio.gather(first, second) + + assert "result" in one, one + assert two["error"] == {"code": protocol.BUSY, "message": "1 agents are already running on this machine"} + assert len(host.node.daemon.running()) == 1 + await host.ask(protocol.AGENT_CANCEL, {"channel": 1}, request_id=3) + await host.until(lambda: host.node.daemon.running() == []) From bd07fdd6145b53d7e7773d5f2d36ce9f8624c3a3 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:23:15 +0800 Subject: [PATCH 27/44] fix(node): a relaunch that fails under a call is tried again on the backoff for the agents that wait The previous fix had a link replaced by an ordinary call bring the parked agents along, but only when the relaunch succeeded. When it failed -- the machine unreachable at that moment -- the call raised and nothing tried again: the agents stayed parked with no reconnect task, where the same failure under the heartbeat would have been retried on the backoff. On a real machine, the relay killed and the relaunch made to fail once, the agent was still parked thirty seconds after the machine was reachable again. A failed relaunch with agents waiting now schedules the same reconnect the heartbeat loss does; the agent is attached again on the first retry once the machine answers. Co-authored-by: Claude (claude-fable-5-1) --- raven/node/client.py | 10 ++++++++++ tests/test_node_client.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/raven/node/client.py b/raven/node/client.py index d27e99487..7f0cf97c9 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -654,6 +654,12 @@ async def _acquire(self, machine_id: str) -> _Node: node = await self._launch(machine_id) except NodeError as exc: self._errors[machine_id] = str(exc) + if self._parked.get(machine_id): + # Agents wait for this machine: the link is tried again + # on the backoff, as after a heartbeat loss, and not only + # by whatever call comes next. + self._errors[machine_id] = f"{exc}; reconnecting" + self._retry_later(machine_id) raise self._nodes[machine_id] = node self._errors.pop(machine_id, None) @@ -781,6 +787,10 @@ async def _lost(self, machine_id: str, node: _Node, said: str) -> None: logger.warning("raven node on machine {}: link lost: {}", node.target.id, said) self._errors[machine_id] = f"{said}; reconnecting" await self._drop(machine_id, said) + self._retry_later(machine_id) + + def _retry_later(self, machine_id: str) -> None: + """Try the link again on the backoff, unless a try is under way already or the pool is closing.""" if not self._closing and machine_id not in self._reconnects: self._reconnects[machine_id] = asyncio.create_task( self._reconnect(machine_id), name=f"node-reconnect-{machine_id}" diff --git a/tests/test_node_client.py b/tests/test_node_client.py index 3ce34cdd5..a373607a8 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -1231,6 +1231,37 @@ async def launch(machine_id: str) -> _Node: await pool.close_all() +async def test_a_relaunch_that_fails_under_a_call_still_reconnects_for_the_parked_agents(tmp_path, monkeypatch): + """The link died unnoticed and the call's own relaunch failed: the agents wait on the backoff, as after a heartbeat loss.""" + pool = NodePool(heartbeat_s=60.0, backoff_s=(0.01, 0.01)) + client = _live(pool, tmp_path) + endpoint = ChannelEndpoint(client, 1, target=_box(), agent="echo@box") # type: ignore[arg-type] + pool._nodes["box"].channels[1] = endpoint + tried: list[str] = [] + again = _Client() + + async def launch(machine_id: str) -> _Node: + tried.append(machine_id) + if len(tried) == 1: + raise NodeError("machine 'Lab box' (box) could not be reached") + return _Node(again, _box(), tmp_path / "box.log", {"home": "/h"}, {}) # type: ignore[arg-type] + + monkeypatch.setattr(pool, "_launch", launch) + client.alive = False + + with pytest.raises(NodeError, match="could not be reached"): + await pool.call("box", protocol.PING, {}) + assert pool.waiting("box") == 1 and pool.status("box").state == "lost" + assert "box" in pool._reconnects, ( + "an agent waits: the link is tried again on the backoff, not only by the next call" + ) + + await _until(lambda: endpoint.detached is None and len(tried) == 2) + assert (protocol.AGENT_ATTACH, {"channel": 1, "task": endpoint.task, "after": 0}) in [a[:2] for a in again.asked] + assert pool.status("box").state == "connected" + await pool.close_all() + + class _Gone(_Client): """A connection that answers requests but can carry no notification: it died right after the attach.""" From 283d79024c13abadd86bbd7741696499e5f86600 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:52:16 +0800 Subject: [PATCH 28/44] fix(*): the written rule for a machine's address matches the code: scp-shaped in paths, no key The machine docstring, two comments in remote.py and two sentences of the CONTEXT entry still promised that a registered machine's address reaches no model, while a file on the machine is named to a model the way scp writes it -- user, address and port -- and that is the rule for this lane: what never shows is the key and the ssh line built from the row. The five sites now say so, with the format, and the acceptance test has its positive half: where an address appears it is scp-shaped and the registry's, never the machine's id. Two sentences sent a reader to file tools this change does not give a remote path to; they name exec alone now. Two comments still said the machine lane was acp-only. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 13 ++++++++----- raven/acp_client/remote.py | 17 +++++++++++++---- raven/agent/subagent/backends/__init__.py | 15 ++++++++------- raven/agent/subagent/probe_state.py | 2 +- raven/agent/subagent/registry.py | 3 +-- raven/config/schema.py | 6 +++--- raven/rpc/methods/subagents.py | 2 +- tests/test_acp_remote.py | 18 ++++++++++++++++++ tests/test_subagent_acp_remote.py | 2 ++ tests/test_subagent_cli_remote.py | 2 ++ 10 files changed, 57 insertions(+), 23 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index ae1ec0883..c3d50a2b8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -388,15 +388,18 @@ machine's **Raven node**, reached over the one ssh link the host keeps to the ma an `openai` entry's endpoint is reached from that machine through an ssh tunnel the host opens for the turn and closes after it (`remote.tunnel`); a `builtin` entry cannot name one -- Raven's own loop runs here -- and is loaded with a warning and runs here. -The row's address, port, key and user are read for that link or tunnel alone -(`raven/acp_client/remote.py`). +The row's key opens that link or tunnel and goes nowhere else, and the ssh line built +from the row stays in memory; the row's user, address and port open it too and also name +every path on the machine that a model, a reply, a record or an RPC result carries, written +the way scp writes one -- `user@host:/abs`, or `scp://user@host:port//abs` off port 22 +(`remote.shown`, `raven/acp_client/remote.py`) -- and never by the machine's id. Its dispatch reads the registry once (`remote.machine`), and the run fails before anything starts when the id is not registered, is listed twice, cannot be used, or is this computer; the `spawn` itself has already answered that it started. No other dispatch reads the registry, so no graph and no spawn is ever refused over the state -of it. The address stays in the ssh line, in memory; the entry names only the -machine's id, and the entry's `env` reaches the agent on the link, never on a -command line. Such an entry is added the way any preset is, with `machine` (and +of it. The entry names only the machine's id, and the entry's `env` reaches the +agent on the link, never on a command line. Such an entry is added the way any +preset is, with `machine` (and `remote_cwd`, for an `acp` or `cli` one) on `subagents.add` — from a chat through `raven_config`, which lists the registered machines while the lane is on; the page's add passes no machine yet — refused before anything is spent when the lane is off, diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index 329d46ced..8cdca091f 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -79,7 +79,10 @@ #: What a POSIX shell exits with when it cannot find the command it was given. NOT_FOUND_RC = 127 -# The fields of a registry row that are the way onto the machine, never shown. +# The fields of a registry row that are the way onto the machine: read to open +# the ssh, and kept off the launch line a model, a log or a record could see. +# The user, address and port do name a path there, written the way scp writes +# it (:func:`shown`); the key never shows anywhere. _WAY_IN = frozenset({"host", "port", "user", "key"}) _LEAF_UNSAFE = re.compile(r"[^A-Za-z0-9._-]") _LEAF_MAX = 64 @@ -96,7 +99,12 @@ class RemoteMachineError(RuntimeError): @dataclass(frozen=True) class Machine: - """A registered ssh machine. The row, with the address in it, never prints.""" + """A registered ssh machine. + + The row never prints whole: the key and the ssh line built from the row + reach no model, log or record, while its user, address and port name a + path on the machine the way scp writes it (:func:`shown`). + """ id: str display_name: str @@ -114,8 +122,9 @@ def field(self, name: str) -> Any: For what a caller needs to know about the machine itself -- where a Raven node is kept on it (``node_dir``), the directories its owner - listed (``paths``). The address, port, account and key stay behind - :func:`ssh_command`, :func:`run` and :func:`push`. + listed (``paths``). The address, port, account and key open the way + there behind :func:`ssh_command`, :func:`run` and :func:`push`; the + first three also name a path there (:func:`shown`), the key never shows. """ if name in _WAY_IN: raise KeyError(f"{name!r} is how raven reaches the machine, not a fact about it") diff --git a/raven/agent/subagent/backends/__init__.py b/raven/agent/subagent/backends/__init__.py index 50a29d69b..4db45d3d9 100644 --- a/raven/agent/subagent/backends/__init__.py +++ b/raven/agent/subagent/backends/__init__.py @@ -98,13 +98,14 @@ class AgentMeta(NamedTuple): """The registered machine this agent runs on, by id, or ``""`` for this computer. Rendered as a tag so the model can send work to a named machine and knows - the files it gets back are there, not here. The id only, which is all the - roster says of the machine: the way onto it -- the address, port, account - and key the registry holds -- stays out of anything a model reads, while a - path on it is written to a model the way scp writes it, address included, - because that is how a file there is named (``remote.shown``). Read off - every kind: whichever kind runs there, the tag is the same. Appended last, - for the reason ``model_choices`` gives.""" + the files it gets back are there, not here. The tag carries the id only. + A file on the machine is named to a model the way scp writes it, with the + registry's user, address and port -- ``user@host:/abs``, or + ``scp://user@host:port//abs`` off port 22 (``remote.shown``) -- and never + by the id; what a model, a reply, a record and an RPC result never carry + is the key and the assembled ssh line. Read off every kind: whichever kind + runs there, the tag is the same. Appended last, for the reason + ``model_choices`` gives.""" def agent_meta(cfg: Any, *, snapshot: Any = None) -> AgentMeta: diff --git a/raven/agent/subagent/probe_state.py b/raven/agent/subagent/probe_state.py index de3d62b33..bef20595e 100644 --- a/raven/agent/subagent/probe_state.py +++ b/raven/agent/subagent/probe_state.py @@ -197,7 +197,7 @@ def fingerprint(cfg: Any) -> str: # set, so every verdict recorded before the field existed holds. if getattr(cfg, "lend_keys", None): payload["lend_keys"] = list(cfg.lend_keys) - # The machine an acp row runs on, by the same only-when-set rule. + # The machine a row of any kind runs on, by the same only-when-set rule. for field in ("machine", "remote_cwd"): if getattr(cfg, field, None): payload[field] = getattr(cfg, field) diff --git a/raven/agent/subagent/registry.py b/raven/agent/subagent/registry.py index 0ccbce63d..f78937398 100644 --- a/raven/agent/subagent/registry.py +++ b/raven/agent/subagent/registry.py @@ -71,8 +71,7 @@ class AgentCaps: A menu rather than a capability, so unlike the three above it is not rendered as a roster tag -- see ``AgentMeta.modes``.""" machine: str = "" - """The registered machine the agent runs on, or ``""`` for this computer - (acp only) -- see ``AgentMeta.machine``.""" + """The registered machine the agent runs on, or ``""`` for this computer -- see ``AgentMeta.machine``.""" @dataclass(frozen=True) diff --git a/raven/config/schema.py b/raven/config/schema.py index 43cc2c8c8..ed59dd160 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -2310,8 +2310,8 @@ class BuiltinAgentConfig(Base): A built-in agent is Raven's own loop in this process, where its model keys and tools are, so it has no process to start on another machine; a value here is dropped on load with a warning and the row runs on this computer, - reaching a machine the way the main loop does (``exec`` with a ``machine``, - paths written by address). Undeclared, the key would be dropped without a + reaching a machine the way the main loop does (``exec`` with a + ``machine``). Undeclared, the key would be dropped without a word, which is how a row ends up running here while its author believes it runs there.""" @@ -2322,7 +2322,7 @@ def _runs_here(self) -> "BuiltinAgentConfig": logger.warning( "built-in sub-agent {!r} names machine {!r}, which it cannot run on: Raven's own loop runs in " "this process, where its model keys and tools are. The entry runs on this computer and reaches " - "the machine as the main loop does (exec with a machine, paths by address); remove `machine` " + "the machine as the main loop does (exec with a machine); remove `machine` " "from it", self.name, machine, diff --git a/raven/rpc/methods/subagents.py b/raven/rpc/methods/subagents.py index 21b440f25..1b4551615 100644 --- a/raven/rpc/methods/subagents.py +++ b/raven/rpc/methods/subagents.py @@ -544,7 +544,7 @@ def _refuse_unless_it_can_run_there(entry: dict, machine: str, *, remote_cwd: st if entry.get("kind") == "builtin": raise ConfigValidationError( f"preset {shown!r} is a built-in agent: Raven's own loop in this process, which runs on no other " - "machine; reach the machine from it with exec and the file tools instead", + "machine; reach the machine from it with exec instead", data={"field": "machine", "machine": machine}, ) if entry.get("kind") == "openai" and remote_cwd: diff --git a/tests/test_acp_remote.py b/tests/test_acp_remote.py index 5fe0894ff..3842f606e 100644 --- a/tests/test_acp_remote.py +++ b/tests/test_acp_remote.py @@ -9,6 +9,7 @@ import json import os +import re import shlex import subprocess import sys @@ -547,6 +548,23 @@ def test_a_path_on_the_machine_is_shown_the_way_scp_writes_it(): ) +_SCP_ADDRESS = re.compile(r"^(?:scp://[^@/\s]+@[^:/\s]+:\d+//\S+|[^@/\s]+@[^:/\s]+:/\S+)$") + + +def test_an_address_a_model_reads_is_scp_shaped_by_address_and_never_by_id(): + """The positive half of the rule: where an address appears it is scp-shaped and the registry's, never the id.""" + far = _box() + near = Machine(id="box", display_name="Lab box", _row={**far._row, "port": 22}) + for target, want in ( + (far, f"scp://worker@{HOST}:{PORT}//home/worker/raven-work/t1"), + (near, f"worker@{HOST}:/home/worker/raven-work/t1"), + ): + shown = remote.shown(target, "/home/worker/raven-work/t1") + assert shown == want and _SCP_ADDRESS.match(shown) + assert shown.split("@", 1)[1].split(":", 1)[0] == HOST, "the registry's address, not the machine's id" + assert KEY not in shown and "ssh" not in shown + + def test_the_files_note_names_what_came_with_the_task_by_address_and_nothing_else(): from types import SimpleNamespace diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 2e36c34fa..2754c3bd2 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -190,6 +190,8 @@ async def test_a_dispatch_runs_on_the_machine_in_a_directory_made_there_and_says assert "[raven] Ran on machine 'Lab box' (box), in " in reply # Shown the way scp writes it, which the file tools read back as it is. assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in reply + where = reply.split("in scp://", 1)[1].split(";", 1)[0] + assert where.split("@", 1)[1].split(":", 1)[0] == HOST, "the address a model reads is the registry's, never the id" assert "not on this computer" in reply assert "the files it wrote are there, not on this computer" in reply assert "read_file, list_dir, find, grep, write_file and edit_file take that path as it is" in reply diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py index 291c9d369..c20b9cd9d 100644 --- a/tests/test_subagent_cli_remote.py +++ b/tests/test_subagent_cli_remote.py @@ -173,6 +173,8 @@ async def test_a_cli_dispatch_runs_on_the_machine_in_a_directory_made_there_and_ assert Path(head.strip()).resolve() == there.resolve(), "the command ran in it" assert note.startswith("Ran on machine 'Lab box' (box), in ") assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in note + where = note.split("in scp://", 1)[1].split(";", 1)[0] + assert where.split("@", 1)[1].split(":", 1)[0] == HOST, "the address a model reads is the registry's, never the id" assert "the files it wrote are there, not on this computer" in note assert not local.exists(), "the caller's workspace is never touched" for argv in box.calls(): From e85dd027c0dfcea11b0278c69280907f4dbc83b2 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:46:00 +0800 Subject: [PATCH 29/44] fix(agent): a fetch one byte over the budget is refused in bytes, not as the budget's own figure A file just over deliver_files' budget rounds to the budget's own figure, so the refusal read "20.0 MB is over the 20.0 MB limit" and said nothing (a 20 MiB + 1 byte file on a registered machine; 20 MiB itself is fetched whole). When the two round alike, the sentence gives the bytes of each. Co-authored-by: Claude (claude-fable-5-1) --- raven/agent/tools/deliver.py | 17 +++++++++++++---- tests/test_deliver_files_tool.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/raven/agent/tools/deliver.py b/raven/agent/tools/deliver.py index e8db466ff..4db2a2544 100644 --- a/raven/agent/tools/deliver.py +++ b/raven/agent/tools/deliver.py @@ -97,10 +97,7 @@ async def _brought_back(where: RemotePath, workspace: Path | None) -> tuple[Path return None, f"not found or not a regular file on machine {label}" size = int(info.get("size") or 0) if size > PULL_FILE_BUDGET: - return None, ( - f"{_human_size(size)} is over the {_human_size(PULL_FILE_BUDGET)} " - f"limit for a file brought back from machine {label}; read it there instead" - ) + return None, f"{_over_budget(size)} for a file brought back from machine {label}; read it there instead" try: local = _landing(workspace, where.machine, place) except ValueError: @@ -161,6 +158,18 @@ def _landing(workspace: Path | None, machine: str, place: str) -> Path: return local +def _over_budget(size: int) -> str: + """``size`` against :data:`PULL_FILE_BUDGET`, said so the two never read as one number. + + A file one byte over the budget rounds to the budget's own figure, and + "20.0 MB is over the 20.0 MB limit" says nothing; the bytes do. + """ + shown, cap = _human_size(size), _human_size(PULL_FILE_BUDGET) + if shown == cap: + return f"{size:,} bytes is over the {PULL_FILE_BUDGET:,}-byte ({cap}) limit" + return f"{shown} is over the {cap} limit" + + def _human_size(size: int) -> str: value = float(size) for unit in ("B", "KB", "MB", "GB"): diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index f15f4bfbd..7e17ead28 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -551,3 +551,24 @@ async def test_what_was_not_delivered_says_why(tool, machines) -> None: outside.count("203.0.113.7") == 1 and "scp://worker@203.0.113.7:58717//root/.ssh/id_rsa is outside" in outside ) assert "203.0.113.7" not in big + nobody + folder + + +async def test_a_file_one_byte_over_the_budget_is_refused_in_bytes_not_in_a_size_that_reads_as_the_limit( + tool, machines +) -> None: + """20 MiB goes; 20 MiB + 1 is refused, and the sentence does not read "20.0 MB is over the 20.0 MB limit".""" + from raven.agent.tools.deliver import PULL_FILE_BUDGET + + machines.size = PULL_FILE_BUDGET + exact = await tool.execute(files=[{"path": "box:raven-work/t1/exact.bin"}]) + tool.take_metadata() + machines.size = PULL_FILE_BUDGET + 1 + over = await tool.execute(files=[{"path": "box:raven-work/t1/over.bin"}]) + + assert exact.startswith("Delivered 1 file(s): exact.bin (20.0 MB)") + assert ( + "20,971,521 bytes is over the 20,971,520-byte (20.0 MB) limit for a file brought back from machine " + "'Lab box' (box); read it there instead" in over + ) + assert "20.0 MB is over the 20.0 MB" not in over + assert [where for where, _, _ in machines.pulled] == ["/root/raven-work/t1/exact.bin"] From 96b08771e5e223fa5593177ed20bf61085dd2b9a Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:46:01 +0800 Subject: [PATCH 30/44] fix(*): a run is picked up by one process, and a stop under the pick-up leaves it for the next start Two processes of one Raven home starting together -- a gateway and a page stack -- each picked the same run up: the machine's daemon handed the task to whichever attached last, and the other waited out a prompt that would never answer it, then announced to the conversation a failure the run did not have (on a registered machine: one process announced the reply after 23 s, the other an error after 40 s). A pick-up is made under a claim beside the record now, a file naming the process that holds it, made whole and linked into place so two processes cannot both make it; a claim left by a process that is gone is taken over. A stop of this computer that landed under a pick-up closed the adopted connection, which the pick-up read as the agent having ended: the run was announced as an error, its record removed, and the agent ran on unreported (the task still running on the machine, the next start finding nothing). While this computer is stopping, a pick-up keeps its record for the next start, and no further pick-up is begun. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/remote_resume.py | 14 ++++- raven/acp_client/remote_runs.py | 83 ++++++++++++++++++++++++++- raven/agent/subagent/resume_remote.py | 18 +++++- tests/test_remote_resume.py | 16 ++++++ tests/test_remote_runs.py | 42 ++++++++++++++ tests/test_resume_remote.py | 49 ++++++++++++++++ tests/test_subagent_acp_remote.py | 53 +++++++++++++++++ 7 files changed, 268 insertions(+), 7 deletions(-) diff --git a/raven/acp_client/remote_resume.py b/raven/acp_client/remote_resume.py index 8a9f6e05a..fc0847296 100644 --- a/raven/acp_client/remote_resume.py +++ b/raven/acp_client/remote_resume.py @@ -52,9 +52,10 @@ def error(self, said: str) -> None: async def pick_up(run: RemoteRun, backend: AcpAgentBackend, *, pool: Any = None) -> PickedUp: """The reply the turn in ``run`` owes, read off the machine; or why there is none. - Raises :class:`UnreachableError` when the machine cannot be asked at all, so - the caller keeps the record for a later start; every other outcome is a - :class:`PickedUp`, and the record is the caller's to remove. + Raises :class:`UnreachableError` when the machine cannot be asked at all, + or when this computer stops under the pick-up, so the caller keeps the + record for a later start; every other outcome is a :class:`PickedUp`, + and the record is the caller's to remove. """ link = pool if pool is not None else remote_path.nodes() if link is None: @@ -107,6 +108,13 @@ async def pick_up(run: RemoteRun, backend: AcpAgentBackend, *, pool: Any = None) timeout=_prompt_deadline(backend.timeout), ) except AcpConnectionError as exc: + if remote.leaving(): + # This computer is stopping under the pick-up: the agent + # stays with its machine, and the run is the next start's. + raise UnreachableError( + f"this computer stopped while the run of {run.agent!r} on machine {target.label} was being " + "picked up; it waits for the next start" + ) from None rc = attached.get("rc") ended = f" (exit {rc})" if isinstance(rc, int) else "" return PickedUp( diff --git a/raven/acp_client/remote_runs.py b/raven/acp_client/remote_runs.py index 3fa846231..bc4912cc3 100644 --- a/raven/acp_client/remote_runs.py +++ b/raven/acp_client/remote_runs.py @@ -11,7 +11,9 @@ ``initialize`` on the connection. One file per run under ``/machines/runs/.json``, owner-only, written when the turn starts, kept current as frames arrive, and removed when the turn ends here. A -file left behind is a run to pick up (:mod:`raven.acp_client.remote_resume`). +file left behind is a run to pick up (:mod:`raven.acp_client.remote_resume`), +by one process at a time: the pick-up is made under a claim beside the record +(:func:`claim`), since two processes of one Raven home can start together. No address is in it: the machine is named by its registry id. """ @@ -28,6 +30,7 @@ from loguru import logger from raven.home import raven_home +from raven.utils.pid import pid_alive #: Where the records live under the raven home. RUNS_DIR = ("machines", "runs") @@ -106,9 +109,82 @@ def record(run: RemoteRun) -> Path: def forget(task: str) -> None: - """Remove the record of ``task``: the turn ended here.""" + """Remove the record of ``task``, and the claim on it: the turn ended here, or was concluded here.""" with contextlib.suppress(FileNotFoundError, OSError): path_of(task).unlink() + release(task) + + +def claim_path(task: str) -> Path: + """Beside the record: the file naming the process that is picking ``task`` up.""" + return runs_dir() / f"{task}.json.claim" + + +def claim(task: str) -> bool: + """Take the pick-up of ``task`` for this process; ``False`` while a live process of this computer holds it. + + Two processes of one Raven home can start together -- a gateway and a + page stack -- and each would pick the same run up: the machine's daemon + hands the task to whichever attaches last, and the other waits out a + prompt that will never answer it, then reports to the conversation a + failure the run did not have. So a run is picked up under a claim: a + file beside its record naming the process that holds it, made whole and + linked into place, so two processes cannot both make it. A claim whose + process is gone is taken over. :func:`forget` removes it with the + record; :func:`release` alone, when the run is left for a later start. + """ + path = claim_path(task) + for _ in range(2): + if _take(path): + return True + holder = _holder(path) + if holder is not None and pid_alive(holder): + return False + with contextlib.suppress(OSError): + path.unlink() + return False + + +def release(task: str) -> None: + """Give the claim on ``task`` up: the run waits for a later start.""" + with contextlib.suppress(FileNotFoundError, OSError): + claim_path(task).unlink() + + +def _take(path: Path) -> bool: + """``path`` made as this process's claim, whole, and only where nothing is: ``True`` when it is ours.""" + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + try: + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(str(os.getpid())) + try: + os.link(tmp, path) + except FileExistsError: + return False + except OSError: + # No hard links here: made in place, with a moment in which it names nobody yet. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(str(os.getpid())) + return True + except FileExistsError: + return False + except OSError as exc: + logger.warning("remote run record {}: a claim could not be made: {}", path.name, exc) + return False + finally: + with contextlib.suppress(OSError): + tmp.unlink() + + +def _holder(path: Path) -> int | None: + """The process a claim names, or ``None`` for one that names nobody.""" + try: + text = path.read_text(encoding="utf-8").strip() + except OSError: + return None + return int(text) if text.isdigit() else None def pending() -> list[RemoteRun]: @@ -187,11 +263,14 @@ def was_left(task_id: str) -> bool: "RUNS_DIR", "RemoteRun", "Tracker", + "claim", + "claim_path", "forget", "mark_left", "path_of", "pending", "record", + "release", "runs_dir", "was_left", ] diff --git a/raven/agent/subagent/resume_remote.py b/raven/agent/subagent/resume_remote.py index 82fab294a..fdc98d994 100644 --- a/raven/agent/subagent/resume_remote.py +++ b/raven/agent/subagent/resume_remote.py @@ -7,7 +7,10 @@ the session is finished with it, the registry row and the node index are closed, and the result is announced into the conversation that asked, as a turn that ended here would have been. A machine that cannot be asked keeps -its record for the next start. +its record for the next start; so does a stop of this computer that lands +under a pick-up. Each run is picked up under a claim on its record +(:func:`raven.acp_client.remote_runs.claim`): two processes of one Raven home +that start together pick it up once, not once each. """ from __future__ import annotations @@ -19,7 +22,7 @@ from loguru import logger -from raven.acp_client import remote_resume, remote_runs +from raven.acp_client import remote, remote_resume, remote_runs from raven.acp_client.acp_agent import AcpAgentBackend from raven.agent.subagent.history import SpawnRecord, nodes_root from raven.agent.subagent.instances import get_registry @@ -55,6 +58,16 @@ async def resume_remote_runs(manager: Any) -> int: """Every run left on a machine, picked up and reported; how many were concluded here.""" concluded = 0 for run in remote_runs.pending(): + if remote.leaving(): + logger.info("this computer is stopping: the runs left on machines wait for the next start") + break + if not remote_runs.claim(run.task): + logger.info( + "the run of {!r} on machine {} is being picked up by another process of this computer", + run.agent, + run.machine, + ) + continue backend = _backend_of(manager, run) if backend is None: await _conclude( @@ -70,6 +83,7 @@ async def resume_remote_runs(manager: Any) -> int: try: picked = await remote_resume.pick_up(run, backend) except remote_resume.UnreachableError as exc: + remote_runs.release(run.task) logger.warning("the run of {!r} on machine {} waits for a later start: {}", run.agent, run.machine, exc) continue except Exception as exc: # noqa: BLE001 - one run's failure is reported as its own, the next is tried diff --git a/tests/test_remote_resume.py b/tests/test_remote_resume.py index 7727e4842..b36196e78 100644 --- a/tests/test_remote_resume.py +++ b/tests/test_remote_resume.py @@ -204,3 +204,19 @@ async def test_a_pick_up_beside_a_live_connection_of_the_name_closes_its_own_aft assert picked.status == "ok" assert _pool.connections("stub@box") == [live], "the live connection stays the pool's" assert link.pipe.returncode == 0, "the pick-up's own connection was closed" + + +async def test_a_connection_closed_while_this_computer_is_leaving_keeps_the_record(_pool): + """The stop that closes the pools lands under a pick-up: the run is the next start's, not an error.""" + from raven.acp_client import remote + + link = _Link() + picking = asyncio.ensure_future(remote_resume.pick_up(_run(), _backend(), pool=link)) + await asyncio.sleep(0.05) + remote.leave_agents_running() + try: + await link.pipe.end() + with pytest.raises(remote_resume.UnreachableError, match="stopped while the run of 'stub@box'"): + await asyncio.wait_for(picking, timeout=5) + finally: + remote.staying() diff --git a/tests/test_remote_runs.py b/tests/test_remote_runs.py index 5c7c7270a..69062991f 100644 --- a/tests/test_remote_runs.py +++ b/tests/test_remote_runs.py @@ -130,3 +130,45 @@ def test_a_dispatch_left_on_its_machine_is_read_as_left_once() -> None: def test_the_directory_is_made_under_the_raven_home(_home: Path) -> None: assert remote_runs.runs_dir() == _home / "machines" / "runs" assert os.path.isdir(_home / "machines" / "runs") + + +def test_a_claim_is_taken_once_per_live_process_and_goes_with_the_record() -> None: + """Two processes of one home starting together: the one that claims a run picks it up; the other leaves it.""" + remote_runs.record(_run()) + + assert remote_runs.claim("echo@box-1a2b3c4d") is True + assert remote_runs.claim("echo@box-1a2b3c4d") is False, "held by a process that is alive: this one" + assert remote_runs.claim_path("echo@box-1a2b3c4d").read_text(encoding="utf-8") == str(os.getpid()) + remote_runs.release("echo@box-1a2b3c4d") + assert remote_runs.claim("echo@box-1a2b3c4d") is True, "released for a later start, it can be taken again" + remote_runs.forget("echo@box-1a2b3c4d") + assert not remote_runs.claim_path("echo@box-1a2b3c4d").exists() and remote_runs.pending() == [] + remote_runs.release("never-claimed") + + +def test_a_claim_left_by_a_process_that_is_gone_is_taken_over(monkeypatch) -> None: + remote_runs.claim_path("gone").write_text("4000000", encoding="utf-8") + remote_runs.claim_path("held").write_text("12345", encoding="utf-8") + remote_runs.claim_path("blank").write_text("", encoding="utf-8") + monkeypatch.setattr(remote_runs, "pid_alive", lambda pid: pid == 12345) + + assert remote_runs.claim("gone") is True and remote_runs.claim_path("gone").read_text() == str(os.getpid()) + assert remote_runs.claim("held") is False and remote_runs.claim_path("held").read_text() == "12345" + assert remote_runs.claim("blank") is True, "a claim that names nobody holds nothing" + + +def test_a_claim_is_made_whole_without_hard_links_too_and_one_that_cannot_be_made_is_named( + tmp_path, monkeypatch, logs +) -> None: + def no_links(src, dst, *a, **k): + raise OSError("hard links are not supported here") + + monkeypatch.setattr(os, "link", no_links) + assert remote_runs.claim("plain") is True + assert remote_runs.claim_path("plain").read_text(encoding="utf-8") == str(os.getpid()) + assert remote_runs.claim("plain") is False + + (tmp_path / "a-file").write_text("x", encoding="utf-8") + monkeypatch.setattr(remote_runs, "runs_dir", lambda: tmp_path / "a-file" / "runs") + assert remote_runs.claim("nowhere") is False + assert "a claim could not be made" in "\n".join(logs) diff --git a/tests/test_resume_remote.py b/tests/test_resume_remote.py index 81db61abc..2f4772bdf 100644 --- a/tests/test_resume_remote.py +++ b/tests/test_resume_remote.py @@ -276,3 +276,52 @@ async def fail(manager): await asyncio.wait_for(task, timeout=5) assert "picking up the runs left on machines failed" in "\n".join(logs) + + +async def test_two_processes_starting_together_pick_a_run_up_once(tmp_path, monkeypatch, logs): + """A gateway and a page stack of one home start at once: the run is picked up by one and reported once.""" + backend = _backend() + managers = [_Manager(backend, tmp_path / "s1"), _Manager(backend, tmp_path / "s1")] + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + _open_record(tmp_path / "s1") + _run() + picked_by: list[str] = [] + + async def picked(run, backend, *, pool=None): + picked_by.append(run.task) + await asyncio.sleep(0.1) + return remote_resume.PickedUp("ok", "pong") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + + counts = await asyncio.gather(*(resume_remote.resume_remote_runs(manager) for manager in managers)) + + assert sorted(counts) == [0, 1] and picked_by == ["stub@box-1a2b3c4d"] + assert sum(len(manager.announced) for manager in managers) == 1 + assert "is being picked up by another process of this computer" in "\n".join(logs) + assert remote_runs.pending() == [] and not remote_runs.claim_path("stub@box-1a2b3c4d").exists() + + +async def test_a_stop_during_a_pick_up_keeps_that_record_and_the_ones_after_it(tmp_path, monkeypatch, logs): + from raven.acp_client import remote + + manager = _Manager(_backend(), tmp_path / "s1") + _run(task="first", started=1.0) + _run(task="second", started=2.0) + asked: list[str] = [] + + async def picked(run, backend, *, pool=None): + asked.append(run.task) + remote.leave_agents_running() + raise remote_resume.UnreachableError("this computer stopped while the run was being picked up") + + monkeypatch.setattr(remote_resume, "pick_up", picked) + try: + assert await resume_remote.resume_remote_runs(manager) == 0 + finally: + remote.staying() + + assert asked == ["first"] and manager.announced == [] + assert [run.task for run in remote_runs.pending()] == ["first", "second"] + assert remote_runs.claim("first") and remote_runs.claim("second"), "nothing is left claimed" + assert "wait for the next start" in "\n".join(logs) diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 2754c3bd2..2a4c68994 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -558,6 +558,59 @@ async def test_a_run_left_on_the_machine_when_this_computer_stops_is_picked_up_b assert _agent_pid() == agent, "the agent's process is the pool's connection again" +async def test_a_stop_that_lands_under_a_pick_up_leaves_the_run_for_the_start_after(box, tmp_path, monkeypatch): + """This computer stops again while a pick-up is attached: the record and the agent stay, and the next start reads the answer.""" + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + backend = build_third_party_backend(remote_config(env={"ACP_STUB_PROMPT_DELAY_S": "4"})) + turn = asyncio.ensure_future( + backend.run("hi", task_id="t1", workspace=tmp_path, executor=None, session_key="tui:s1") + ) + deadline = time.monotonic() + 20 + while not any(run.prompt_id > 0 for run in remote_runs.pending()): + assert time.monotonic() < deadline and not turn.done(), "the prompt never went out" + await asyncio.sleep(0.05) + agent = _agent_pid() + remote.leave_agents_running() + try: + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await turn + remote_runs.was_left("t1") + await close_pool() + await box.nodes.close_all() + finally: + remote.staying() + (run,) = remote_runs.pending() + session_dir = tmp_path / "sessions" / "s1" + meta = {"call_id": "n1", "session_key": "tui:s1", "agent": backend.name, "handle": "t1", "task_summary": "say hi"} + SpawnRecord.open(session_dir, task_id="t1", task="hi", meta=meta, node_id="n1") + manager = _Manager(backend, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + + # The next start begins the pick-up, and stops the moment its attach is on the machine. + picking = asyncio.ensure_future(resume_remote.resume_remote_runs(manager)) + deadline = time.monotonic() + 10 + while not any(row.get("task") == run.task and row.get("attached") for row in await box.nodes.tasks("box")): + assert time.monotonic() < deadline and not picking.done(), "the pick-up never attached" + await asyncio.sleep(0.05) + remote.leave_agents_running() + try: + await close_pool() + await box.nodes.close_all() + assert await asyncio.wait_for(picking, timeout=10) == 0 + finally: + remote.staying() + assert manager.announced == [], "nothing is reported for a run this computer did not finish reading" + assert [kept.task for kept in remote_runs.pending()] == [run.task], "the record is the next start's" + assert _alive(agent), "the agent stays with the machine's daemon" + + # The start after that reads the answer. + assert await asyncio.wait_for(resume_remote.resume_remote_runs(manager), timeout=30) == 1 + (said,) = manager.announced + assert said["status"] == "ok" and said["result"].split("\n\n[raven]")[0] == "pong" + assert remote_runs.pending() == [] and _agent_pid() == agent + + async def test_a_daemon_that_stops_mid_turn_fails_the_turn_in_one_sentence(box, tmp_path): box.nodes._heartbeat_s = 0.1 box.nodes._lost_after_s = 1.0 From 4a73e740302beb58ef70c6d076073e7adbc8566b Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:58:17 +0800 Subject: [PATCH 31/44] test(agent): the stand-in machine closes the pool before its links, so no agent outlives a test The fixture closed the machine's links before the ACP pool, and a link closed on purpose leaves its agents with the daemon -- right for a host, whose stop closes its pool first, and a leak for a test: the agent stayed, the daemon never went idle, and every dispatch test of this module left one of each behind (ten per run, counted on one computer). The pool closes first now, the teardown checks that nothing is left under the machine's daemons and ends a daemon that is, and one test shows the order that the teardown relies on. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_subagent_acp_remote.py | 72 ++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 2a4c68994..8494d8a2e 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -11,9 +11,11 @@ from __future__ import annotations import asyncio +import contextlib import json import os import signal +import subprocess import sys import time from pathlib import Path @@ -148,6 +150,20 @@ def calls(self) -> list[list[str]]: return [] return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + def daemons(self) -> list[int]: + """The daemons serving this machine's node directory, as the process table shows them.""" + mark = f"--serve --dir {self.home / '.raven-node'}" + listed = subprocess.run(["ps", "-axo", "pid=,command="], capture_output=True, text=True).stdout + return [int(line.split(None, 1)[0]) for line in listed.splitlines() if mark in line] + + def agents_left(self) -> list[int]: + """The agents still under this machine's daemons: what a test must not leave behind.""" + out: list[int] = [] + for daemon in self.daemons(): + children = subprocess.run(["pgrep", "-P", str(daemon)], capture_output=True, text=True).stdout + out += [int(pid) for pid in children.split()] + return out + @pytest.fixture async def box(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): @@ -155,7 +171,21 @@ async def box(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): pytest.skip("the stand-in ssh hands the ssh line to a POSIX shell") machine = Machine(tmp_path, monkeypatch) yield machine + # The pool before the links, as the host's own stop orders them: an agent + # is ended through its link, and a link closed first leaves its agent with + # the daemon -- right for a host, a leak for a test, whose daemon then + # never goes idle (this module left ten daemons and their stubs behind per + # run, before this). The links the test itself closed are covered by the + # check after: an agent still under a daemon fails the test and is ended. + await close_pool() await machine.nodes.close_all() + deadline = time.monotonic() + 3.0 + while (left := machine.agents_left()) and time.monotonic() < deadline: + await asyncio.sleep(0.05) + for daemon in machine.daemons(): + with contextlib.suppress(ProcessLookupError): + os.kill(daemon, signal.SIGTERM) + assert not left, f"agents left running under the stand-in machine's daemon: {left}" def _agent_pid(name: str = "stub@box") -> int | None: @@ -164,6 +194,16 @@ def _agent_pid(name: str = "stub@box") -> int | None: return connection.client._endpoint.pid +def _alive(pid: int | None) -> bool: + if pid is None: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + def remote_config(*, mode: str = "ok", command: str | None = None, **kw: Any) -> ThirdPartyAcpSubagentConfig: return ThirdPartyAcpSubagentConfig( name=kw.pop("name", "stub@box"), @@ -455,16 +495,6 @@ async def test_a_link_that_goes_mid_turn_is_waited_out_and_the_turn_finishes_whe assert len([argv for argv in box.calls() if "-T" in argv]) == 2 -def _alive(pid: int | None) -> bool: - if pid is None: - return False - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - return True - - class _Manager: """What the pick-up asks of the sub-agent manager, recorded.""" @@ -611,6 +641,28 @@ async def test_a_stop_that_lands_under_a_pick_up_leaves_the_run_for_the_start_af assert remote_runs.pending() == [] and _agent_pid() == agent +async def test_the_pool_closed_before_the_links_leaves_no_agent_and_the_daemon_goes_idle(box, tmp_path): + """What the fixture's teardown relies on: ended through its link, the agent goes, and the daemon leaves on its own.""" + backend = build_third_party_backend(remote_config()) + await backend.run("hi", task_id="t1", workspace=tmp_path, executor=None) + agent = _agent_pid() + (daemon,) = box.daemons() + assert box.agents_left() == [agent] + + await close_pool() + await box.nodes.close_all() + + deadline = time.monotonic() + 3.0 + while _alive(agent) and time.monotonic() < deadline: + await asyncio.sleep(0.05) + assert not _alive(agent) and box.agents_left() == [] + # Idle two seconds after the last link closed (the stand-in ssh's RAVEN_NODE_IDLE_S), checked every second. + deadline = time.monotonic() + 8.0 + while daemon in box.daemons() and time.monotonic() < deadline: + await asyncio.sleep(0.1) + assert daemon not in box.daemons(), "the daemon stays only while an agent runs or a link is open" + + async def test_a_daemon_that_stops_mid_turn_fails_the_turn_in_one_sentence(box, tmp_path): box.nodes._heartbeat_s = 0.1 box.nodes._lost_after_s = 1.0 From 9f8d2b0979e706d86137041025219c01a0d8417c Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:41:11 +0800 Subject: [PATCH 32/44] test(agent): the stand-in machine lists its daemons whole, whatever the terminal width The lookup that finds the daemons serving the stand-in machine read the process table with ps at the terminal's width, and on Linux the daemon's command line runs past 80 columns before the directory that names the machine, so a narrow run listed none of them: the lifecycle test failed on its first line, and the fixture's leak check could neither see nor end a daemon it should have. ps is asked for whole command lines now, and a test plants a process with a long command line under an 80-column setting and finds it. The fixture's comment keeps its reason and drops the count of what an earlier run left behind. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_subagent_acp_remote.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 8494d8a2e..090f0e7e8 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -151,9 +151,14 @@ def calls(self) -> list[list[str]]: return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] def daemons(self) -> list[int]: - """The daemons serving this machine's node directory, as the process table shows them.""" + """The daemons serving this machine's node directory, as the process table shows them. + + ``-ww``: ps cuts a command line at the terminal's width otherwise, and + the daemon's runs past 80 columns before the ``--dir`` that names + this machine, so a narrow run listed none of them. + """ mark = f"--serve --dir {self.home / '.raven-node'}" - listed = subprocess.run(["ps", "-axo", "pid=,command="], capture_output=True, text=True).stdout + listed = subprocess.run(["ps", "-axww", "-o", "pid=,command="], capture_output=True, text=True).stdout return [int(line.split(None, 1)[0]) for line in listed.splitlines() if mark in line] def agents_left(self) -> list[int]: @@ -174,8 +179,7 @@ async def box(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): # The pool before the links, as the host's own stop orders them: an agent # is ended through its link, and a link closed first leaves its agent with # the daemon -- right for a host, a leak for a test, whose daemon then - # never goes idle (this module left ten daemons and their stubs behind per - # run, before this). The links the test itself closed are covered by the + # never goes idle. The links the test itself closed are covered by the # check after: an agent still under a daemon fails the test and is ended. await close_pool() await machine.nodes.close_all() @@ -188,6 +192,22 @@ async def box(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): assert not left, f"agents left running under the stand-in machine's daemon: {left}" +async def test_the_daemon_lookup_reads_whole_command_lines_whatever_the_terminal_width(box, monkeypatch): + """A stand-in daemon whose command line runs past 80 columns is still found, and so can still be ended.""" + monkeypatch.setenv("COLUMNS", "80") + mark = f"--serve --dir {box.home / '.raven-node'}" + padding = "x" * 120 + proc = await asyncio.create_subprocess_exec( + sys.executable, "-c", "import time; time.sleep(30)", padding, *mark.split(), stdout=asyncio.subprocess.DEVNULL + ) + try: + assert proc.pid in box.daemons(), "the mark sits past the 80th column of the command line" + finally: + proc.kill() + await proc.wait() + assert proc.pid not in box.daemons() + + def _agent_pid(name: str = "stub@box") -> int | None: """The pid of the agent process on the machine, read off the pooled connection's endpoint.""" (connection,) = get_pool().connections(name) From c60d463f27b0251aa4443d02a1f306e131734c21 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:46:20 +0800 Subject: [PATCH 33/44] fix(acp_client): taking over a dead process's claim on a run is one step, not two Judging a claim stale and taking it over were two steps with nothing between them: two processes starting together could both read the same dead holder, and the one that got there second removed the fresh claim the first had just made and won its own retry, so both picked the run up and the conversation heard its result twice. The claims of a home are taken one process at a time now, under an advisory lock the kernel drops with the process; a test holds the second starter's judgement until the first has claimed, the interleaving that produced two winners, and sees one. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/remote_runs.py | 63 ++++++++++++++++++++++++++++----- tests/test_remote_runs.py | 47 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/raven/acp_client/remote_runs.py b/raven/acp_client/remote_runs.py index bc4912cc3..46b71f1ba 100644 --- a/raven/acp_client/remote_runs.py +++ b/raven/acp_client/remote_runs.py @@ -23,6 +23,7 @@ import json import os import time +from collections.abc import Iterator from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any @@ -134,17 +135,63 @@ def claim(task: str) -> bool: record; :func:`release` alone, when the run is left for a later start. """ path = claim_path(task) - for _ in range(2): - if _take(path): - return True - holder = _holder(path) - if holder is not None and pid_alive(holder): - return False - with contextlib.suppress(OSError): - path.unlink() + with _one_at_a_time(): + for _ in range(2): + if _take(path): + return True + holder = _holder(path) + if holder is not None and pid_alive(holder): + return False + with contextlib.suppress(OSError): + path.unlink() return False +@contextlib.contextmanager +def _one_at_a_time() -> Iterator[None]: + """The claims of this home taken one process at a time, so the claim a process removes is the one it judged dead. + + Judging a claim stale and taking it over are two steps, and between them + another process could make a fresh claim at the same path -- which the + first then removed and took, two processes picking one run up after all. + An advisory lock on one file beside the claims holds the two steps + together; the kernel drops it with the process, so a holder that dies + leaves no lock behind. Where the lock file cannot be made, the claim + goes on without it and says so if it fails. + """ + try: + fd = os.open(runs_dir() / "claims.lock", os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + yield + return + try: + _lock(fd) + yield + finally: + with contextlib.suppress(OSError): + _unlock(fd) + os.close(fd) + + +try: + import fcntl + + def _lock(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_EX) + + def _unlock(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_UN) + +except ImportError: # pragma: no cover - the other platform + import msvcrt + + def _lock(fd: int) -> None: + msvcrt.locking(fd, msvcrt.LK_LOCK, 1) + + def _unlock(fd: int) -> None: + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + + def release(task: str) -> None: """Give the claim on ``task`` up: the run waits for a later start.""" with contextlib.suppress(FileNotFoundError, OSError): diff --git a/tests/test_remote_runs.py b/tests/test_remote_runs.py index 69062991f..b6d1d4bcc 100644 --- a/tests/test_remote_runs.py +++ b/tests/test_remote_runs.py @@ -4,6 +4,7 @@ import json import os +import threading from pathlib import Path import pytest @@ -157,6 +158,52 @@ def test_a_claim_left_by_a_process_that_is_gone_is_taken_over(monkeypatch) -> No assert remote_runs.claim("blank") is True, "a claim that names nobody holds nothing" +def test_two_starters_taking_over_one_dead_holder_leave_one_claim(monkeypatch) -> None: + """A judged the claim dead and took it; B, who judged it dead a moment earlier, must not remove A's. + + Judging and taking are two steps: B's liveness check is held until A has + claimed, which is the interleaving in which B used to unlink A's fresh + claim and win its own retry, two processes picking one run up. + """ + stale = 4000000 + remote_runs.claim_path("t").write_text(str(stale), encoding="utf-8") + a_inside = threading.Event() + a_done = threading.Event() + real_take, real_alive = remote_runs._take, remote_runs.pid_alive + + def take(path): + if threading.current_thread().name == "A": + a_inside.set() + return real_take(path) + + def alive(pid): + if pid == stale: + if threading.current_thread().name == "B": + a_done.wait(5) + return False + return real_alive(pid) + + monkeypatch.setattr(remote_runs, "_take", take) + monkeypatch.setattr(remote_runs, "pid_alive", alive) + results: dict[str, bool] = {} + + def run(name: str) -> None: + results[name] = remote_runs.claim("t") + if name == "A": + a_done.set() + + a = threading.Thread(target=run, args=("A",), name="A") + b = threading.Thread(target=run, args=("B",), name="B") + a.start() + assert a_inside.wait(5) + b.start() + a.join(10) + b.join(10) + + assert sorted(results.values()) == [False, True], results + assert remote_runs.claim_path("t").read_text(encoding="utf-8") == str(os.getpid()) + + def test_a_claim_is_made_whole_without_hard_links_too_and_one_that_cannot_be_made_is_named( tmp_path, monkeypatch, logs ) -> None: From 0eaac19bffdad1c8b182897b30650a35f8dfc43b Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:22:34 +0800 Subject: [PATCH 34/44] fix(*): files by address for openai rows, commands split for the machine, IPv6 endpoints bracketed Three findings from the agent and Windows reviews of this PR, each with a test that failed on the code before and a run on a registered machine with the code before and after. - An openai entry on a machine told its endpoint about the files that came with the task with this computer's note (absolute paths under "[attachments, by absolute path]"), as if the endpoint could read them. It now says what the acp and cli lanes say: the files by their address on the computer that sent them, nothing copied, say so and go on where one cannot be found (remote.files_note). The machine is read from the registry once per turn, before the message is built, and the tunnel is opened with it. - A cli entry on a machine had its command split by this computer's grammar (command_argv: CommandLineToArgvW on Windows), so from a Windows host `sh -c 'hostname; pwd; cat'` reached the machine as five words and its shell refused the script. _build_argv takes the splitter, and the launch on a machine passes shlex.split: the node runs the command under a POSIX shell, whatever this computer is. - A tunnel to an endpoint on the machine's IPv6 loopback was asked of ssh as `-L 127.0.0.1::::1:8000`, which ssh refuses; the address is bracketed now, as it is in the URL. The stand-in forwarder reads a bracketed far host, and the comment on the address shown in a reply no longer says the file tools read it, which this PR's do not. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/remote.py | 5 +- raven/agent/subagent/backends/cli_agent.py | 21 +++++- raven/agent/subagent/backends/openai_api.py | 59 +++++++++++++---- raven/config/schema.py | 13 ++-- tests/test_subagent_acp_remote.py | 2 +- tests/test_subagent_cli_remote.py | 73 +++++++++++++++++++-- 6 files changed, 147 insertions(+), 26 deletions(-) diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index 8cdca091f..db0153f82 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -426,7 +426,10 @@ async def tunnel( raise RemoteMachineError(f"machine {target.label}: the endpoint's URL names no host to reach from it") host, port, key, user = _target(target) local = _free_port() - forward = f"127.0.0.1:{local}:{parts.hostname}:{parts.port or 80}" + # An IPv6 address is bracketed in the forward as it is in the URL, which + # ``hostname`` gives bare; ssh refuses the bare one. + far = f"[{parts.hostname}]" if ":" in parts.hostname else parts.hostname + forward = f"127.0.0.1:{local}:{far}:{parts.port or 80}" extra = TUNNEL + ("-L", forward, "-E", os.fspath(ssh_log)) argv = ssh_argv(host, port, key, user=user, connect_timeout=connect_timeout, extra=extra) try: diff --git a/raven/agent/subagent/backends/cli_agent.py b/raven/agent/subagent/backends/cli_agent.py index e92aee049..d5fef30b5 100644 --- a/raven/agent/subagent/backends/cli_agent.py +++ b/raven/agent/subagent/backends/cli_agent.py @@ -292,10 +292,20 @@ def _build_argv( prompt_file: str, agent_id: str | None, mcp_file: str | None = None, + *, + split: Callable[[str], list[str]] = command_argv, ) -> tuple[list[str], bool]: + """The argv of one launch: ``template`` split by ``split``, its placeholders filled in. + + ``split`` reads this computer's grammar by default (``command_argv``: + POSIX here, CommandLineToArgvW on Windows), right for a process + started here. A launch on a machine passes the machine's instead: its + node runs the command under a POSIX shell, so ``shlex.split`` reads + the template the way that shell will, whatever this computer is. + """ argv: list[str] = [] used_placeholder = False - for tok in command_argv(template): + for tok in split(template): if agent_id is not None: tok = tok.replace("{agent_id}", agent_id) if mcp_file is not None: @@ -603,7 +613,9 @@ async def _exec_there( ) -> tuple[str, str]: """One invocation on the entry's machine, through its node: the same argv, run and read there. - What differs from :meth:`_exec` is where each piece lives. The prompt + What differs from :meth:`_exec` is where each piece lives, and whose + grammar splits the command -- the machine's POSIX shell's, not this + computer's. The prompt file is written into the run's directory on the machine, under its ``.raven/``, since a path here means nothing there; stdin travels on the link and is closed after it -- for every command, not only one @@ -637,7 +649,10 @@ async def _exec_there( await placed.link.write_file(placed.target.id, mcp_there, empty, roots=[cwd]) except remote_path.NodeCallError as exc: raise RuntimeError(f"CLI agent {self.name!r}: {exc}") from None - argv, used_placeholder = self._build_argv(template, task, prompt_path, agent_id, mcp_there) + # Split for the machine's shell, not this computer's: the node runs + # the command under a POSIX shell, and a Windows host's own splitter + # would hand it ``sh -c 'a; b'`` as the words ``'a;`` and ``b'``. + argv, used_placeholder = self._build_argv(template, task, prompt_path, agent_id, mcp_there, split=shlex.split) env = {**subagent_role_env(), **self.env} logger.info("Subagent [{}] CLI agent {!r} on machine {}: {}", task_id, self.name, placed.target.id, argv[:1]) try: diff --git a/raven/agent/subagent/backends/openai_api.py b/raven/agent/subagent/backends/openai_api.py index 37a9aef2c..96245542e 100644 --- a/raven/agent/subagent/backends/openai_api.py +++ b/raven/agent/subagent/backends/openai_api.py @@ -13,7 +13,10 @@ machine: each turn opens an ssh tunnel to it (:func:`raven.acp_client.remote.tunnel`) and the request goes to ``base_url`` as the machine resolves it, for an endpoint that answers only on that machine's network, or only on the machine -itself. The tunnel ends with the turn. +itself. The tunnel ends with the turn. The files that came with the task +are named to such an entry by their address on this computer, nothing +being copied to the machine, as the acp and cli lanes to a machine say it +(:func:`raven.acp_client.remote.files_note`). """ from __future__ import annotations @@ -36,9 +39,27 @@ from raven.spine.message import Media if TYPE_CHECKING: + from raven.acp_client.remote import Machine from raven.contracts.llm_provider import LLMProvider +def _with_files(task: str, media: Sequence[Media], target: Machine | None) -> str: + """The task with the files that came with it named at the end, the way the entry's reader can use them. + + From this computer, by absolute path (``with_attachment_note``). On a + machine, by their address on this computer and with the word that nothing + was copied there (``remote.files_note``), as the acp and cli lanes to a + machine say it: the endpoint answers from the machine, where a path of + this computer's names nothing unless the disk is shared. + """ + if target is None: + return with_attachment_note(task, media) + from raven.acp_client import remote + + note = remote.files_note(media, target) + return f"{task}\n\n{note}" if note else task + + class OpenAIApiBackend: kind = "openai" streams = True @@ -69,24 +90,37 @@ def __init__( # The registered machine the endpoint is reached from, or None for this computer. self.machine = (machine or "").strip() or None + async def _placed(self) -> Machine | None: + """The entry's machine, as the registry has it; ``None`` for an entry posted to from this computer.""" + if not self.machine: + return None + from raven.acp_client import remote + + try: + return await asyncio.to_thread(remote.machine, self.machine) + except remote.RemoteMachineError as exc: + raise RuntimeError(f"OpenAI-API agent {self.name!r}: {exc}") from None + @contextlib.asynccontextmanager - async def _reached(self, url: str, headers: dict[str, str]) -> AsyncIterator[tuple[str, dict[str, str]]]: + async def _reached( + self, url: str, headers: dict[str, str], target: Machine | None + ) -> AsyncIterator[tuple[str, dict[str, str]]]: """``url`` and ``headers`` as this turn posts them. - As they are from this computer; through a tunnel from the entry's - machine otherwise, opened for this turn and closed after it. The - request then arrives at a port on this computer's loopback: ``Host`` - carries the name the server knows itself by, and the proxy variables - of this computer's environment are not read for it (``_post_chat``), - since a proxy would take a loopback address somewhere else. + As they are from this computer, for no ``target``; through a tunnel + from the entry's machine otherwise, opened for this turn and closed + after it. The request then arrives at a port on this computer's + loopback: ``Host`` carries the name the server knows itself by, and + the proxy variables of this computer's environment are not read for + it (``_post_chat``), since a proxy would take a loopback address + somewhere else. """ - if not self.machine: + if target is None: yield url, headers return from raven.acp_client import remote try: - target = await asyncio.to_thread(remote.machine, self.machine) log = await asyncio.to_thread(remote.ssh_log_path, f"{self.name}-tunnel") except remote.RemoteMachineError as exc: raise RuntimeError(f"OpenAI-API agent {self.name!r}: {exc}") from None @@ -257,7 +291,8 @@ async def run( messages: list[dict[str, Any]] = list(history) if history else [] if not messages and self.system_prompt: messages.append({"role": "system", "content": self.system_prompt}) - messages.append({"role": "user", "content": with_attachment_note(task, media)}) + target = await self._placed() + messages.append({"role": "user", "content": _with_files(task, media, target)}) # Always explicit, never omitted: a provider whose default is SSE # (mirothinker) answers an omitted `stream` with text/event-stream, which # `resp.json()` cannot read. A spawn asks for false and keeps the single @@ -284,7 +319,7 @@ async def run( client_timeout = aiohttp.ClientTimeout(total=self.timeout, connect=30) reader = OpenAIStepReader() - async with self._reached(url, headers) as (url, headers): + async with self._reached(url, headers, target) as (url, headers): if on_delta is not None: content: Any = await self._stream_chat( url, diff --git a/raven/config/schema.py b/raven/config/schema.py index bd46f3a67..e87db6356 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -1798,11 +1798,14 @@ def _null_stateful_is_the_default(cls, value: Any) -> Any: Set, each turn opens an ssh tunnel to that machine and closes it after: a port on this computer's loopback forwarded to ``baseUrl``'s host and port as the machine resolves them, for an endpoint that answers only on that - machine's network, or only on the machine itself (``http://127.0.0.1:...``). - ``baseUrl`` has to be ``http``: the tunnel ends at this computer's - loopback, which an ``https`` certificate would not name. The address, port - and key the registry holds are read for the tunnel alone and written - nowhere here. Debug-only as for the other kinds: unless raven was started + machine's network, or only on the machine itself (``http://127.0.0.1:...``, + or ``http://[::1]:...``). ``baseUrl`` has to be ``http``: the tunnel ends + at this computer's loopback, which an ``https`` certificate would not + name. The files that come with a task are named to the endpoint by their + address on this computer and not copied, as to the other kinds on a + machine. The address, port and key the registry holds are read for the + tunnel alone and written nowhere here. Debug-only as for the other kinds: + unless raven was started with ``RAVEN_DEBUG_MACHINES=1``, the entry is left off the table and no machine is reached for it. There is no ``remoteCwd``: an endpoint has no working directory on the machine, and one written here is dropped with a diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 090f0e7e8..3ebf8593c 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -248,7 +248,7 @@ async def test_a_dispatch_runs_on_the_machine_in_a_directory_made_there_and_says there = box.home / "raven-work" / "reviewer" assert there.is_dir(), "the session directory is made on the machine" assert "[raven] Ran on machine 'Lab box' (box), in " in reply - # Shown the way scp writes it, which the file tools read back as it is. + # Shown the way scp writes it. assert f"in scp://worker@{HOST}:{PORT}//{str(there.resolve()).lstrip('/')};" in reply where = reply.split("in scp://", 1)[1].split(";", 1)[0] assert where.split("@", 1)[1].split(":", 1)[0] == HOST, "the address a model reads is the registry's, never the id" diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py index c20b9cd9d..88248fef3 100644 --- a/tests/test_subagent_cli_remote.py +++ b/tests/test_subagent_cli_remote.py @@ -15,6 +15,7 @@ import sys import time from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest @@ -64,7 +65,10 @@ if "-N" not in args or "-L" not in args: sys.stderr.write("this stand-in only forwards\\n") sys.exit(2) -bind_host, bind_port, far_host, far_port = args[args.index("-L") + 1].rsplit(":", 3) +# The forward as ssh reads it: a bracketed IPv6 far host keeps its own colons. +bind_host, bind_port, far = args[args.index("-L") + 1].split(":", 2) +far_host, far_port = far.rsplit(":", 1) +far_host = far_host.strip("[]") listener = socket.socket() listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind((bind_host, int(bind_port))) @@ -333,6 +337,21 @@ async def test_the_files_that_came_with_the_task_are_named_by_address_and_not_co assert not (box.home / "raven-work" / "counter" / "sales.csv").exists() +async def test_the_command_is_split_the_way_the_machine_s_shell_reads_it_whatever_this_computer_s_is( + box, tmp_path, monkeypatch +): + """From a Windows host, ``sh -c 'echo one; cat'`` reaches the machine as the three words its shell reads.""" + from raven.utils import commands + + cfg = cli_config("sh -c 'echo one; cat'") + # This computer as Windows, for the splitter alone: ``command_argv`` reads ``os.name`` from its own module. + monkeypatch.setattr(commands, "os", SimpleNamespace(**{**vars(os), "name": "nt"})) + + head, _ = _split(await _run(cfg, "two", tmp_path)) + + assert head.strip() == "one\ntwo" + + async def test_mcp_servers_are_not_lent_to_a_cli_agent_on_a_machine_and_the_reply_says_why(box, tmp_path): cfg = cli_config("cat {mcp_file}", mcps=["pg"]) grant = McpGrant(granted=(GrantedServer("pg", "stdio", MCPServerConfig(command="/bin/true")),)) @@ -434,8 +453,8 @@ async def test_a_failed_ping_of_a_cli_row_says_the_fix_is_made_on_the_machine(bo # ---- an openai agent reached from the machine ------------------------------------------- -async def _chat_completions_stub() -> tuple[Any, int, list[tuple[list[str], dict[str, Any]]]]: - """An endpoint on this computer's loopback: the far end of the stand-in tunnel.""" +async def _chat_completions_stub(host: str = "127.0.0.1") -> tuple[Any, int, list[tuple[list[str], dict[str, Any]]]]: + """An endpoint on this computer's loopback, at ``host``: the far end of the stand-in tunnel.""" seen: list[tuple[list[str], dict[str, Any]]] = [] async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: @@ -459,7 +478,7 @@ async def handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> await writer.drain() writer.close() - server = await asyncio.start_server(handle, "127.0.0.1", 0) + server = await asyncio.start_server(handle, host, 0) return server, server.sockets[0].getsockname()[1], seen @@ -487,6 +506,52 @@ async def test_an_openai_row_on_a_machine_is_reached_through_a_tunnel_from_it(fo await _until(lambda: _gone(pid)) # the tunnel ends with the turn +async def test_the_files_that_came_with_an_openai_row_s_task_are_named_by_address_and_not_copied(forwarding, tmp_path): + """The openai lane says of the files what the acp and cli lanes say: their addresses here, nothing carried.""" + server, port, seen = await _chat_completions_stub() + data = tmp_path / "sales.csv" + data.write_text("day,amount\n1,3\n", encoding="utf-8") + media = (Media(path=str(data), mime="text/csv", kind="file"),) + try: + reply = await _run(http_config(f"http://localhost:{port}/v1"), "count the rows", tmp_path, media=media) + finally: + server.close() + await server.wait_closed() + + assert reply == "pong through the tunnel" + sent = seen[-1][1]["messages"][-1]["content"] + assert sent.startswith("count the rows\n\nFiles that came with this task, by their address on the computer") + assert "Nothing was copied here" in sent and f"- {data}" in sent + assert "[attachments, by absolute path]" not in sent, "not the local note, whose paths mean nothing there" + + +async def test_an_ipv6_endpoint_is_forwarded_to_with_its_address_bracketed_as_ssh_reads_it(forwarding, tmp_path): + """``http://[::1]:8000/v1`` asks for ``-L 127.0.0.1::[::1]:8000``; bare, ssh refuses the forward.""" + forwarding.mode("unreachable") + + with pytest.raises(RuntimeError, match="did not answer within the connection timeout"): + await _run(http_config("http://[::1]:8000/v1"), "hi", tmp_path) + + (argv,) = forwarding.calls() + forward = argv[argv.index("-L") + 1] + assert forward.startswith("127.0.0.1:") and forward.endswith(":[::1]:8000"), forward + + +async def test_an_endpoint_on_the_machine_s_ipv6_loopback_answers_through_the_tunnel(forwarding, tmp_path): + try: + server, port, seen = await _chat_completions_stub(host="::1") + except OSError: + pytest.skip("this computer has no IPv6 loopback") + try: + reply = await _run(http_config(f"http://[::1]:{port}/v1"), "hi", tmp_path) + finally: + server.close() + await server.wait_closed() + + assert reply == "pong through the tunnel" + assert any(h.lower() == f"host: [::1]:{port}" for h in seen[-1][0]), "the name the server knows itself by" + + async def test_the_free_probe_of_an_openai_row_asks_no_network_and_a_test_asks_through_the_tunnel(forwarding, tmp_path): server, port, seen = await _chat_completions_stub() cfg = http_config(f"http://localhost:{port}/v1", api_key="sk-raven-test-not-real") From 14c1e2afe492792d2663549aece41534ef77b214 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:25:27 +0800 Subject: [PATCH 35/44] test(agent): the copy test's stand-in for the client runs on a Windows host too `true` and `touch` stood for the copy command and its marker, and neither is on a Windows host, where the test failed on the marker. Python touches the marker now, with the copy command's words as its arguments, as before them: what the test checks is unchanged -- a line with scp, sftp or rsync to a registered machine in it is the shell's to run, and only a raw ssh to one is refused. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_shell_machine_channel.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index b21bd39c4..e19b968bf 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -828,13 +828,19 @@ async def test_a_copy_to_a_registered_machine_is_not_refused(registry, tmp_path, The file tools cannot carry what a copy carries -- a directory, a binary file, a file over the fetch budget -- so a refusal left those with no - route, and taught the very spelling it objected to. ``true`` stands for + route, and taught the very spelling it objected to. Python stands for the client: nothing here connects anywhere. """ + import sys + tool = ExecTool(working_dir=str(tmp_path)) marker = tmp_path / "ran" + # Python stands for the client on any host (``true`` and ``touch`` are not + # on a Windows one): the copy's words are its arguments, and the marker + # says the line ran. + client = f'"{sys.executable}" -c "import pathlib, sys; pathlib.Path(sys.argv[-1]).touch()"' - out = await tool.execute(command=f"true {command}; touch {marker}") + out = await tool.execute(command=f'{client} {command} "{marker}"') assert "Error:" not in out assert marker.exists() From c0964b2feb0d4ccaa8cd6537c62341f3682faf64 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:24:08 +0800 Subject: [PATCH 36/44] test: a worker goes on after a test ran a host's shutdown `tests/test_rpc_bootstrap.py` runs the server's shutdown, which says `leave_agents_running()` -- once, never unsaid, right for a process about to end. In a pytest worker that goes on, every later close of a stand-in machine's agent then left it with its daemon, and the leak check in `tests/test_subagent_acp_remote.py` failed for the rest of that worker's tests: CI's shard 2/4 at c60d463f2, six errors in one worker. An autouse fixture in conftest says `staying()` after each test. Reproduced by running that file and then one machine test in one process, and fixed the same way. Co-authored-by: Claude (claude-fable-5-1) --- tests/conftest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index dcdd6b092..054f583d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,6 +39,7 @@ def _is_utf8_codec(encoding: str) -> bool: import fnmatch import functools import shutil +import sys import tempfile import threading import time @@ -1129,3 +1130,21 @@ def _without_the_cloud_plugin(**kwargs): return metadata.EntryPoints(ep for ep in eps if ep.name != "everos-cloud-memory") monkeypatch.setattr(metadata, "entry_points", _without_the_cloud_plugin) + + +@pytest.fixture(autouse=True) +def _this_process_goes_on() -> Iterator[None]: + """After a test that ran a host's shutdown, the next test is a process that goes on. + + ``raven.acp_client.remote.leave_agents_running`` is said once by a host + that is stopping and never unsaid, which is right for a process about to + end and wrong for a worker that runs more tests: with it still said, + every later close leaves a stand-in machine's agent with its daemon, and + the leak check of ``tests/test_subagent_acp_remote.py`` fails for the + rest of the worker's life (``tests/test_rpc_bootstrap.py`` runs that + shutdown). + """ + yield + remote = sys.modules.get("raven.acp_client.remote") + if remote is not None: + remote.staying() From b4993708857456c59df6c825a9a65b01a70aa4a9 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:24:08 +0800 Subject: [PATCH 37/44] test(acp_client): the record's modes are a test of their own, POSIX only The read-back test asserted 0600/0700 on the record and its directory, which Windows's st_mode does not express, so the test failed there for a reason that is not the record's. The modes are a test of their own now, skipped on Windows as the launcher tests skip theirs; the read-back runs everywhere. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_remote_runs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_remote_runs.py b/tests/test_remote_runs.py index b6d1d4bcc..3e33539ae 100644 --- a/tests/test_remote_runs.py +++ b/tests/test_remote_runs.py @@ -52,16 +52,22 @@ def _run(task: str = "echo@box-1a2b3c4d", **kw) -> remote_runs.RemoteRun: return remote_runs.RemoteRun(**fields) -def test_a_record_is_written_owner_only_and_read_back_whole(_home: Path) -> None: +def test_a_record_is_read_back_whole(_home: Path) -> None: path = remote_runs.record(_run(received=7, started=1.5)) assert path == _home / "machines" / "runs" / "echo@box-1a2b3c4d.json" - assert path.stat().st_mode & 0o777 == 0o600 and path.parent.stat().st_mode & 0o777 == 0o700 (back,) = remote_runs.pending() assert back == _run(received=7, started=1.5) assert json.loads(path.read_text(encoding="utf-8"))["version"] == remote_runs.VERSION +@pytest.mark.skipif(os.name == "nt", reason="POSIX file modes are not available on Windows") +def test_a_record_and_its_directory_are_owner_only(_home: Path) -> None: + path = remote_runs.record(_run()) + + assert path.stat().st_mode & 0o777 == 0o600 and path.parent.stat().st_mode & 0o777 == 0o700 + + def test_pending_lists_the_runs_oldest_first_and_names_what_it_cannot_read(_home: Path, logs) -> None: remote_runs.record(_run("later", started=20.0)) remote_runs.record(_run("earlier", started=10.0)) From 3908042eac2ca90596f38821cea7b1dc2c2b0fb0 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:55:28 +0800 Subject: [PATCH 38/44] fix(*): a turn left on a machine survives a TUI exit, a cli run too, and is reported as asked The four lifecycle findings of this PR's review, each with a test that failed on the code before and, for the cli lane, a run on a registered machine with the code before and after. - The TUI's shutdown says the agents on machines stay before it cancels, as the gateway's and the served page's do, and its start schedules the pick-up: an ordinary TUI exit ended a turn running on a machine as cancelled, and a TUI start never picked one up. - A cli entry's run on a machine is recorded while it runs and left with the daemon when this process stops (the manager reads the dispatch as left), and the entry picks it up: the daemon replays the command's whole output from its first line, the exit code ends it, and the reply closes with the machine note. Before, the command went on under the daemon with no record, and the dispatch was announced as cancelled. - The announce after a restart carries what the announce here would have: the record holds the task as the model wrote it (from the dispatch), and the dispatch's origin and summary once the manager read the run as left (remote_runs.annotate); a host killed outright gets an origin made up from the spawn record. The rendered prompt on disk is never quoted: a `{{ ref:... }}` spawn expands a whole file into it. - The record goes last, once the outcome has reached its conversation: a conclusion that fails, a stop under it, or an announce the scheduler refuses (_announce_result says so now) leaves the record, its claim released, for the next start to try again. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 6 +- raven/acp_client/remote_resume.py | 3 +- raven/acp_client/remote_runs.py | 44 +++- raven/agent/subagent/backends/cli_agent.py | 241 +++++++++++++++++--- raven/agent/subagent/manager.py | 15 +- raven/agent/subagent/resume_remote.py | 132 ++++++++--- raven/cli/tui_commands.py | 11 + tests/test_cli_tui_commands.py | 28 +++ tests/test_remote_runs.py | 43 ++++ tests/test_resume_remote.py | 156 ++++++++++++- tests/test_subagent_cli_remote.py | 248 +++++++++++++++++++++ tests/test_subagent_manager.py | 68 ++++++ 12 files changed, 915 insertions(+), 80 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 241a3befa..809e28829 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -442,8 +442,10 @@ link is tried again on a backoff for as long as any waits; a link that comes bac each agent again by its task id. A turn running on a machine when this computer stops is left with the daemon (`remote.leave_agents_running()`), remembered while in flight in `/machines/runs/` (`raven/acp_client/remote_runs.py`), and picked up by the -next start (`raven/agent/subagent/resume_remote.py`): attached from the frame last read, -its reply read off the machine and announced into the conversation that asked. Only +next start (`raven/agent/subagent/resume_remote.py`): attached from the frame last read +(a cli entry's command from its first line, its whole output being the reply), its reply +read off the machine and announced into the conversation that asked -- the task quoted as +the model wrote it, the record removed only once the announce was taken. Only `--stdio` (tests, a fallback) ties the agents to the connection. _Avoid_: "node" alone (a DAG node is a `run_subagent_dag` step, and the Machine entry already rules the bare word out for a computer); "node id" for the install diff --git a/raven/acp_client/remote_resume.py b/raven/acp_client/remote_resume.py index fc0847296..f9de0a857 100644 --- a/raven/acp_client/remote_resume.py +++ b/raven/acp_client/remote_resume.py @@ -8,7 +8,8 @@ a connection over that channel is taken into the pool as if started here, the turn's collector reads the agent's messages off it, and the answer to the prompt -- sent by the other process, awaited by this one -- ends the wait. What -to do with the reply is the manager's business (:mod:`raven.agent.subagent.resume_remote`). +to do with the reply is the manager's business (:mod:`raven.agent.subagent.resume_remote`), +and a cli entry's run is read by the entry itself (``CliAgentBackend.pick_up``). """ from __future__ import annotations diff --git a/raven/acp_client/remote_runs.py b/raven/acp_client/remote_runs.py index 46b71f1ba..fa70de1f2 100644 --- a/raven/acp_client/remote_runs.py +++ b/raven/acp_client/remote_runs.py @@ -8,9 +8,11 @@ the instance handle, the dispatch), the ACP session the turn runs in, the id of the prompt whose answer is awaited, how many of the agent's frames reached here, the ids this side has used, and what the agent answered to -``initialize`` on the connection. One file per run under +``initialize`` on the connection -- none of the ACP part for a cli entry's +command, whose whole output is its reply, read from the start. One file per run under ``/machines/runs/.json``, owner-only, written when the turn -starts, kept current as frames arrive, and removed when the turn ends here. A +starts, kept current as frames arrive, and removed when the turn ends here or +once the process that picked it up has reported it. A file left behind is a run to pick up (:mod:`raven.acp_client.remote_resume`), by one process at a time: the pick-up is made under a claim beside the record (:func:`claim`), since two processes of one Raven home can start together. @@ -66,6 +68,16 @@ class RemoteRun: received: int = 0 """How many of the agent's frames this computer has read on the connection: where an attach goes on from.""" started: float = field(default_factory=time.time) + kind: str = "acp" + """``acp`` for a turn in an ACP session, ``cli`` for a command whose whole output is the reply + (:mod:`raven.agent.subagent.backends.cli_agent`): which entry picks the run up, and how.""" + authored_task: str = "" + """The task as the dispatching model wrote it: what the announce after a restart quotes, never the rendered prompt.""" + task_summary: str = "" + """The dispatch's summary, as the announce names the run.""" + origin: dict[str, Any] | None = None + """The dispatch's origin as the manager built it (instance, handle, node, tool call), added when the run is + left (:func:`annotate`); ``None`` for a host that stopped before it could say so.""" def as_json(self) -> dict[str, Any]: return {"version": VERSION, **asdict(self)} @@ -80,7 +92,7 @@ def from_json(cls, data: Any) -> RemoteRun | None: run = cls(**{key: value for key, value in data.items() if key in fields}) except TypeError: return None - if not (run.machine and run.task and run.agent and run.session_id): + if not (run.machine and run.task and run.agent and (run.session_id or run.kind == "cli")): return None return run @@ -104,11 +116,34 @@ def record(run: RemoteRun) -> Path: tmp = path.with_suffix(".json.tmp") fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as handle: - json.dump(run.as_json(), handle) + json.dump(run.as_json(), handle, default=str) os.replace(tmp, path) return path +def annotate(task_id: str, **fields: Any) -> bool: + """Add ``fields`` to the record of the dispatch ``task_id`` (the manager's id); whether there was one. + + What the manager knows and the backend that wrote the record did not -- + the dispatch's origin and summary -- written when the manager learns the + run was left, so the announce after a restart carries what the announce + here would have. A host killed outright never gets here, and the record + is read without them then (:mod:`raven.agent.subagent.resume_remote`). + """ + for run in pending(): + if run.task_id != task_id: + continue + for name, value in fields.items(): + setattr(run, name, value) + try: + record(run) + except OSError as exc: + logger.warning("remote run record {} could not be annotated: {}", run.task, exc) + return False + return True + return False + + def forget(task: str) -> None: """Remove the record of ``task``, and the claim on it: the turn ended here, or was concluded here.""" with contextlib.suppress(FileNotFoundError, OSError): @@ -310,6 +345,7 @@ def was_left(task_id: str) -> bool: "RUNS_DIR", "RemoteRun", "Tracker", + "annotate", "claim", "claim_path", "forget", diff --git a/raven/agent/subagent/backends/cli_agent.py b/raven/agent/subagent/backends/cli_agent.py index d5fef30b5..fa62ecb40 100644 --- a/raven/agent/subagent/backends/cli_agent.py +++ b/raven/agent/subagent/backends/cli_agent.py @@ -21,6 +21,12 @@ a local run's are. Nothing of this computer goes with it: not its login shell, not its working directory, not the files that came with the task, which are named to it by address. + +While this computer stops, a run on a machine is left with the machine's +daemon and its record with this computer (:mod:`raven.acp_client.remote_runs`), +and the next process reads the command's whole output off the daemon and +reports it (``pick_up``, through :mod:`raven.agent.subagent.resume_remote`), +as an acp entry's turn is picked up. """ from __future__ import annotations @@ -38,7 +44,7 @@ import uuid from collections.abc import Awaitable, Callable, Sequence from contextlib import nullcontext -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import TYPE_CHECKING, Any @@ -194,6 +200,18 @@ class _OnMachine: """The run's directory under the entry's root, made there and resolved to an absolute path.""" link: Any """The machine's node pool: the launch, the prompt file, the frames and the end.""" + skey: str = "" + """The dispatch's session key, for the run's record (:mod:`raven.acp_client.remote_runs`).""" + handle: str = "" + """The dispatch's handle, for the record.""" + authored_task: str = "" + """The task as the dispatching model wrote it, for the record: what an announce after a restart quotes.""" + + +def _untrack(tracker: Any, *, keep: bool) -> None: + """The run's record on this computer goes with a turn that ended here, or stays for the next process.""" + if tracker is not None: + tracker.close(keep=keep) class CliAgentBackend: @@ -661,6 +679,9 @@ async def _exec_there( ) except remote_path.NodeCallError as exc: raise RuntimeError(f"CLI agent {self.name!r}: {exc}") from None + # The run's record on this computer: for the process after this one + # to pick up, should this one stop while the command runs. + tracker = self._track_on_machine(endpoint, placed, task_id=task_id, agent_id=agent_id) out_lines: list[str] = [] err_lines: list[str] = [] publish_out_line = self._line_publisher(on_delta) @@ -693,12 +714,29 @@ async def drive() -> int | None: rc = await asyncio.wait_for(drive(), timeout=self.timeout) if self.timeout is not None else await drive() except asyncio.TimeoutError: await endpoint.end() + _untrack(tracker, keep=False) raise CliAgentTimeoutError(f"CLI agent {self.name!r} timed out after {self.timeout}s") from None + except asyncio.CancelledError: + # While this process stops, the command stays with the machine's + # daemon (``ChannelEndpoint.end``) and its record with this + # computer, for the next process to pick up, and the manager + # reads the dispatch as left, not cancelled. Any other cancel + # ends the command there. See ``_kill_process_group``. + left = tracker is not None and bool(getattr(endpoint, "stays", False)) + await endpoint.end() + if left: + from raven.acp_client import remote_runs + + remote_runs.mark_left(task_id) + _untrack(tracker, keep=left) + raise except BaseException: - # A cancel, or a delta sink that raised: either way nothing may - # be left running on the machine. See ``_kill_process_group``. + # A delta sink that raised: nothing may be left running on the + # machine. await endpoint.end() + _untrack(tracker, keep=False) raise + _untrack(tracker, keep=False) if (why := endpoint.ended()) is not None: raise RuntimeError(f"CLI agent {self.name!r}: {why}") stdout = "".join(f"{line}\n" for line in out_lines) @@ -830,6 +868,7 @@ async def run( runtime_env=runtime_env, mcp_file=mcp_path, media=media, + authored_task=authored_task, ) for note in filter(None, (self._withheld_note(grant), grant.note_text())): notice = f"\n\n[raven] {note}." @@ -897,6 +936,7 @@ async def _dispatch( runtime_env: dict[str, str] | None = None, mcp_file: str | None = None, media: Sequence[Media] = (), + authored_task: str | None = None, ) -> str: skey = session_key or "default" if self.machine: @@ -910,6 +950,7 @@ async def _dispatch( on_delta=on_delta, notice_sink=notice_sink, media=media, + authored_task=authored_task, ) cwd = self.cwd or str(workspace) # Here rather than beside either launch: this lane sees nothing of what @@ -947,6 +988,7 @@ async def _dispatch_there( on_delta: Callable[[str], Awaitable[None]] | None, notice_sink: Callable[[str], Awaitable[None]] | None, media: Sequence[Media], + authored_task: str | None = None, ) -> str: """The run on the entry's machine: placed there first, with the task's files named by address. @@ -957,7 +999,7 @@ async def _dispatch_there( """ from raven.acp_client import remote - placed = await self._on_machine(handle) + placed = replace(await self._on_machine(handle), skey=skey, handle=handle, authored_task=authored_task or "") # Nothing is carried to the machine: the files that come with the # task are named to the agent by address, and it looks for them there. if note := remote.files_note(media, placed.target): @@ -1144,6 +1186,167 @@ async def _run_stateful( placed=placed, ) + def _transcript(self, stdout: str) -> tuple[str | None, str | None]: + """The session id and the reply the command's transcript carries, by its format; ``None`` each when it has none.""" + if self.transcript_format == "codex_jsonl": + return parse_codex_jsonl(stdout) + if self.transcript_format == "claude_stream_json": + jsonl_id, jsonl_reply, is_error = parse_claude_stream_json(stdout) + # Claude does not reliably exit non-zero under -p, so the exit code + # check is not enough on its own. + if is_error: + raise CliAgentReportedError( + f"CLI agent {self.name!r} reported an error: {(jsonl_reply or stdout).strip()[-2000:]}" + ) + return jsonl_id, jsonl_reply + if self.transcript_format == "openclaw_json": + return parse_openclaw_json(stdout) + if self.transcript_format == "opencode_json": + return parse_opencode_json(stdout) + return None, None + + def _reply_from(self, stdout: str, stderr: str, parsed: str | None) -> str: + """The answer in what the command wrote: the transcript's reply when its format gives one, else the matched or whole stdout. + + stderr is a log lane, not part of the answer: it streams to the live + console and rides in the attempts record, and folding it into a + successful reply is what forced quiet launchers to bury their + progress in files instead of logging it. It still stands in when + stdout said nothing at all -- an answerless run's stderr is the only + account it left. + """ + if parsed is not None: + return parsed.strip() + if self._output_re is not None and (m := self._output_re.search(stdout)) is not None: + return m.group(1).strip() + return stdout.strip() or stderr.strip() + + def _track_on_machine(self, endpoint: Any, placed: _OnMachine, *, task_id: str, agent_id: str | None) -> Any: + """The record of this run on its machine (``remote_runs.Tracker``), or ``None`` for an endpoint that is not a node's. + + A cli run's reply is the whole of its output, read from the start, + so the record never moves past frame 0: the pick-up replays every + line (``pick_up``). The session id is the agent id the launch minted, + when it minted one; an id derived from the output is not known here. + """ + from raven.acp_client import remote_runs + + task = getattr(endpoint, "task", None) + if not isinstance(task, str): + return None + run = remote_runs.RemoteRun( + kind="cli", + machine=self.machine or "", + task=task, + agent=self.name, + session_key=placed.skey, + handle=placed.handle, + task_id=task_id, + session_id=agent_id or "", + prompt_id=0, + next_id=0, + initialize=None, + launch_key="", + config_key="", + session_cwd=placed.session_cwd, + stateful=self.is_stateful, + authored_task=placed.authored_task, + ) + try: + return remote_runs.Tracker(run) + except OSError as exc: + logger.warning( + "CLI agent {!r}: the run on machine {} could not be recorded: {}", self.name, self.machine, exc + ) + return None + + async def pick_up(self, run: Any, *, pool: Any = None) -> Any: + """The output the command in ``run`` owes, read off its machine after this computer restarted; or why there is none. + + What :mod:`raven.agent.subagent.resume_remote` asks of a cli entry for + a run a process before this one left on the machine + (:class:`raven.acp_client.remote_runs.RemoteRun`, kind ``cli``). The + machine's daemon replays every line the command wrote and hands the + rest over as it comes -- a cli run's reply is the whole of its + output, read from the start -- and the exit code ends it, as a run + here ends (``_exec_there``). Raises + :class:`raven.acp_client.remote_resume.UnreachableError` when the + machine cannot be asked, or this computer stops under the pick-up, + so the record stays for a later start; every other outcome is a + ``PickedUp``, and the record is the caller's to remove. + """ + from raven.acp_client import remote, remote_resume + from raven.ops import remote_path + + link = pool if pool is not None else remote_path.nodes() + if link is None: + raise remote_resume.UnreachableError( + f"this process does not reach registered machines, so the run on machine {run.machine} waits" + ) + try: + target = remote.machine(run.machine) + except remote.RemoteMachineError as exc: + return remote_resume.PickedUp("error", f"the run of {run.agent!r} cannot be picked up: {exc}") + try: + rows = await link.tasks(run.machine) + except remote_path.NodeCallError as exc: + raise remote_resume.UnreachableError(str(exc)) from None + if not any(row.get("task") == run.task for row in rows): + return remote_resume.PickedUp( + "error", + f"machine {target.label} no longer has the run of {run.agent!r} (task {run.task}); " + "its record there was removed, or the machine's node directory was", + ) + try: + endpoint, _attached = await link.attach(run.machine, run.task, name=run.agent, after=0) + except remote_path.NodeCallError as exc: + if exc.reason in ("unknown_task", None) and "no longer runs" in str(exc): + return remote_resume.PickedUp("error", str(exc)) + raise remote_resume.UnreachableError(str(exc)) from None + out_lines: list[str] = [] + err_lines: list[str] = [] + + async def read_out() -> None: + while raw := await endpoint.readline(): + out_lines.append(raw.decode("utf-8", "replace").rstrip("\n")) + + async def read_err() -> None: + while raw := await endpoint.readerr(): + err_lines.append(raw.decode("utf-8", "replace").rstrip("\n")) + + try: + await asyncio.gather(read_out(), read_err()) + rc = await endpoint.wait() + except BaseException: + await endpoint.end() + raise + if (why := endpoint.ended()) is not None: + if remote.leaving(): + raise remote_resume.UnreachableError( + f"this computer stopped while the run of {run.agent!r} on machine {target.label} was being " + "picked up; it waits for the next start" + ) + return remote_resume.PickedUp( + "error", f"the run of {run.agent!r} on machine {target.label} ended while this computer was away: {why}" + ) + stdout = "".join(f"{line}\n" for line in out_lines) + stderr = "".join(f"{line}\n" for line in err_lines) + if rc != 0: + said = _human_failure(stdout, stderr) + return remote_resume.PickedUp( + "error", f"CLI agent {run.agent!r} exited {rc}" + (f": {said}" if said else "") + ) + try: + _jsonl_id, parsed = self._transcript(stdout) + except CliAgentReportedError as exc: + return remote_resume.PickedUp("error", str(exc)) + output = await clamp_output(self._reply_from(stdout, stderr, parsed), self.max_output_chars, agent=self.name) + return remote_resume.PickedUp( + "ok", + output + f"\n\n[raven] Ran on machine {target.label}, in {remote.shown(target, run.session_cwd)}, " + f"{remote_resume.PICKED_UP_NOTE}; the files it wrote are there, not on this computer.", + ) + async def _attempt( self, task: str, @@ -1180,22 +1383,7 @@ async def _attempt( mcp_file=mcp_file, ) - jsonl_id: str | None = None - jsonl_reply: str | None = None - if self.transcript_format == "codex_jsonl": - jsonl_id, jsonl_reply = parse_codex_jsonl(stdout) - elif self.transcript_format == "claude_stream_json": - jsonl_id, jsonl_reply, is_error = parse_claude_stream_json(stdout) - # Claude does not reliably exit non-zero under -p, so the exit code - # check above is not enough on its own. - if is_error: - raise CliAgentReportedError( - f"CLI agent {self.name!r} reported an error: {(jsonl_reply or stdout).strip()[-2000:]}" - ) - elif self.transcript_format == "openclaw_json": - jsonl_id, jsonl_reply = parse_openclaw_json(stdout) - elif self.transcript_format == "opencode_json": - jsonl_id, jsonl_reply = parse_opencode_json(stdout) + jsonl_id, jsonl_reply = self._transcript(stdout) if created and self.id_source == "derived": if jsonl_id is not None: @@ -1214,18 +1402,7 @@ async def _attempt( if created and agent_id is not None: await self._registry.commit(skey, self.name, handle, agent_id) - if jsonl_reply is not None: - output = jsonl_reply.strip() - elif self._output_re is not None and (m := self._output_re.search(stdout)) is not None: - output = m.group(1).strip() - else: - # stderr is a log lane, not part of the answer: it streams to the - # live console and rides in the attempts record, and folding it - # into a successful reply is what forced quiet launchers to bury - # their progress in files instead of logging it. It still stands - # in when stdout said nothing at all -- an answerless run's stderr - # is the only account it left. - output = stdout.strip() or stderr.strip() + output = self._reply_from(stdout, stderr, jsonl_reply) warning = None if created and self.id_source == "derived" and agent_id is None: diff --git a/raven/agent/subagent/manager.py b/raven/agent/subagent/manager.py index 2348f62c0..882283d5a 100644 --- a/raven/agent/subagent/manager.py +++ b/raven/agent/subagent/manager.py @@ -1897,6 +1897,10 @@ async def _run_subagent_inner( # the next process picks the run up and does both # (raven.agent.subagent.resume_remote). left = True + # The record the backend wrote lacks what only this side + # knows and the announce after a restart shows: the + # dispatch's origin (instance, node, tool call) and summary. + remote_runs.annotate(task_id, origin=origin, task_summary=task_summary) logger.info( "Subagent [{}] stays running on its machine; picked up when this computer starts again", task_id ) @@ -2092,8 +2096,12 @@ async def _announce_result( status: str, record_path: str | None = None, activity: Any = None, - ) -> None: - """Announce the subagent result to the main agent via the spine. + ) -> bool: + """Announce the subagent result to the main agent via the spine; whether a turn took it. + + ``False`` when the scheduler is draining and no turn could carry the + words (``_inject``): what a caller that can report again later reads + (the pick-up after a restart, :mod:`raven.agent.subagent.resume_remote`). Note: this inbound system message already triggers a main-agent turn, so subagent completion is event-driven end-to-end. Do NOT also @@ -2176,7 +2184,7 @@ async def _announce_result( # The delivered marker draws the seam where a result re-entered its # conversation; a refused inject re-entered nothing, so there is none. if not self._inject(announce_content, origin, mark): - return + return False # `content` is the text that was injected, verbatim. A client draws the # reader-facing part of it by dropping everything outside the untrusted # fence -- and a client REPLAYING this turn later reads the same string @@ -2184,6 +2192,7 @@ async def _announce_result( # disagree about what was delivered. self._emit_delivered(origin, {**mark, "content": announce_content}) logger.debug("Subagent [{}] announced result to {}", task_id, origin["session_key"]) + return True async def _report_undispatched_cancel( self, task_id: str, task_summary: str, task: str, origin: dict[str, Any], reason: str diff --git a/raven/agent/subagent/resume_remote.py b/raven/agent/subagent/resume_remote.py index fdc98d994..b513ab8ed 100644 --- a/raven/agent/subagent/resume_remote.py +++ b/raven/agent/subagent/resume_remote.py @@ -8,7 +8,10 @@ closed, and the result is announced into the conversation that asked, as a turn that ended here would have been. A machine that cannot be asked keeps its record for the next start; so does a stop of this computer that lands -under a pick-up. Each run is picked up under a claim on its record +under a pick-up, a conclusion that fails here, or an announce the scheduler +refuses: the record goes last, once the outcome has reached its conversation. +A cli entry's run is picked up the same way, its command's whole output being +its reply (``CliAgentBackend.pick_up``). Each run is picked up under a claim on its record (:func:`raven.acp_client.remote_runs.claim`): two processes of one Raven home that start together pick it up once, not once each. """ @@ -24,6 +27,7 @@ from raven.acp_client import remote, remote_resume, remote_runs from raven.acp_client.acp_agent import AcpAgentBackend +from raven.agent.subagent.backends.cli_agent import CliAgentBackend from raven.agent.subagent.history import SpawnRecord, nodes_root from raven.agent.subagent.instances import get_registry @@ -55,7 +59,14 @@ async def _later() -> None: async def resume_remote_runs(manager: Any) -> int: - """Every run left on a machine, picked up and reported; how many were concluded here.""" + """Every run left on a machine, picked up and reported; how many were concluded here. + + A run's record goes only once its outcome has reached its conversation: + a machine that cannot be asked, a stop of this computer under the + pick-up, a conclusion that fails here or an announce the scheduler + refuses all leave the record, its claim released, for the next start to + try again. + """ concluded = 0 for run in remote_runs.pending(): if remote.leaving(): @@ -70,43 +81,75 @@ async def resume_remote_runs(manager: Any) -> int: continue backend = _backend_of(manager, run) if backend is None: - await _conclude( - manager, - run, + picked = remote_resume.PickedUp( "error", f"the run of {run.agent!r} left on machine {run.machine} cannot be picked up: the agent is no " - "longer on the agent table as an acp agent on that machine; its frames are on the machine, " + f"longer on the agent table as an {run.kind} agent on that machine; its frames are on the machine, " f"task {run.task}", ) - concluded += 1 - continue + else: + try: + picked = await _pick_up(run, backend) + except remote_resume.UnreachableError as exc: + remote_runs.release(run.task) + logger.warning("the run of {!r} on machine {} waits for a later start: {}", run.agent, run.machine, exc) + continue + except Exception as exc: # noqa: BLE001 - one run's failure is reported as its own, the next is tried + logger.opt(exception=True).warning( + "picking up the run of {!r} on machine {} failed", run.agent, run.machine + ) + picked = remote_resume.PickedUp("error", f"the run of {run.agent!r} could not be picked up: {exc}") try: - picked = await remote_resume.pick_up(run, backend) - except remote_resume.UnreachableError as exc: + taken = await _conclude(manager, run, picked.status, picked.reply) + except Exception: # noqa: BLE001 - the outcome is recovered and not yet reported: the next start reports it remote_runs.release(run.task) - logger.warning("the run of {!r} on machine {} waits for a later start: {}", run.agent, run.machine, exc) - continue - except Exception as exc: # noqa: BLE001 - one run's failure is reported as its own, the next is tried logger.opt(exception=True).warning( - "picking up the run of {!r} on machine {} failed", run.agent, run.machine + "concluding the run of {!r} on machine {} failed; its record stays for the next start", + run.agent, + run.machine, ) - picked = remote_resume.PickedUp("error", f"the run of {run.agent!r} could not be picked up: {exc}") - await _conclude(manager, run, picked.status, picked.reply) + continue + except BaseException: + # A stop under the conclusion: the record stays, and the claim + # names a process that is about to be gone. + remote_runs.release(run.task) + raise + if not taken: + remote_runs.release(run.task) + logger.warning( + "the run of {!r} on machine {} was not announced, the scheduler is draining; " + "its record stays for the next start", + run.agent, + run.machine, + ) + continue + remote_runs.forget(run.task) concluded += 1 return concluded -def _backend_of(manager: Any, run: remote_runs.RemoteRun) -> AcpAgentBackend | None: +async def _pick_up(run: remote_runs.RemoteRun, backend: Any) -> remote_resume.PickedUp: + """The reply the run owes, read the way its kind is read: a cli entry its command's output, an acp one its turn.""" + if run.kind == "cli": + return await backend.pick_up(run) + return await remote_resume.pick_up(run, backend) + + +def _backend_of(manager: Any, run: remote_runs.RemoteRun) -> Any: registry = getattr(manager, "registry", None) backend = registry.backend(run.agent) if registry is not None else None - if not isinstance(backend, AcpAgentBackend) or backend.machine != run.machine: + wanted = CliAgentBackend if run.kind == "cli" else AcpAgentBackend + if not isinstance(backend, wanted) or backend.machine != run.machine: return None return backend -async def _conclude(manager: Any, run: remote_runs.RemoteRun, status: str, reply: str) -> None: - """Close what the dispatch left open here and announce the outcome into its conversation.""" - remote_runs.forget(run.task) +async def _conclude(manager: Any, run: remote_runs.RemoteRun, status: str, reply: str) -> bool: + """Close what the dispatch left open here and announce the outcome into its conversation; whether it was taken. + + The record is the caller's to remove, after this: a conclusion cut short + leaves it for the next start. + """ ok = status == "ok" session_dir = Path(manager.session_dir_for(run.session_key)) record, meta = _spawn_record(session_dir, run) @@ -117,8 +160,8 @@ async def _conclude(manager: Any, run: remote_runs.RemoteRun, status: str, reply record.finish(status="failed", error=f"Error: {reply}") await manager._finalize_node_claim(run.session_key, record) try: - if ok and run.stateful: - await get_registry().commit(run.session_key, run.agent, run.handle, run.session_id, kind="acp") + if ok and run.stateful and run.session_id: + await get_registry().commit(run.session_key, run.agent, run.handle, run.session_id, kind=run.kind) else: await get_registry().upsert_spawn(run.session_key, run.agent, run.handle, "completed" if ok else "failed") except Exception: # noqa: BLE001 - the registry row must not keep the result from its reader @@ -126,25 +169,46 @@ async def _conclude(manager: Any, run: remote_runs.RemoteRun, status: str, reply channel, sep, chat_id = run.session_key.partition(":") if not (sep and channel and chat_id): logger.warning("the run of {!r} has no conversation to report to (session {!r})", run.agent, run.session_key) - return + return True manager.remember_session(run.session_key) - origin = {"session_key": run.session_key, "channel": channel, "chat_id": chat_id} - summary = str(meta.get("task_summary") or f"{run.agent} on machine {run.machine}") - task_text = "" - if record is not None: - try: - task_text = record.file("prompt.md").read_text(encoding="utf-8") - except OSError: - task_text = "" - await manager._announce_result( + origin = _origin_of(run, record, meta, channel=channel, chat_id=chat_id) + summary = str(run.task_summary or meta.get("task_summary") or f"{run.agent} on machine {run.machine}") + # The task as the model wrote it, or the summary: never the rendered + # prompt on disk, which a `{{ ref:... }}` spawn expands a whole file into. + asked = str(run.authored_task or origin.get("authored_task") or summary) + taken = await manager._announce_result( run.task_id, summary, - task_text, + asked, reply if ok else f"Error: {reply}", origin, "ok" if ok else "error", record_path=str(record.file("out.md")) if record is not None else None, ) + return taken is not False + + +def _origin_of( + run: remote_runs.RemoteRun, record: SpawnRecord | None, meta: dict[str, Any], *, channel: str, chat_id: str +) -> dict[str, Any]: + """The dispatch's origin for the announce: as the manager added it to the record, or made up from what is left. + + What ``spawn`` builds -- instance, handle, node, tool call, the task as + authored -- the manager adds to the record when the run is left + (:func:`raven.acp_client.remote_runs.annotate`). A host killed outright + never did: then the spawn record names the node, a handle that is not + the dispatch's id is the instance, and the conversation is read off the + session key. + """ + origin: dict[str, Any] = dict(run.origin) if isinstance(run.origin, dict) else {} + origin.setdefault("agent", run.agent) + origin.setdefault("handle", run.handle) + origin.setdefault("instance", run.handle if run.handle != run.task_id else None) + origin.setdefault("node_id", (record.node_id if record is not None else None) or meta.get("call_id") or run.task_id) + if run.authored_task: + origin.setdefault("authored_task", run.authored_task) + origin.update({"channel": channel, "chat_id": chat_id, "session_key": run.session_key}) + return origin def _spawn_record(session_dir: Path, run: remote_runs.RemoteRun) -> tuple[SpawnRecord | None, dict[str, Any]]: diff --git a/raven/cli/tui_commands.py b/raven/cli/tui_commands.py index ec047fb8f..3383f1556 100644 --- a/raven/cli/tui_commands.py +++ b/raven/cli/tui_commands.py @@ -446,6 +446,13 @@ def _agent_loop_factory(): ) # Subagent result re-injection submits a SUBAGENT-origin turn. agent_loop.subagents.set_submit(turn_scheduler.submit) + # The runs a process before this one left on registered machines are + # picked up and reported to their conversations, as the gateway and + # the served page do; after the submit is wired, so the announce has + # a spine to go to. + from raven.agent.subagent.resume_remote import schedule_remote_resume + + schedule_remote_resume(agent_loop.subagents) # Cron reminders run as CRON turns through the scheduler; the wrapper fans # the reply out as cron.delivered to every session. on_job must be wired # before cron.start() so an immediately-firing job has its callback. @@ -571,7 +578,11 @@ async def _start_backend() -> None: # bounded wait, where the drain costs its whole budget. try: from raven.acp_client.client import begin_drain + from raven.acp_client.remote import leave_agents_running + # The agents on registered machines stay with their daemons (see + # the gateway's shutdown): the next process picks their runs up. + leave_agents_running() begin_drain() if agent_loop is not None: await agent_loop.subagents.cancel_all(reason="the TUI exited") diff --git a/tests/test_cli_tui_commands.py b/tests/test_cli_tui_commands.py index a261f7738..5a3ec0591 100644 --- a/tests/test_cli_tui_commands.py +++ b/tests/test_cli_tui_commands.py @@ -1330,3 +1330,31 @@ async def test_the_terminal_host_starts_the_heap_trimmer(monkeypatch: pytest.Mon await _run_until_done_with_immediate_proc_done(monkeypatch, rpc_server_deps) assert started == [1] + + +async def test_the_tui_host_leaves_the_agents_on_machines_running_at_exit_and_picks_them_up_at_start( + monkeypatch: pytest.MonkeyPatch, rpc_server_deps +) -> None: + """As the gateway and the served page do: the stop says the agents stay before it cancels, the start schedules the pick-up.""" + from raven.acp_client import remote + + ctx = rpc_server_deps + leaving_at_cancel: list[bool] = [] + + async def cancel_all(**kwargs: Any) -> int: + leaving_at_cancel.append(remote.leaving()) + return 0 + + ctx["agent_loop"].subagents.cancel_all = cancel_all + scheduled: list[Any] = [] + monkeypatch.setattr( + "raven.agent.subagent.resume_remote.schedule_remote_resume", + lambda subagents: scheduled.append(subagents) or None, + ) + try: + await _run_until_done_with_immediate_proc_done(monkeypatch, ctx) + finally: + remote.staying() + + assert leaving_at_cancel == [True], "said before the cancel, so a run on a machine is left, not ended" + assert scheduled == [ctx["agent_loop"].subagents] diff --git a/tests/test_remote_runs.py b/tests/test_remote_runs.py index 3e33539ae..9e418dcc8 100644 --- a/tests/test_remote_runs.py +++ b/tests/test_remote_runs.py @@ -225,3 +225,46 @@ def no_links(src, dst, *a, **k): monkeypatch.setattr(remote_runs, "runs_dir", lambda: tmp_path / "a-file" / "runs") assert remote_runs.claim("nowhere") is False assert "a claim could not be made" in "\n".join(logs) + + +def test_a_record_carries_what_the_announce_after_a_restart_needs_and_an_older_one_reads_with_defaults( + _home: Path, +) -> None: + """``kind``, the task as authored, the summary and the dispatch's origin ride the record; one without them reads with defaults.""" + origin = {"node_id": "n1", "instance": None, "handle": "t1"} + remote_runs.record( + _run( + "echo@box-cli1", + kind="cli", + session_id="", + authored_task="count the rows", + task_summary="count", + origin=origin, + ) + ) + older = {k: v for k, v in _run().as_json().items() if k not in ("kind", "authored_task", "task_summary", "origin")} + (remote_runs.runs_dir() / "echo@box-1a2b3c4d.json").write_text(json.dumps(older), encoding="utf-8") + + back = {run.task: run for run in remote_runs.pending()} + + cli = back["echo@box-cli1"] + assert (cli.kind, cli.authored_task, cli.task_summary, cli.origin) == ("cli", "count the rows", "count", origin) + acp = back["echo@box-1a2b3c4d"] + assert (acp.kind, acp.authored_task, acp.task_summary, acp.origin) == ("acp", "", "", None) + + +def test_a_cli_record_needs_no_acp_session_and_an_acp_one_still_does() -> None: + assert remote_runs.RemoteRun.from_json(_run(kind="cli", session_id="").as_json()) is not None + assert remote_runs.RemoteRun.from_json(_run(session_id="").as_json()) is None + + +def test_annotate_adds_the_manager_s_fields_to_the_record_of_a_dispatch_and_says_when_there_is_none( + _home: Path, +) -> None: + remote_runs.record(_run(task_id="d1")) + + assert remote_runs.annotate("d1", origin={"node_id": "n1", "workspace": Path("/w")}, task_summary="say hi") is True + + (back,) = remote_runs.pending() + assert back.origin == {"node_id": "n1", "workspace": "/w"} and back.task_summary == "say hi" + assert remote_runs.annotate("nobody", task_summary="x") is False diff --git a/tests/test_resume_remote.py b/tests/test_resume_remote.py index 2f4772bdf..83c8c2403 100644 --- a/tests/test_resume_remote.py +++ b/tests/test_resume_remote.py @@ -139,10 +139,18 @@ async def picked(got_run, got_backend, *, pool=None): assert manager.finalized == ["n1"] and manager.remembered == ["tui:s1"] assert registry.committed == [("tui:s1", "stub@box", "t1", "sess-1", "acp")] (said,) = manager.announced - assert said["status"] == "ok" and said["result"].startswith("pong") and said["task"] == "say hi to the machine" - assert ( - said["origin"] == {"session_key": "tui:s1", "channel": "tui", "chat_id": "s1"} and said["summary"] == "say hi" - ) + # The summary stands for the task: nothing authored was recorded, and the prompt file is never quoted. + assert said["status"] == "ok" and said["result"].startswith("pong") and said["task"] == "say hi" + assert said["origin"] == { + "session_key": "tui:s1", + "channel": "tui", + "chat_id": "s1", + "agent": "stub@box", + "handle": "t1", + "instance": None, + "node_id": "n1", + } + assert said["summary"] == "say hi" assert said["record"].endswith("n1.out.md") assert remote_runs.pending() == [] @@ -325,3 +333,143 @@ async def picked(run, backend, *, pool=None): assert [run.task for run in remote_runs.pending()] == ["first", "second"] assert remote_runs.claim("first") and remote_runs.claim("second"), "nothing is left claimed" assert "wait for the next start" in "\n".join(logs) + + +def _picked(status: str, reply: str): + async def pick_up(run, backend, *, pool=None): + return remote_resume.PickedUp(status, reply) + + return pick_up + + +async def test_a_resumed_announce_carries_the_dispatch_s_origin_and_quotes_the_task_as_authored(tmp_path, monkeypatch): + """What the announce here would have said: instance, handle, node and the task as the model wrote it, never the rendered prompt.""" + backend = _backend() + session_dir = tmp_path / "s1" + record = _open_record(session_dir, handle="r1") + record.file("prompt.md").write_text( + "say hi to the machine\n\n", encoding="utf-8" + ) + origin = { + "channel": "tui", + "chat_id": "s1", + "session_key": "tui:s1", + "agent": "stub@box", + "instance": "r1", + "instance_auto": False, + "handle": "r1", + "workspace": "/w", + "authored_task": "say hi {{ ref:notes.md }}", + "tool_call_id": "call-9", + "node_id": "n1", + } + run = _run(handle="r1", authored_task="say hi {{ ref:notes.md }}", task_summary="say hi", origin=origin) + manager = _Manager(backend, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + monkeypatch.setattr(remote_resume, "pick_up", _picked("ok", "pong")) + + assert await resume_remote.resume_remote_runs(manager) == 1 + + (said,) = manager.announced + assert said["task"] == "say hi {{ ref:notes.md }}" and said["summary"] == "say hi" + assert said["origin"]["instance"] == "r1" and said["origin"]["node_id"] == "n1" + assert said["origin"]["tool_call_id"] == "call-9" and said["origin"]["authored_task"] == "say hi {{ ref:notes.md }}" + assert said["origin"]["chat_id"] == "s1" and said["task_id"] == run.task_id + + +async def test_without_the_manager_s_fields_the_origin_comes_from_the_record_and_the_prompt_file_is_never_quoted( + tmp_path, monkeypatch +): + """A host killed outright never annotated the record: the spawn record names the node, the handle is the instance, the summary stands for the task.""" + backend = _backend() + session_dir = tmp_path / "s1" + record = _open_record(session_dir, handle="r1", node_id="n7") + record.file("prompt.md").write_text("say hi\n\n", encoding="utf-8") + _run(handle="r1") + manager = _Manager(backend, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + monkeypatch.setattr(remote_resume, "pick_up", _picked("ok", "pong")) + + assert await resume_remote.resume_remote_runs(manager) == 1 + + (said,) = manager.announced + assert said["task"] == "say hi" and "rendered" not in said["task"] + assert said["origin"]["node_id"] == "n7" and said["origin"]["instance"] == "r1" and said["origin"]["handle"] == "r1" + assert said["origin"]["agent"] == "stub@box" and said["origin"]["session_key"] == "tui:s1" + + +async def test_the_record_goes_only_once_the_announce_is_taken_and_stays_when_the_scheduler_refuses_it( + tmp_path, monkeypatch +): + backend = _backend() + session_dir = tmp_path / "s1" + _open_record(session_dir) + run = _run() + manager = _Manager(backend, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + monkeypatch.setattr(remote_resume, "pick_up", _picked("ok", "pong")) + seen: list[list[str]] = [] + + async def refused(*args, **kwargs): + seen.append([kept.task for kept in remote_runs.pending()]) + return False + + manager._announce_result = refused + + assert await resume_remote.resume_remote_runs(manager) == 0 + + assert seen == [[run.task]], "the record is still there while the announce runs" + assert [kept.task for kept in remote_runs.pending()] == [run.task], "and stays for the next start" + assert remote_runs.claim(run.task), "with its claim released" + + +async def test_a_conclusion_that_fails_keeps_the_record_and_releases_the_claim(tmp_path, monkeypatch, logs): + backend = _backend() + session_dir = tmp_path / "s1" + _open_record(session_dir) + run = _run() + manager = _Manager(backend, session_dir) + + async def boom(session_key, record): + raise RuntimeError("the index is locked") + + manager._finalize_node_claim = boom + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + monkeypatch.setattr(remote_resume, "pick_up", _picked("ok", "pong")) + + assert await resume_remote.resume_remote_runs(manager) == 0 + + assert [kept.task for kept in remote_runs.pending()] == [run.task] and manager.announced == [] + assert remote_runs.claim(run.task) + assert any("its record stays for the next start" in line for line in logs) + + +async def test_a_cli_run_is_picked_up_by_its_cli_entry_and_an_acp_record_is_not_a_cli_entry_s(tmp_path, monkeypatch): + from raven.agent.subagent.backends import build_third_party_backend + from raven.config.schema import ThirdPartyCliSubagentConfig + + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") + cli = build_third_party_backend( + ThirdPartyCliSubagentConfig(name="sh@box", command="cat", machine="box", remote_cwd="~/raven-work") + ) + picked: list[str] = [] + + async def pick_up(run, *, pool=None): + picked.append(run.task) + return remote_resume.PickedUp("ok", "one\ntwo") + + monkeypatch.setattr(cli, "pick_up", pick_up) + session_dir = tmp_path / "s1" + _open_record(session_dir, agent="sh@box") + run = _run( + kind="cli", agent="sh@box", session_id="", prompt_id=0, next_id=0, initialize=None, launch_key="", config_key="" + ) + manager = _Manager(cli, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + + assert await resume_remote.resume_remote_runs(manager) == 1 + assert picked == [run.task] and manager.announced[0]["result"] == "one\ntwo" + + _run(task="stub@box-2", agent="sh@box") + assert await resume_remote.resume_remote_runs(manager) == 1 + assert "no longer on the agent table as an acp agent" in manager.announced[-1]["result"] diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py index 88248fef3..064070d8c 100644 --- a/tests/test_subagent_cli_remote.py +++ b/tests/test_subagent_cli_remote.py @@ -710,3 +710,251 @@ def get_default_model(self) -> str: mgr = SubagentManager(provider=_Provider(), workspace=Path("/tmp"), agents=[cli_config("cat")]) assert mgr._machine_workdir("where@box", "reviewer") == ("box", "~/raven-work/reviewer") + + +async def test_a_cli_run_left_on_the_machine_when_this_computer_stops_is_picked_up_by_the_next_start( + box, tmp_path, monkeypatch +): + """The command goes on under the machine's daemon when this process stops; the next one reads its output and reports it.""" + from raven.acp_client import remote, remote_runs + from raven.agent.subagent import resume_remote + from raven.agent.subagent.history import SpawnRecord, nodes_root + from tests.test_subagent_acp_remote import _Manager, _Registry + + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + backend = build_third_party_backend(cli_config("sh -c 'echo $$ > pid.txt; sleep 2; echo done; cat'")) + turn = asyncio.ensure_future( + backend.run( + "hi", + task_id="t1", + workspace=tmp_path, + executor=None, + session_key="tui:s1", + instance="left", + authored_task="say done {{ ref:notes.md }}", + ) + ) + pid_file = box.home / "raven-work" / "left" / "pid.txt" + await _until(pid_file.exists) + pid = int(pid_file.read_text(encoding="utf-8")) + (run,) = remote_runs.pending() + assert (run.kind, run.machine, run.agent, run.session_key, run.handle, run.task_id) == ( + "cli", + "box", + "where@box", + "tui:s1", + "left", + "t1", + ) + assert run.authored_task == "say done {{ ref:notes.md }}" and run.stateful is False + + # This computer stops: the turn is cancelled the way the shutdown does it, then the links are closed. + remote.leave_agents_running() + try: + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await turn + assert remote_runs.was_left("t1"), "the manager would read this dispatch as left, not cancelled" + await box.nodes.close_all() + finally: + remote.staying() + assert not _gone(pid), "the command stays with the machine's daemon" + assert [kept.task for kept in remote_runs.pending()] == [run.task] + + # The next start: the dispatch's record is still open under its session, as the manager leaves one. + session_dir = tmp_path / "sessions" / "s1" + meta = { + "call_id": "n1", + "session_key": "tui:s1", + "agent": backend.name, + "handle": "left", + "task_summary": "say done", + } + SpawnRecord.open(session_dir, task_id="t1", task="hi", meta=meta, node_id="n1") + manager = _Manager(backend, session_dir) + registry = _Registry() + monkeypatch.setattr(resume_remote, "get_registry", lambda: registry) + + assert await asyncio.wait_for(resume_remote.resume_remote_runs(manager), timeout=30) == 1 + + (said,) = manager.announced + assert said["status"] == "ok" and said["task_id"] == "t1" and said["origin"]["chat_id"] == "s1" + assert said["result"].split("\n\n[raven]")[0] == "done\nhi", "the command's whole output, read after the restart" + assert ( + "picked up after this computer restarted" in said["result"] + and "Ran on machine 'Lab box' (box)" in said["result"] + ) + meta_back = json.loads((nodes_root(session_dir) / "n1.meta.json").read_text(encoding="utf-8")) + assert meta_back["status"] == "completed" and manager.finalized == ["n1"] + assert registry.rows == [("status", "tui:s1", "where@box", "left", "completed")] + assert remote_runs.pending() == [] + await _until(lambda: _gone(pid)) + + +async def test_a_cancel_while_this_computer_goes_on_ends_the_command_and_keeps_no_record(box, tmp_path, monkeypatch): + from raven.acp_client import remote_runs + + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + turn = asyncio.ensure_future( + _run(cli_config("sh -c 'echo $$ > pid.txt; exec sleep 30'"), "hi", tmp_path, instance="gone") + ) + pid_file = box.home / "raven-work" / "gone" / "pid.txt" + await _until(pid_file.exists) + pid = int(pid_file.read_text(encoding="utf-8")) + assert [run.kind for run in remote_runs.pending()] == ["cli"], "recorded while it runs" + + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await turn + + await _until(lambda: _gone(pid)) + assert remote_runs.pending() == [] and not remote_runs.was_left("t1") + + +async def test_a_command_that_ends_here_leaves_no_record(box, tmp_path, monkeypatch): + from raven.acp_client import remote_runs + + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + + head, _ = _split(await _run(cli_config("cat"), "hi", tmp_path)) + + assert head == "hi" and remote_runs.pending() == [] + + +class _PickUpLink: + """A node pool with one task, whose attach hands over ``endpoint``: what a pick-up reads a left run off.""" + + def __init__(self, endpoint: Any, *, rows: Any = None, refuse: Exception | None = None) -> None: + self.endpoint = endpoint + self.rows = rows if rows is not None else [{"task": "where@box-1", "state": "running"}] + self.refuse = refuse + self.attached: list[tuple[str, str, int]] = [] + + async def tasks(self, machine_id: str) -> list[dict[str, Any]]: + if isinstance(self.rows, Exception): + raise self.rows + return self.rows + + async def attach(self, machine_id: str, task: str, *, name: str, after: int = 0) -> tuple[Any, dict[str, Any]]: + if self.refuse is not None: + raise self.refuse + self.attached.append((machine_id, task, after)) + return self.endpoint, {"pid": 7, "frames": 2, "replayed": 2, "rc": None} + + +def _left_run(**kw: Any) -> Any: + from raven.acp_client import remote_runs + + fields: dict[str, Any] = dict( + kind="cli", + machine="box", + task="where@box-1", + agent="where@box", + session_key="tui:s1", + handle="t1", + task_id="t1", + session_id="", + prompt_id=0, + next_id=0, + initialize=None, + launch_key="", + config_key="", + session_cwd="/home/worker/raven-work/t1", + stateful=False, + ) + fields.update(kw) + return remote_runs.RemoteRun(**fields) + + +def _left_endpoint(*lines: str, rc: int | None = 0, lost: str | None = None) -> Any: + """The channel a pick-up is handed, already fed what the daemon replays: ``lines``, then the exit or the loss.""" + from raven.acp_client import remote + from raven.node.client import ChannelEndpoint + + endpoint = ChannelEndpoint(_Link(), 1, target=remote.machine("box"), agent="where@box", task="where@box-1") # type: ignore[arg-type] + for n, line in enumerate(lines, 1): + endpoint.feed_frame(line, n) + if lost is not None: + endpoint.lost(lost) + elif rc is not None: + endpoint.exited(rc) + return endpoint + + +async def test_a_left_cli_run_is_read_whole_off_the_daemon_and_its_exit_ends_it(box): + """The pick-up replays every line from the first, the exit code ends it, and the reply closes with the machine note.""" + from raven.acp_client import remote_resume + + link = _PickUpLink(_left_endpoint("done", "hi", rc=0)) + + picked = await build_third_party_backend(cli_config("cat")).pick_up(_left_run(), pool=link) + + assert picked.status == "ok" and picked.reply.split("\n\n[raven]")[0] == "done\nhi" + assert link.attached == [("box", "where@box-1", 0)], ( + "from the first line: a cli run's reply is the whole of its output" + ) + assert remote_resume.PICKED_UP_NOTE in picked.reply and f"scp://worker@{HOST}:{PORT}//home/worker" in picked.reply + + +async def test_a_left_cli_run_that_exited_non_zero_is_an_error_with_its_one_line(box): + picked = await build_third_party_backend(cli_config("cat")).pick_up( + _left_run(), pool=_PickUpLink(_left_endpoint("token expired", rc=3)) + ) + + assert picked.status == "error" and picked.reply == "CLI agent 'where@box' exited 3: token expired" + + +async def test_a_left_cli_run_the_machine_cannot_be_asked_about_waits_for_a_later_start(box, monkeypatch): + from raven.acp_client import remote_resume + from raven.ops import remote_path + + backend = build_third_party_backend(cli_config("cat")) + monkeypatch.setattr(remote_path, "_NODES", None) + with pytest.raises(remote_resume.UnreachableError, match="does not reach registered machines"): + await backend.pick_up(_left_run()) + down = _PickUpLink(_left_endpoint(), rows=remote_path.NodeCallError("the link to machine 'Lab box' (box) is down")) + with pytest.raises(remote_resume.UnreachableError, match="is down"): + await backend.pick_up(_left_run(), pool=down) + silent = _PickUpLink( + _left_endpoint(), refuse=remote_path.NodeCallError("the Raven node did not answer agent/attach") + ) + with pytest.raises(remote_resume.UnreachableError, match="did not answer"): + await backend.pick_up(_left_run(), pool=silent) + + +async def test_a_left_cli_run_the_machine_no_longer_has_or_an_unregistered_machine_is_an_error_in_one_sentence(box): + from raven.ops import remote_path + + backend = build_third_party_backend(cli_config("cat")) + gone = _PickUpLink(_left_endpoint(), rows=[]) + picked = await backend.pick_up(_left_run(), pool=gone) + assert picked.status == "error" and "no longer has the run of 'where@box'" in picked.reply and gone.attached == [] + + refused = _PickUpLink( + _left_endpoint(), + refuse=remote_path.NodeCallError( + "machine 'Lab box' (box) no longer runs 'where@box' (task where@box-1)", reason="unknown_task" + ), + ) + picked = await backend.pick_up(_left_run(), pool=refused) + assert picked.status == "error" and "no longer runs" in picked.reply + + picked = await backend.pick_up(_left_run(machine="elsewhere"), pool=gone) + assert picked.status == "error" and "is not registered" in picked.reply + + +async def test_a_left_cli_run_whose_link_goes_is_said_so_or_kept_while_this_computer_is_leaving(box): + from raven.acp_client import remote, remote_resume + + backend = build_third_party_backend(cli_config("cat")) + picked = await backend.pick_up(_left_run(), pool=_PickUpLink(_left_endpoint("partial", lost="the link went"))) + assert picked.status == "error" and "ended while this computer was away: the link went" in picked.reply + + remote.leave_agents_running() + try: + with pytest.raises(remote_resume.UnreachableError, match="stopped while the run of 'where@box'"): + await backend.pick_up( + _left_run(), pool=_PickUpLink(_left_endpoint("partial", lost="left running: this computer stopped")) + ) + finally: + remote.staying() diff --git a/tests/test_subagent_manager.py b/tests/test_subagent_manager.py index 4cca140e4..1f0e34078 100644 --- a/tests/test_subagent_manager.py +++ b/tests/test_subagent_manager.py @@ -5235,3 +5235,71 @@ async def run(self, task: str, **kwargs: Any) -> str: ) assert not (nodes / "r1.out.md").exists() and not (nodes / "r1.error.md").exists() assert submitted == [] and finalized == [], "nothing is announced or closed here" + + +async def test_a_dispatch_left_on_its_machine_has_its_origin_and_summary_added_to_its_record_for_the_next_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """What the next host's announce needs and the backend never had: written when this host learns the run was left.""" + from raven.acp_client import remote_runs + from raven.agent.subagent.spawn_tool import SpawnTool + from raven.config.schema import ThirdPartyCliSubagentConfig + from raven.session.manager import SessionManager + + home = tmp_path / "home" + monkeypatch.setenv("RAVEN_HOME", str(home)) + mgr = SubagentManager( + provider=_StubProvider(), + workspace=home, + session_dir=lambda key: SessionManager(home).session_dir(key), + max_concurrent=2, + agents=[ThirdPartyCliSubagentConfig(name="Remote", command="cat {prompt_file}")], + ) + mgr.set_submit(lambda request: None) + started = asyncio.Event() + + class _OnMachine: + async def run(self, task: str, *, task_id: str, authored_task: str | None = None, **kwargs: Any) -> str: + remote_runs.record( + remote_runs.RemoteRun( + kind="cli", + machine="box", + task=f"remote-{task_id}", + agent="Remote", + session_key=kwargs["session_key"], + handle=kwargs.get("instance") or task_id, + task_id=task_id, + session_id="", + prompt_id=0, + next_id=0, + initialize=None, + launch_key="", + config_key="", + session_cwd="/root/raven-work/x", + stateful=False, + authored_task=authored_task or "", + ) + ) + started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + remote_runs.mark_left(task_id) + raise + return "never" + + mgr.registry._backends["Remote"] = _OnMachine() + monkeypatch.setattr(manager_mod, "build_executor", lambda *a, **k: _DummyExecutor()) + tool = SpawnTool(manager=mgr) + tool.set_context("cli", "direct", "cli:direct") + await tool.execute(node_id="r1", task_summary="Run on the machine", task="Go.", subagent="Remote") + await asyncio.wait_for(started.wait(), timeout=5) + + assert await mgr.cancel_all(reason="the gateway stopped") == 1 + + (run,) = remote_runs.pending() + assert run.task_summary == "Run on the machine" and run.authored_task == "Go." + assert run.origin["node_id"] == "r1" and run.origin["session_key"] == "cli:direct" + assert run.origin["authored_task"] == "Go." and run.origin["handle"] == run.task_id, ( + "no instance: the handle is the dispatch" + ) From 1ec3b798bbdff1d2e94f26d8c9f71c05a34218e7 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:32:09 +0800 Subject: [PATCH 39/44] fix(*): the pick-up types move to a leaf, so the cli backend stops closing a cycle The guard in `tests/test_import_cycle_budget.py` counts a function-local import like a top-level one, so `CliAgentBackend.pick_up` importing `remote_resume` for two types drew the cli backend and the pick-up module into one cycle (`cli_agent -> remote_resume -> acp_agent -> backends -> cli_agent`) and took the module count from 44 to 46, over the ceiling. `PickedUp`, `UnreachableError` and `PICKED_UP_NOTE` now live in `raven/acp_client/remote_pickup.py`, which imports nothing of raven's. `remote_resume` imports and re-exports them, so every existing `remote_resume.PickedUp` reference still holds, and the cli backend takes them straight from the leaf. Co-Authored-By: Claude Fable 5.1 --- raven/acp_client/__init__.py | 3 ++ raven/acp_client/remote_pickup.py | 32 ++++++++++++++++++++++ raven/acp_client/remote_resume.py | 17 +----------- raven/agent/subagent/backends/cli_agent.py | 31 ++++++++++----------- 4 files changed, 51 insertions(+), 32 deletions(-) create mode 100644 raven/acp_client/remote_pickup.py diff --git a/raven/acp_client/__init__.py b/raven/acp_client/__init__.py index f0effa4f1..47457be96 100644 --- a/raven/acp_client/__init__.py +++ b/raven/acp_client/__init__.py @@ -16,6 +16,9 @@ - :mod:`raven.acp_client.capabilities` -- turning one ``initialize`` handshake into a stored snapshot of what the agent can actually do, so the roster advertises measurements rather than hand-typed declarations. +- :mod:`raven.acp_client.remote_pickup` -- the types a picked-up run is reported + with, in a leaf of their own so the cli backend need not import the pick-up + module (see the guard in ``tests/test_import_cycle_budget.py``). """ from raven.acp_client.capabilities import ( diff --git a/raven/acp_client/remote_pickup.py b/raven/acp_client/remote_pickup.py new file mode 100644 index 000000000..7c39d94c0 --- /dev/null +++ b/raven/acp_client/remote_pickup.py @@ -0,0 +1,32 @@ +"""The types a picked-up run is reported with, apart from the code that picks one up. + +``PickedUp``, ``UnreachableError`` and ``PICKED_UP_NOTE`` are what every caller of +a pick-up pass needs: the two entries that pick a run up (``remote_resume`` for an +acp run, ``raven.agent.subagent.backends.cli_agent.CliAgentBackend.pick_up`` for a +cli one) and ``resume_remote``, which routes between them. They live in this leaf so +that the cli backend does not have to import ``remote_resume``: that import drew the +cli backend and the pick-up module into one cycle (``cli_agent -> remote_resume -> +acp_agent -> backends -> cli_agent``), and the guard in +``tests/test_import_cycle_budget.py`` counts a function-local import exactly like a +top-level one. Nothing here imports anything of raven's. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +#: What the reply of a picked-up turn ends with: the reader learns it crossed a restart. +PICKED_UP_NOTE = "picked up after this computer restarted" + + +class UnreachableError(RuntimeError): + """The machine could not be asked: the record stays, for a later start.""" + + +@dataclass(frozen=True) +class PickedUp: + status: str + """``ok`` with the reply, or ``error`` with the sentence that says why there is none.""" + reply: str + stop_reason: Any = None diff --git a/raven/acp_client/remote_resume.py b/raven/acp_client/remote_resume.py index f9de0a857..847540ac0 100644 --- a/raven/acp_client/remote_resume.py +++ b/raven/acp_client/remote_resume.py @@ -14,7 +14,6 @@ from __future__ import annotations -from dataclasses import dataclass from typing import Any from loguru import logger @@ -24,24 +23,10 @@ from raven.acp_client.acp_dialects import dialect_for from raven.acp_client.pool import get_pool from raven.acp_client.protocol import AcpConnectionError, AcpTimeoutError +from raven.acp_client.remote_pickup import PICKED_UP_NOTE, PickedUp, UnreachableError from raven.acp_client.remote_runs import RemoteRun from raven.ops import remote_path -#: What the reply of a picked-up turn ends with: the reader learns it crossed a restart. -PICKED_UP_NOTE = "picked up after this computer restarted" - - -class UnreachableError(RuntimeError): - """The machine could not be asked: the record stays, for a later start.""" - - -@dataclass(frozen=True) -class PickedUp: - status: str - """``ok`` with the reply, or ``error`` with the sentence that says why there is none.""" - reply: str - stop_reason: Any = None - class _Span: """What :meth:`AcpAgentBackend._finished` tells about a turn cut short; there is no trace here.""" diff --git a/raven/agent/subagent/backends/cli_agent.py b/raven/agent/subagent/backends/cli_agent.py index fa62ecb40..11f0feb68 100644 --- a/raven/agent/subagent/backends/cli_agent.py +++ b/raven/agent/subagent/backends/cli_agent.py @@ -1270,29 +1270,30 @@ async def pick_up(self, run: Any, *, pool: Any = None) -> Any: rest over as it comes -- a cli run's reply is the whole of its output, read from the start -- and the exit code ends it, as a run here ends (``_exec_there``). Raises - :class:`raven.acp_client.remote_resume.UnreachableError` when the + :class:`raven.acp_client.remote_pickup.UnreachableError` when the machine cannot be asked, or this computer stops under the pick-up, so the record stays for a later start; every other outcome is a ``PickedUp``, and the record is the caller's to remove. """ - from raven.acp_client import remote, remote_resume + from raven.acp_client import remote + from raven.acp_client.remote_pickup import PICKED_UP_NOTE, PickedUp, UnreachableError from raven.ops import remote_path link = pool if pool is not None else remote_path.nodes() if link is None: - raise remote_resume.UnreachableError( + raise UnreachableError( f"this process does not reach registered machines, so the run on machine {run.machine} waits" ) try: target = remote.machine(run.machine) except remote.RemoteMachineError as exc: - return remote_resume.PickedUp("error", f"the run of {run.agent!r} cannot be picked up: {exc}") + return PickedUp("error", f"the run of {run.agent!r} cannot be picked up: {exc}") try: rows = await link.tasks(run.machine) except remote_path.NodeCallError as exc: - raise remote_resume.UnreachableError(str(exc)) from None + raise UnreachableError(str(exc)) from None if not any(row.get("task") == run.task for row in rows): - return remote_resume.PickedUp( + return PickedUp( "error", f"machine {target.label} no longer has the run of {run.agent!r} (task {run.task}); " "its record there was removed, or the machine's node directory was", @@ -1301,8 +1302,8 @@ async def pick_up(self, run: Any, *, pool: Any = None) -> Any: endpoint, _attached = await link.attach(run.machine, run.task, name=run.agent, after=0) except remote_path.NodeCallError as exc: if exc.reason in ("unknown_task", None) and "no longer runs" in str(exc): - return remote_resume.PickedUp("error", str(exc)) - raise remote_resume.UnreachableError(str(exc)) from None + return PickedUp("error", str(exc)) + raise UnreachableError(str(exc)) from None out_lines: list[str] = [] err_lines: list[str] = [] @@ -1322,29 +1323,27 @@ async def read_err() -> None: raise if (why := endpoint.ended()) is not None: if remote.leaving(): - raise remote_resume.UnreachableError( + raise UnreachableError( f"this computer stopped while the run of {run.agent!r} on machine {target.label} was being " "picked up; it waits for the next start" ) - return remote_resume.PickedUp( + return PickedUp( "error", f"the run of {run.agent!r} on machine {target.label} ended while this computer was away: {why}" ) stdout = "".join(f"{line}\n" for line in out_lines) stderr = "".join(f"{line}\n" for line in err_lines) if rc != 0: said = _human_failure(stdout, stderr) - return remote_resume.PickedUp( - "error", f"CLI agent {run.agent!r} exited {rc}" + (f": {said}" if said else "") - ) + return PickedUp("error", f"CLI agent {run.agent!r} exited {rc}" + (f": {said}" if said else "")) try: _jsonl_id, parsed = self._transcript(stdout) except CliAgentReportedError as exc: - return remote_resume.PickedUp("error", str(exc)) + return PickedUp("error", str(exc)) output = await clamp_output(self._reply_from(stdout, stderr, parsed), self.max_output_chars, agent=self.name) - return remote_resume.PickedUp( + return PickedUp( "ok", output + f"\n\n[raven] Ran on machine {target.label}, in {remote.shown(target, run.session_cwd)}, " - f"{remote_resume.PICKED_UP_NOTE}; the files it wrote are there, not on this computer.", + f"{PICKED_UP_NOTE}; the files it wrote are there, not on this computer.", ) async def _attempt( From 62c5c64c5908a6923d53c225f820274c324421db Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:10:33 +0800 Subject: [PATCH 40/44] fix(*): a run's record outlives its reply until the announce was taken, and the turn replays whole The backend removed the run's record on its machine the moment the prompt returned, before the manager had announced the reply: a host killed in between had no way back to the run, and its reply was lost. And the record's frame count had moved past the answer by then, so a record left behind would have resumed after it and waited forever. Now the manager dispatches with `reported_by_caller=True`, the backend leaves the record behind with the reply, and the manager removes it once the announce was taken (`remote_runs.forget_dispatch`) -- or, when the draining scheduler refused the announce, annotates it with the dispatch's origin and summary for the next start to report. A caller that takes the reply as its own result (a graph node, a probe, a direct chat) dispatches without the keyword, and the record goes as before. The record's frame count stops at the prompt: the turn's own frames are never checkpointed, so a pick-up replays the turn whole, the chunks before the answer as much as the answer, and the reply it reports is complete. The cli lane keeps its record the same way; its pick-up read the whole output from the start already. Co-authored-by: Claude (claude-fable-5-1) --- CONTEXT.md | 11 ++-- raven/acp_client/acp_agent.py | 17 +++++- raven/acp_client/remote_runs.py | 42 ++++++++++++--- raven/agent/subagent/backends/cli_agent.py | 27 ++++++++-- raven/agent/subagent/manager.py | 20 ++++++- raven/node/client.py | 3 +- tests/test_remote_runs.py | 20 +++++-- tests/test_subagent_acp_remote.py | 43 +++++++++++++++ tests/test_subagent_cli_remote.py | 53 ++++++++++++++++++ tests/test_subagent_manager.py | 63 ++++++++++++++++++++++ 10 files changed, 277 insertions(+), 22 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 809e28829..b12514989 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -442,10 +442,13 @@ link is tried again on a backoff for as long as any waits; a link that comes bac each agent again by its task id. A turn running on a machine when this computer stops is left with the daemon (`remote.leave_agents_running()`), remembered while in flight in `/machines/runs/` (`raven/acp_client/remote_runs.py`), and picked up by the -next start (`raven/agent/subagent/resume_remote.py`): attached from the frame last read -(a cli entry's command from its first line, its whole output being the reply), its reply -read off the machine and announced into the conversation that asked -- the task quoted as -the model wrote it, the record removed only once the announce was taken. Only +next start (`raven/agent/subagent/resume_remote.py`): attached from the frame its prompt +went out after, so the turn is read again whole (a cli entry's command from its first line, +its whole output being the reply), its reply read off the machine and announced into the +conversation that asked -- the task quoted as the model wrote it, the record removed only +once the announce was taken. A turn that returned here keeps its record the same way until +the manager's announce of the reply was taken (`remote_runs.forget_dispatch`), so a host +that stops between the two reads the reply again. Only `--stdio` (tests, a fallback) ties the agents to the connection. _Avoid_: "node" alone (a DAG node is a `run_subagent_dag` step, and the Machine entry already rules the bare word out for a computer); "node id" for the install diff --git a/raven/acp_client/acp_agent.py b/raven/acp_client/acp_agent.py index f393938f2..ff609fb14 100644 --- a/raven/acp_client/acp_agent.py +++ b/raven/acp_client/acp_agent.py @@ -1536,6 +1536,11 @@ async def run( authored_task: str | None = None, on_delta: Callable[[str], Awaitable[None]] | None = None, media: Sequence[Media] = (), + # Whether the caller reports the reply itself (the manager's announce) + # and removes the run's record on its machine then + # (`remote_runs.forget_dispatch`); otherwise the record goes as this + # returns, as for a caller that takes the reply as its own result. + reported_by_caller: bool = False, ) -> str: # The parent binding is forwarded when the pooled ACP worker launches. # Without this, a long-lived worker keeps the provider/model captured @@ -1717,6 +1722,7 @@ async def run( connection.responders.attach(session_id, responder) if self.can_steer or steer_offered(connection.initialize): activity.offer_steer(collector._run, self._steerer(client, session_id)) + answered = False try: result = await client.request( "session/prompt", @@ -1738,6 +1744,7 @@ async def run( cancel_session=session_id, sent=tracker.prompt_sent if tracker is not None else None, ) + answered = True except asyncio.CancelledError: if tracker is not None and remote.leaving(): # This process is stopping: the turn goes on under @@ -1776,7 +1783,13 @@ async def run( raise finally: if tracker is not None: - self._untrack(client, tracker, keep=remote.leaving()) + # The record goes with a turn that ended here with no + # reply. It stays while this process leaves the turn + # running, and with the reply for a caller that reports + # it (`remote_runs.forget_dispatch`): a host that stops + # between the answer and the announce has no other way + # back to the run. + self._untrack(client, tracker, keep=remote.leaving() or (answered and reported_by_caller)) activity.offer_steer(collector._run, None) connection.router.detach(session_id, collector) connection.elicitors.detach(session_id, elicitor) @@ -1915,7 +1928,7 @@ def _track_on_machine( task_id: str, session_cwd: str, ) -> remote_runs.Tracker | None: - """The record of this turn on its machine, kept current as the agent's frames arrive.""" + """The record of this turn on its machine, kept current until the prompt is out (``remote_runs.Tracker``).""" endpoint = client.endpoint task = getattr(endpoint, "task", None) watchers = getattr(endpoint, "watchers", None) diff --git a/raven/acp_client/remote_runs.py b/raven/acp_client/remote_runs.py index fa70de1f2..ed7d6e60e 100644 --- a/raven/acp_client/remote_runs.py +++ b/raven/acp_client/remote_runs.py @@ -11,8 +11,13 @@ ``initialize`` on the connection -- none of the ACP part for a cli entry's command, whose whole output is its reply, read from the start. One file per run under ``/machines/runs/.json``, owner-only, written when the turn -starts, kept current as frames arrive, and removed when the turn ends here or -once the process that picked it up has reported it. A +starts and removed once its outcome has reached the conversation: by the +manager, after its announce of the reply was taken (:func:`forget_dispatch`), +by the backend when the turn ended here with no reply, or by the process that +picked the run up, once it has reported it. A host that stops between the +agent's answer and the announce has the record still, and the next start +reads the reply again: the record moves past no frame of the turn itself, so +the turn is replayed whole. A file left behind is a run to pick up (:mod:`raven.acp_client.remote_resume`), by one process at a time: the pick-up is made under a claim beside the record (:func:`claim`), since two processes of one Raven home can start together. @@ -66,7 +71,8 @@ class RemoteRun: session_cwd: str stateful: bool received: int = 0 - """How many of the agent's frames this computer has read on the connection: where an attach goes on from.""" + """How many of the agent's frames this computer had read when the prompt went out: where an attach goes on + from, so a pick-up reads the turn from its first frame.""" started: float = field(default_factory=time.time) kind: str = "acp" """``acp`` for a turn in an ACP session, ``cli`` for a command whose whole output is the reply @@ -151,6 +157,21 @@ def forget(task: str) -> None: release(task) +def forget_dispatch(task_id: str) -> bool: + """Remove the record of the dispatch ``task_id`` (the manager's id), and its claim; whether there was one. + + The manager's end of a run that returned here. The backend leaves the + record behind with the reply when the caller reports the reply itself + (``reported_by_caller``), and it goes once the announce was taken: a + host that stops between the two has the way back to the run still. + """ + for run in pending(): + if run.task_id == task_id: + forget(run.task) + return True + return False + + def claim_path(task: str) -> Path: """Beside the record: the file naming the process that is picking ``task`` up.""" return runs_dir() / f"{task}.json.claim" @@ -286,11 +307,14 @@ def pending() -> list[RemoteRun]: class Tracker: - """One run's record, kept current as the agent's frames arrive, and removed when the turn ends here. + """One run's record, kept current until the prompt is out, and removed when the turn ends here. ``received`` is what a channel endpoint's watcher calls after every frame - the connection has handed on (:class:`raven.node.client.ChannelEndpoint`), - so a later attach replays only what this computer never read. + the connection has handed on (:class:`raven.node.client.ChannelEndpoint`). + It moves the record only until ``prompt_sent``: the turn's own frames are + never checkpointed, since the reply is put together from all of them -- + the chunks before the answer as much as the answer -- and a process that + picks the run up has to read the turn from its first frame. """ def __init__(self, run: RemoteRun) -> None: @@ -308,7 +332,7 @@ def prompt_sent(self, request_id: int) -> None: logger.warning("remote run record {} could not be updated: {}", self.run.task, exc) def received(self, n: int) -> None: - if n <= self.run.received: + if self.run.prompt_id > 0 or n <= self.run.received: return self.run.received = n try: @@ -317,7 +341,8 @@ def received(self, n: int) -> None: logger.warning("remote run record {} could not be updated: {}", self.run.task, exc) def close(self, *, keep: bool) -> None: - """The turn ended here (``keep`` false) or goes on without this process (``keep`` true).""" + """The turn ended here (``keep`` false), or the record stays (``keep`` true): the turn goes on without + this process, or the caller reports the reply and forgets the record then (:func:`forget_dispatch`).""" self.kept = keep if not keep: forget(self.run.task) @@ -349,6 +374,7 @@ def was_left(task_id: str) -> bool: "claim", "claim_path", "forget", + "forget_dispatch", "mark_left", "path_of", "pending", diff --git a/raven/agent/subagent/backends/cli_agent.py b/raven/agent/subagent/backends/cli_agent.py index 11f0feb68..cd98fdc76 100644 --- a/raven/agent/subagent/backends/cli_agent.py +++ b/raven/agent/subagent/backends/cli_agent.py @@ -206,10 +206,14 @@ class _OnMachine: """The dispatch's handle, for the record.""" authored_task: str = "" """The task as the dispatching model wrote it, for the record: what an announce after a restart quotes.""" + reported_by_caller: bool = False + """Whether the caller reports the reply itself and forgets the record then (``remote_runs.forget_dispatch``): + the record outlives the command's output; otherwise it goes with it.""" def _untrack(tracker: Any, *, keep: bool) -> None: - """The run's record on this computer goes with a turn that ended here, or stays for the next process.""" + """The run's record on this computer goes with a command that ended here, or stays: for the next process, or + for the caller that reports the output (``_OnMachine.reported_by_caller``).""" if tracker is not None: tracker.close(keep=keep) @@ -736,7 +740,9 @@ async def drive() -> int | None: await endpoint.end() _untrack(tracker, keep=False) raise - _untrack(tracker, keep=False) + # The output is the reply: a caller that reports it keeps the record + # until it has, so a host that stops in between reads the output again. + _untrack(tracker, keep=placed.reported_by_caller) if (why := endpoint.ended()) is not None: raise RuntimeError(f"CLI agent {self.name!r}: {why}") stdout = "".join(f"{line}\n" for line in out_lines) @@ -809,6 +815,11 @@ async def run( authored_task: str | None = None, on_delta: Callable[[str], Awaitable[None]] | None = None, media: Sequence[Media] = (), + # Whether the caller reports the reply itself (the manager's announce) + # and removes the run's record on its machine then + # (`remote_runs.forget_dispatch`); otherwise the record goes as this + # returns. Read on a machine only: a command here leaves no record. + reported_by_caller: bool = False, ) -> str: # A path here means nothing on another machine: the files that came # with the task are named to an agent there by address instead, once @@ -869,6 +880,7 @@ async def run( mcp_file=mcp_path, media=media, authored_task=authored_task, + reported_by_caller=reported_by_caller, ) for note in filter(None, (self._withheld_note(grant), grant.note_text())): notice = f"\n\n[raven] {note}." @@ -937,6 +949,7 @@ async def _dispatch( mcp_file: str | None = None, media: Sequence[Media] = (), authored_task: str | None = None, + reported_by_caller: bool = False, ) -> str: skey = session_key or "default" if self.machine: @@ -951,6 +964,7 @@ async def _dispatch( notice_sink=notice_sink, media=media, authored_task=authored_task, + reported_by_caller=reported_by_caller, ) cwd = self.cwd or str(workspace) # Here rather than beside either launch: this lane sees nothing of what @@ -989,6 +1003,7 @@ async def _dispatch_there( notice_sink: Callable[[str], Awaitable[None]] | None, media: Sequence[Media], authored_task: str | None = None, + reported_by_caller: bool = False, ) -> str: """The run on the entry's machine: placed there first, with the task's files named by address. @@ -999,7 +1014,13 @@ async def _dispatch_there( """ from raven.acp_client import remote - placed = replace(await self._on_machine(handle), skey=skey, handle=handle, authored_task=authored_task or "") + placed = replace( + await self._on_machine(handle), + skey=skey, + handle=handle, + authored_task=authored_task or "", + reported_by_caller=reported_by_caller, + ) # Nothing is carried to the machine: the files that come with the # task are named to the agent by address, and it looks for them there. if note := remote.files_note(media, placed.target): diff --git a/raven/agent/subagent/manager.py b/raven/agent/subagent/manager.py index 882283d5a..0dd9c9006 100644 --- a/raven/agent/subagent/manager.py +++ b/raven/agent/subagent/manager.py @@ -1801,6 +1801,10 @@ async def _run_subagent_inner( # the same question. cancelled = False left = False + # Whether the reply reached the conversation: only a refused announce + # of a run that returned says no, and the run's record on its machine + # then stays for the next start to report. + announced = True # The record's own id, not its directory's name: the artifacts are a # filename prefix in the shared node root now, so the directory names # the namespace rather than the call. @@ -1865,6 +1869,10 @@ async def _run_subagent_inner( ), **optional_keyword(backend, "authored_task", origin.get("authored_task")), **({"mcp_grant": mcp_grant} if mcp_grant is not None else {}), + # The reply is announced here, below: a backend that + # ran it on a machine leaves the run's record until + # then, rather than removing it as it returns. + **optional_keyword(backend, "reported_by_caller", True), **state_kwargs, ) if (failure := llm_error_reply(final_result)) is not None: @@ -1878,7 +1886,7 @@ async def _run_subagent_inner( origin, task_id, task_summary, "completed", call_id=call_id, ended_at=int(time.time() * 1000) ) record.finish(status="completed", output=final_result, activity=did) - await self._announce_result( + taken = await self._announce_result( task_id, task_summary, task, @@ -1888,6 +1896,7 @@ async def _run_subagent_inner( record_path=str(record.file("out.md")), activity=did, ) + announced = taken is not False except asyncio.CancelledError: cancelled = True reason = self._cancel_reasons.pop(task_id, UNEXPLAINED_CANCEL) @@ -1957,6 +1966,15 @@ async def _run_subagent_inner( # machine: its claim stays open for the process that picks it up. if not left: await self._finalize_node_claim(session_key, record) + # The run's record on its machine, when the backend left + # one with the reply: gone once the announce was taken. An + # announce the draining scheduler refused leaves it, with + # what the announce after a restart needs, for the next + # start to report (raven.agent.subagent.resume_remote). + if announced: + remote_runs.forget_dispatch(task_id) + else: + remote_runs.annotate(task_id, origin=origin, task_summary=task_summary) # Not for a cancelled call: this poller would be created after # the cancellation sweep took its snapshot, so nothing could # reap it (see cancel_all / cancel_by_session). diff --git a/raven/node/client.py b/raven/node/client.py index 7f0cf97c9..512d0882e 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -176,7 +176,8 @@ def __init__( """How many of the agent's frames reached here: where an attach goes on from.""" self.watchers: list[Callable[[int], None]] = [] """Told, after each frame the connection has dealt with, how many it has read: what a record of the - run keeps, so a process that comes after this one attaches from there (:mod:`raven.acp_client.remote_runs`).""" + run keeps until its prompt is out, so a process that comes after this one attaches from there and reads + the turn whole (:mod:`raven.acp_client.remote_runs`).""" self._handing = 0 self._frames: asyncio.Queue[tuple[int, bytes] | None] = asyncio.Queue() self._stderr: asyncio.Queue[bytes | None] = asyncio.Queue() diff --git a/tests/test_remote_runs.py b/tests/test_remote_runs.py index 9e418dcc8..f1e953720 100644 --- a/tests/test_remote_runs.py +++ b/tests/test_remote_runs.py @@ -89,15 +89,16 @@ def test_forget_removes_the_record_and_is_quiet_about_one_already_gone() -> None assert remote_runs.pending() == [] -def test_the_tracker_keeps_the_prompt_s_id_and_the_frames_read_current_and_removes_the_record_when_the_turn_ends() -> ( +def test_the_tracker_keeps_the_prompt_s_id_and_the_frames_read_before_it_and_removes_the_record_when_the_turn_ends() -> ( None ): tracker = remote_runs.Tracker(_run(prompt_id=0, next_id=1)) assert remote_runs.pending()[0].prompt_id == 0 - tracker.prompt_sent(5) tracker.received(3) tracker.received(2) # a frame counted twice is not a step back + tracker.prompt_sent(5) + tracker.received(9) # the turn's own frames are the pick-up's to read again: the record stays at the prompt (kept,) = remote_runs.pending() assert (kept.prompt_id, kept.next_id, kept.received) == (5, 6, 3) @@ -105,6 +106,19 @@ def test_the_tracker_keeps_the_prompt_s_id_and_the_frames_read_current_and_remov assert remote_runs.pending() == [] and tracker.kept is False +def test_a_dispatch_s_record_is_forgotten_by_the_manager_s_id_with_its_claim() -> None: + """The manager's end of a run that returned: the record the backend left with the reply goes once the announce was taken.""" + remote_runs.record(_run(task_id="d1")) + remote_runs.record(_run("other", task_id="d2")) + assert remote_runs.claim("echo@box-1a2b3c4d") + + assert remote_runs.forget_dispatch("d1") is True + assert remote_runs.forget_dispatch("d1") is False, "nothing is left of it" + + assert [run.task for run in remote_runs.pending()] == ["other"] + assert not remote_runs.claim_path("echo@box-1a2b3c4d").exists() + + def test_a_tracker_closed_to_keep_leaves_the_record_for_the_next_start() -> None: tracker = remote_runs.Tracker(_run()) @@ -114,7 +128,7 @@ def test_a_tracker_closed_to_keep_leaves_the_record_for_the_next_start() -> None def test_a_record_that_cannot_be_updated_is_named_not_raised(logs, monkeypatch) -> None: - tracker = remote_runs.Tracker(_run()) + tracker = remote_runs.Tracker(_run(prompt_id=0)) def refuse(run): raise OSError("disk full") diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 3ebf8593c..c2ec6ae51 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -661,6 +661,49 @@ async def test_a_stop_that_lands_under_a_pick_up_leaves_the_run_for_the_start_af assert remote_runs.pending() == [] and _agent_pid() == agent +async def test_a_reply_the_caller_reports_keeps_its_record_and_a_host_that_stops_before_the_announce_reads_it_again( + box, tmp_path, monkeypatch +): + """The manager announces the reply after the backend returns it: the record stays until then, at the prompt, so the next start replays the turn whole.""" + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + backend = build_third_party_backend(remote_config()) + await backend.run("hi", task_id="t0", workspace=tmp_path, executor=None, session_key="tui:s1") + assert remote_runs.pending() == [], "a caller that takes the reply as its own result leaves no record" + + reply = await backend.run( + "hi", task_id="t1", workspace=tmp_path, executor=None, session_key="tui:s1", reported_by_caller=True + ) + + assert reply.split("\n\n[raven]")[0] == "pong" + agent = _agent_pid() + (run,) = remote_runs.pending() + assert run.task_id == "t1" and run.prompt_id > 0, "the record outlives the reply, for the manager to forget" + (connection,) = get_pool().connections(backend.name) + assert run.received < connection.client.endpoint.received, ( + "and it stayed at the prompt: the turn is read again whole" + ) + + # This computer stops between the reply and its announce. + remote.leave_agents_running() + try: + await close_pool() + await box.nodes.close_all() + finally: + remote.staying() + assert _alive(agent) + session_dir = tmp_path / "sessions" / "s1" + meta = {"call_id": "n1", "session_key": "tui:s1", "agent": backend.name, "handle": "t1", "task_summary": "say hi"} + SpawnRecord.open(session_dir, task_id="t1", task="hi", meta=meta, node_id="n1") + manager = _Manager(backend, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + + assert await asyncio.wait_for(resume_remote.resume_remote_runs(manager), timeout=30) == 1 + + (said,) = manager.announced + assert said["status"] == "ok" and said["result"].split("\n\n[raven]")[0] == "pong", "the whole reply, read again" + assert remote_runs.pending() == [] and _agent_pid() == agent + + async def test_the_pool_closed_before_the_links_leaves_no_agent_and_the_daemon_goes_idle(box, tmp_path): """What the fixture's teardown relies on: ended through its link, the agent goes, and the daemon leaves on its own.""" backend = build_third_party_backend(remote_config()) diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py index 064070d8c..40d30ced6 100644 --- a/tests/test_subagent_cli_remote.py +++ b/tests/test_subagent_cli_remote.py @@ -821,6 +821,59 @@ async def test_a_command_that_ends_here_leaves_no_record(box, tmp_path, monkeypa assert head == "hi" and remote_runs.pending() == [] +async def test_a_reply_the_caller_reports_keeps_the_command_s_record_and_a_host_that_stops_before_the_announce_reads_it_again( + box, tmp_path, monkeypatch +): + """The manager announces the output after the backend returns it: the record stays until then, and the next start reads the whole output again.""" + from raven.acp_client import remote_runs + from raven.agent.subagent import resume_remote + from raven.agent.subagent.history import SpawnRecord + from tests.test_subagent_acp_remote import _Manager, _Registry + + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + backend = build_third_party_backend(cli_config("sh -c 'echo done; cat'")) + + head, _ = _split( + await backend.run( + "hi", + task_id="t1", + workspace=tmp_path, + executor=None, + session_key="tui:s1", + instance="kept", + reported_by_caller=True, + ) + ) + + assert head == "done\nhi" + (run,) = remote_runs.pending() + assert (run.kind, run.task_id, run.received) == ("cli", "t1", 0), ( + "the record outlives the output, for the manager to forget" + ) + + # This computer stops between the output and its announce. + await box.nodes.close_all() + session_dir = tmp_path / "sessions" / "s1" + meta = { + "call_id": "n1", + "session_key": "tui:s1", + "agent": backend.name, + "handle": "kept", + "task_summary": "say done", + } + SpawnRecord.open(session_dir, task_id="t1", task="hi", meta=meta, node_id="n1") + manager = _Manager(backend, session_dir) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + + assert await asyncio.wait_for(resume_remote.resume_remote_runs(manager), timeout=30) == 1 + + (said,) = manager.announced + assert said["status"] == "ok" and said["result"].split("\n\n[raven]")[0] == "done\nhi", ( + "the whole output, read again" + ) + assert remote_runs.pending() == [] + + class _PickUpLink: """A node pool with one task, whose attach hands over ``endpoint``: what a pick-up reads a left run off.""" diff --git a/tests/test_subagent_manager.py b/tests/test_subagent_manager.py index 1f0e34078..4160639c3 100644 --- a/tests/test_subagent_manager.py +++ b/tests/test_subagent_manager.py @@ -1256,6 +1256,69 @@ def _spawn_origin(agent: str | None, instance: str | None) -> dict[str, Any]: } +async def test_a_run_s_record_on_its_machine_goes_once_its_announce_was_taken_and_stays_when_it_was_refused( + monkeypatch, tmp_path +) -> None: + """The backend leaves the run's record with the reply (`reported_by_caller`); the manager forgets it after the + announce, or, when the draining scheduler refused the announce, keeps it with what the next start's announce needs.""" + from raven.acp_client import remote_runs + + monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) + told: list[bool] = [] + + class _OnMachine: + streams = False + + async def run(self, task: str, *, task_id: str, reported_by_caller: bool = False, **_: Any) -> str: + told.append(reported_by_caller) + remote_runs.record( + remote_runs.RemoteRun( + machine="box", + task=f"stub@box-{task_id}", + agent="stub@box", + session_key="tui:s1", + handle=task_id, + task_id=task_id, + session_id="s", + prompt_id=3, + next_id=4, + initialize={}, + launch_key="k", + config_key="c", + session_cwd="/w", + stateful=False, + ) + ) + return "what it found" + + manager = SubagentManager(provider=_StubProvider(), workspace=tmp_path) + monkeypatch.setattr(manager, "_resolve_backend", lambda agent: _OnMachine()) + taken = True + + async def _announce(*_a: Any, **_k: Any) -> bool: + return taken + + monkeypatch.setattr(manager, "_announce_result", _announce) + origin = {"channel": "tui", "chat_id": "s1", "session_key": "tui:s1", "agent": "stub@box"} + + await manager._run_subagent_inner( + "t1", "find it", "find", dict(origin), _DummyExecutor(), manager.provider, manager.model + ) + + assert told == [True], "the backend is told the reply is reported here" + assert remote_runs.pending() == [], "announced: the record is gone" + + taken = False + await manager._run_subagent_inner( + "t2", "find it", "find", dict(origin), _DummyExecutor(), manager.provider, manager.model + ) + + (kept,) = remote_runs.pending() + assert (kept.task_id, kept.task_summary, kept.origin["session_key"]) == ("t2", "find", "tui:s1"), ( + "refused: kept for the next start, with what its announce needs" + ) + + async def test_announcement_carries_the_instance_handle() -> None: mgr = _make_manager(max_concurrent=1) submitted: list[Any] = [] From a0babda596a2ba2e05ef32b2a4c67464888c62a6 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:16:14 +0800 Subject: [PATCH 41/44] feat(*): enable registered machines without a debug environment switch Remove the default-off machine gate from dispatch, node access, file routing, preset addition and configuration messages. Registered machines work without an environment variable, including when an old launch still sets the removed variable to zero. Existing SSH, registry and file permission checks continue to apply. Exercise the previous missing-agent recovery failure with the real registry, and cover default dispatch, file routing and permission checks. Document the former trial switch and the normal startup behavior. Co-authored-by: Codex --- CONTEXT.md | 8 +++- docs-site/docs/agent-collaboration.md | 22 +++++++++++ docs-site/docs/agent-collaboration.zh.md | 18 +++++++++ raven/acp_client/remote.py | 8 ---- raven/agent/subagent/registry.py | 8 ---- raven/agent/tools/raven_config.py | 5 +-- raven/agent/tools/shell.py | 8 +--- raven/config/schema.py | 38 +++---------------- raven/home.py | 17 --------- raven/ops/remote_path.py | 13 +++---- raven/rpc/methods/subagents.py | 2 +- raven/rpc/models.py | 2 +- rpc-schema/openrpc.json | 2 +- tests/test_acp_remote.py | 37 +++++-------------- tests/test_deliver_files_tool.py | 1 - tests/test_node_client.py | 17 +++------ tests/test_node_install.py | 16 ++++---- tests/test_ops_remote_path.py | 14 +++---- tests/test_permissions_remote_read.py | 13 ++++--- tests/test_raven_config_tool.py | 5 +-- tests/test_remote_file_tools.py | 12 +++--- tests/test_remote_resume.py | 1 - tests/test_resume_remote.py | 34 ++++++++++++++++- tests/test_rpc_subagents.py | 28 ++++++-------- tests/test_shell_machine_channel.py | 17 ++------- tests/test_subagent_acp_remote.py | 47 +++++++++--------------- tests/test_subagent_cli_remote.py | 20 ++++------ 27 files changed, 176 insertions(+), 237 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 809e28829..5a43f3238 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -401,10 +401,14 @@ of it. The entry names only the machine's id, and the entry's `env` reaches the agent on the link, never on a command line. Such an entry is added the way any preset is, with `machine` (and `remote_cwd`, for an `acp` or `cli` one) on `subagents.add` — from a chat through -`raven_config`, which lists the registered machines while the lane is on; the page's -add passes no machine yet — refused before anything is spent when the lane is off, +`raven_config`, which lists the usable registered machines; the page's +add passes no machine yet — refused before anything is spent when the id is not usable or the preset is a built-in agent, and proved by the same readiness ping as a local add, run on that machine. +Registered-machine dispatch, file access and recovery are available without an +environment switch. An entry without `machine` still runs here. Registration, +SSH authentication, file permissions and concurrency limits govern remote work; +listing entries and boot capability backfills do not open machine connections. _Avoid_: "host" / "server" / "node" (too broad, no link to the `ops connection` registry that supplies the rows); "GPU box" (only some are GPU hosts, and the term covers any registered compute destination); diff --git a/docs-site/docs/agent-collaboration.md b/docs-site/docs/agent-collaboration.md index 3e4d502bf..fcbde5614 100644 --- a/docs-site/docs/agent-collaboration.md +++ b/docs-site/docs/agent-collaboration.md @@ -7,6 +7,28 @@ tasks and stateful agent instances; they are different things. Set up agents first with [Agent Integrations](agent-integrations.md). This page covers using them, not installing their transports. +## Work on a registered machine + +Register the destination with `raven ops connection add`, then install its node +with `raven ops connection install-node `. Set an agent entry's +`machine` to that registry id, or ask Raven to add a preset on that machine. +Start Raven normally, for example with `raven serve`; no environment switch is +required. Entries without `machine` continue to run locally. + +ACP and CLI entries run under the machine's node. An OpenAI entry reaches its +HTTP endpoint through an SSH tunnel from that machine; built-in entries run +locally. The registered SSH credentials, file permission prompts and concurrency +limits still apply. Attachments are named by address, not copied automatically. + +If Raven stops during a remote ACP or CLI task, the machine keeps working. The +next start uses the records in the same Raven home to recover the result into +the original conversation. Keep the machine registration and agent entry +available for that recovery. + +`RAVEN_DEBUG_MACHINES` was a default-off trial switch for machine dispatch and +remote file access. It is no longer read; an old value of `0` does not disable +these features or prevent task recovery. + ## Choose a collaboration shape | Need | Use | Completion evidence | diff --git a/docs-site/docs/agent-collaboration.zh.md b/docs-site/docs/agent-collaboration.zh.md index c05e26aed..9024ec07c 100644 --- a/docs-site/docs/agent-collaboration.zh.md +++ b/docs-site/docs/agent-collaboration.zh.md @@ -5,6 +5,24 @@ Raven 同时提供委派任务与有状态 Agent 实例，两者不是一回事 先按[Agent 集成](agent-integrations.md)完成配置。本页介绍如何协作，不介绍传输安装。 +## 在登记机器上工作 { #work-on-a-registered-machine } + +先用 `raven ops connection add` 登记设备，再用 +`raven ops connection install-node <机器 id>` 安装节点。把 Agent 配置的 +`machine` 设为该登记 id，或在对话里让 Raven 把预设 Agent 添加到那台机器。 +正常运行 `raven serve` 即可，不需要额外环境开关；未配置 `machine` 的 Agent 仍在本机运行。 + +ACP 和 CLI 类型由设备上的节点启动；OpenAI 类型通过 SSH 隧道访问设备那边的 +HTTP 端点；内置 Agent 仍在本机运行。SSH 登录凭据、文件审批和并发限制照常生效。 +附件只提供地址，不自动复制。 + +远端 ACP 或 CLI 任务运行时退出 Raven，设备上的任务会继续。下次使用同一个 +Raven 主目录启动时，会根据保存的记录接回结果并通告到原会话。接回时需保留 +对应的机器登记和 Agent 配置。 + +`RAVEN_DEBUG_MACHINES` 原来是默认关闭的试用开关，控制远端派发和按地址访问文件。 +现在不再读取这个变量；旧环境里即使还设为 `0`，也不会禁用这些功能或阻止任务恢复。 + ## 选择协作形式 { #choose-a-collaboration-shape } | 需求 | 方式 | 完成证据 | diff --git a/raven/acp_client/remote.py b/raven/acp_client/remote.py index db0153f82..90477e5fb 100644 --- a/raven/acp_client/remote.py +++ b/raven/acp_client/remote.py @@ -44,7 +44,6 @@ from typing import Any from urllib.parse import urlsplit, urlunsplit -from raven.home import MACHINE_LANE_ENV, machine_lane_enabled from raven.ops import connections, remote_path from raven.ops.transport import TIMED_OUT_RC, TransportError, runner_from, ssh_argv, ssh_target @@ -141,13 +140,6 @@ def machine(conn_id: str) -> Machine: launched on half a row fails as a network fault minutes into a dispatch. """ wanted = str(conn_id or "").strip() - if not machine_lane_enabled(): - # The one seam every way onto a machine passes through, so "off" is - # off for a dispatch, a verify, a node and a file alike. - raise RemoteMachineError( - f"machine {wanted!r}: the machine lane is debug-only and off; start raven with " - f"{MACHINE_LANE_ENV}=1 to use agents and files on registered machines" - ) found = connections.read() if found.state == connections.UNREADABLE: raise RemoteMachineError( diff --git a/raven/agent/subagent/registry.py b/raven/agent/subagent/registry.py index f78937398..443347b27 100644 --- a/raven/agent/subagent/registry.py +++ b/raven/agent/subagent/registry.py @@ -40,7 +40,6 @@ from raven.agent.subagent.builtin_agents import LEGACY_AGENT_ALIASES, canonical_agent_name, merge_builtin_seeds from raven.agent.subagent.instances import get_registry from raven.agent.subagent.vendored_agents import discover_product_rows, merge_product_seeds -from raven.home import machine_lane_enabled if TYPE_CHECKING: from raven.agent.subagent.backends.routing import Router, TargetReady @@ -309,13 +308,6 @@ def apply(self, configs: Sequence[Any] | None) -> None: merged = merge_product_seeds(list(configs or []), discover_product_rows()) for cfg in merge_builtin_seeds(merged): name = getattr(cfg, "name", None) - if (getattr(cfg, "machine", None) or "").strip() and not machine_lane_enabled(): - # The machine lane is debug-only: an entry on a registered - # machine stays off the table, so the model is never offered a - # dispatch the host would then refuse. The listing still shows - # the row, and its probe says the same. - logger.info("Sub-agent {!r} stays off the table: the machine lane is debug-only", name) - continue try: row = _row_for(cfg) if not row.name: diff --git a/raven/agent/tools/raven_config.py b/raven/agent/tools/raven_config.py index 354fa263c..fe77dfc04 100644 --- a/raven/agent/tools/raven_config.py +++ b/raven/agent/tools/raven_config.py @@ -310,12 +310,9 @@ def _menu_line(row: dict[str, Any], model: str) -> list[str]: def _machines_line() -> str | None: - """The registered machines a preset can be added on, while the machine lane is on; else nothing.""" - from raven.home import machine_lane_enabled + """The usable registered machines a preset can be added on, or nothing when none are registered.""" from raven.ops import connections - if not machine_lane_enabled(): - return None try: rows = connections.usable(connections.load()) except (OSError, ValueError): diff --git a/raven/agent/tools/shell.py b/raven/agent/tools/shell.py index 59f52f1fd..6a1c1b509 100644 --- a/raven/agent/tools/shell.py +++ b/raven/agent/tools/shell.py @@ -305,17 +305,13 @@ async def execute( # other two paths. Refused by name so the model learns the path, # not just that this one closed. A copy there with scp, sftp or # rsync starts nothing and passes this tool's own question like - # any other command, so it is left alone; and the file tools are - # named only while they read a remote path, with the machine - # lane on -- with it off they would write that spelling here. - from raven.home import machine_lane_enabled + # any other command, so it is left alone. File tools read remote + # paths through the registered machine's node. conn_id = str(row.get("id") or "").strip() files = ( " Its files are read and written with read_file, write_file and edit_file on the path the way " "scp writes it." - if machine_lane_enabled() - else "" ) return ( f"Error: this command reaches {row.get('display_name') or conn_id} over raw ssh from " diff --git a/raven/config/schema.py b/raven/config/schema.py index e87db6356..5949fd165 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -11,7 +11,6 @@ from raven.config.agent_names import THIRD_PARTY_PRESET_NAMES from raven.contracts.path_policy import WORKSPACE_DEFAULT_SENTINEL -from raven.home import MACHINE_LANE_ENV, machine_lane_enabled from raven.sandbox.config import SandboxConfig #: The loop's outer LLM-error retry ladder when the config names none: one wait @@ -1486,22 +1485,8 @@ def _malformed_reads_as_empty(cls, data: Any) -> Any: def _settle_machine(entry: Any) -> str | None: - """``entry.machine`` as an id or ``None``, with the lane's state said once on load. - - Shared by every kind that can name a machine, so a row of any kind that - names one while the lane is off is said to stay off the table in the same - words; the registry's gate reads the same field for every kind. - """ - machine = (getattr(entry, "machine", None) or "").strip() or None - if machine and not machine_lane_enabled(): - logger.info( - "sub-agent {!r} names machine {!r}, and the machine lane is debug-only: the entry stays " - "off the table until raven is started with {}=1", - entry.name, - machine, - MACHINE_LANE_ENV, - ) - return machine + """``entry.machine`` as a stripped registry id, or ``None`` for a local entry.""" + return (getattr(entry, "machine", None) or "").strip() or None class ThirdPartyCliSubagentConfig(Base): @@ -1629,12 +1614,7 @@ class ThirdPartyCliSubagentConfig(Base): on that machine. Raven reaches the machine over one ssh link with the address, port and key the registry holds, none of which is written here. See :mod:`raven.node.client` and - :class:`raven.agent.subagent.backends.cli_agent.CliAgentBackend`. - - Debug-only until the lane matures, as for an acp entry: unless raven was - started with ``RAVEN_DEBUG_MACHINES=1`` (:func:`raven.home.machine_lane_enabled`), - an entry that names a machine is left off the table and no machine is - reached for it.""" + :class:`raven.agent.subagent.backends.cli_agent.CliAgentBackend`.""" remote_cwd: str | None = None """Where on ``machine`` this agent works, ``~/raven-work`` when unset; ignored without ``machine``. ``~`` is the registered user's home there.""" @@ -1804,10 +1784,7 @@ def _null_stateful_is_the_default(cls, value: Any) -> Any: name. The files that come with a task are named to the endpoint by their address on this computer and not copied, as to the other kinds on a machine. The address, port and key the registry holds are read for the - tunnel alone and written nowhere here. Debug-only as for the other kinds: - unless raven was started - with ``RAVEN_DEBUG_MACHINES=1``, the entry is left off the table and no - machine is reached for it. There is no ``remoteCwd``: an endpoint has no + tunnel alone and written nowhere here. There is no ``remoteCwd``: an endpoint has no working directory on the machine, and one written here is dropped with a warning.""" @@ -2135,12 +2112,7 @@ class ThirdPartyAcpSubagentConfig(Base): written here. Each session works in ``/`` there. MCP servers are not lent to it, files are not handed to it, and what it writes stays on that machine. See :mod:`raven.node.client` and - :mod:`raven.acp_client.remote`. - - Debug-only until the lane matures: unless raven was started with - ``RAVEN_DEBUG_MACHINES=1`` (:func:`raven.home.machine_lane_enabled`), an - entry that names a machine is left off the table and no machine is - reached for it.""" + :mod:`raven.acp_client.remote`.""" remote_cwd: str | None = None """Where on ``machine`` this agent works, ``~/raven-work`` when unset; ignored without ``machine``. ``~`` is the registered user's home there.""" diff --git a/raven/home.py b/raven/home.py index 1aabc70f1..28c727446 100644 --- a/raven/home.py +++ b/raven/home.py @@ -26,11 +26,9 @@ from raven.contracts.path_policy import CONFIG_FILENAME, DEFAULT_HOME_DIRNAME, HOME_ENV_VAR __all__ = [ - "MACHINE_LANE_ENV", "SUBAGENT_ENV_VAR", "get_config_path", "is_subagent_process", - "machine_lane_enabled", "raven_home", "set_config_path", "subagent_role_env", @@ -85,18 +83,3 @@ def subagent_role_env() -> dict[str, str]: registry -- the failure this exists to close, and one no error reports. """ return {SUBAGENT_ENV_VAR: "1"} - - -#: Set to ``1`` to turn the machine lane on: acp sub-agents that run on a -#: registered machine, and files read there through a Raven node. Off by -#: default -- the lane is debug-only until it matures (ruled 2026-10-11), so an -#: entry that names a machine stays inert in normal operation. An environment -#: variable rather than a config field, because the model edits config through -#: ``raven_config`` and must not be able to turn the lane on itself; whoever -#: starts the process sets it. -MACHINE_LANE_ENV = "RAVEN_DEBUG_MACHINES" - - -def machine_lane_enabled() -> bool: - """Whether this process may dispatch to, and read files on, registered machines.""" - return os.environ.get(MACHINE_LANE_ENV, "").strip().lower() in ("1", "true", "yes", "on") diff --git a/raven/ops/remote_path.py b/raven/ops/remote_path.py index 89d5b5b09..0df002076 100644 --- a/raven/ops/remote_path.py +++ b/raven/ops/remote_path.py @@ -25,10 +25,9 @@ ``scp://user@host:port//abs/path`` -- the port only fits the URI form. Reads the registry, and for :func:`reading` and :func:`writing` the agent -entries' ``remoteCwd``; never ssh, never a node. The machine lane is debug-only -(:func:`raven.home.machine_lane_enabled`): with it off, :func:`target` names -no machine for any path, and the file tools read every path as they always -did. +entries' ``remoteCwd``; never ssh, never a node. Remote spellings are always +resolved against the registry; an explicit local spelling such as ``./a:b`` +stays on this computer. """ from __future__ import annotations @@ -40,8 +39,6 @@ from dataclasses import dataclass, field from typing import Any -from raven.home import machine_lane_enabled - #: Where an agent works on its machine when its entry names nowhere else, and #: so the one directory every registered machine reads from without asking. #: ``~`` is the registered user's home there, expanded by the machine. @@ -215,8 +212,8 @@ def forget_read(machine: str, path: str, *, reader: str = "") -> None: def target(text: str) -> RemotePath | None: - """:func:`parse` against the registry as it is now; ``None`` for every path while the lane is off.""" - if not machine_lane_enabled() or not looks_remote(text): + """:func:`parse` against the registry as it is now; ``None`` for a local path.""" + if not looks_remote(text): return None from raven.ops.connections import load diff --git a/raven/rpc/methods/subagents.py b/raven/rpc/methods/subagents.py index 1b4551615..72632286f 100644 --- a/raven/rpc/methods/subagents.py +++ b/raven/rpc/methods/subagents.py @@ -535,7 +535,7 @@ def _refuse_unless_it_can_run_there(entry: dict, machine: str, *, remote_cwd: st with it rather than written and ignored. A built-in agent is Raven's own loop in this process and runs on no other machine. The machine is judged the way a dispatch judges it (:func:`raven.acp_client.remote.machine`): - the lane on, the row registered and usable -- each refusal one sentence, + the row registered and usable -- each refusal one sentence, with no address in it. """ from raven.acp_client import remote diff --git a/raven/rpc/models.py b/raven/rpc/models.py index 440cb725e..994613360 100644 --- a/raven/rpc/models.py +++ b/raven/rpc/models.py @@ -2669,7 +2669,7 @@ class SubagentsAddParams(_Strict): "A registered machine, by its id in the machine registry: an acp or cli preset's command is started " "there by the machine's Raven node and signed in there; an openai preset's endpoint is reached from " "that machine through an ssh tunnel. The row's name defaults to @. Refused for a " - "built-in agent, and unless the machine lane is on and the machine is registered and usable." + "built-in agent, and unless the machine is registered and usable." ), ) remote_cwd: str | None = Field( diff --git a/rpc-schema/openrpc.json b/rpc-schema/openrpc.json index 2d0ba9210..241906bf6 100644 --- a/rpc-schema/openrpc.json +++ b/rpc-schema/openrpc.json @@ -2155,7 +2155,7 @@ { "name": "machine", "required": false, - "description": "A registered machine, by its id in the machine registry: an acp or cli preset's command is started there by the machine's Raven node and signed in there; an openai preset's endpoint is reached from that machine through an ssh tunnel. The row's name defaults to @. Refused for a built-in agent, and unless the machine lane is on and the machine is registered and usable.", + "description": "A registered machine, by its id in the machine registry: an acp or cli preset's command is started there by the machine's Raven node and signed in there; an openai preset's endpoint is reached from that machine through an ssh tunnel. The row's name defaults to @. Refused for a built-in agent, and unless the machine is registered and usable.", "schema": { "type": "string" } diff --git a/tests/test_acp_remote.py b/tests/test_acp_remote.py index 3842f606e..3093c3d8d 100644 --- a/tests/test_acp_remote.py +++ b/tests/test_acp_remote.py @@ -34,12 +34,6 @@ } -@pytest.fixture(autouse=True) -def _lane_on(monkeypatch: pytest.MonkeyPatch) -> None: - """These tests are about reaching a machine; the switch that allows it is tested on its own.""" - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") - - def _registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, rows: list | str) -> None: path = tmp_path / "connections.json" path.write_text(rows if isinstance(rows, str) else json.dumps({"connections": rows}), encoding="utf-8") @@ -418,31 +412,18 @@ def test_what_is_sent_is_one_quoted_sh_line_with_csh_s_history_character_escaped assert remote.posix("echo a!b") == "exec /bin/sh -c 'echo; echo a'\\!'b'" -# --- the lane is debug-only ------------------------------------------------------------ +# --- registered machines --------------------------------------------------------------- -def test_with_the_lane_off_a_machine_is_refused_before_the_registry_is_read(tmp_path, monkeypatch): +@pytest.mark.parametrize("value", [None, "0", "1"]) +def test_a_registered_machine_needs_no_debug_switch(tmp_path, monkeypatch, value): _registry(tmp_path, monkeypatch, [ROW]) - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") - monkeypatch.setattr(remote.connections, "read", lambda: pytest.fail("the registry was read")) - - with pytest.raises(RemoteMachineError) as caught: - remote.machine("box") - - assert str(caught.value) == ( - "machine 'box': the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1 " - "to use agents and files on registered machines" - ) - - -@pytest.mark.parametrize("value", ["1", "true", "YES", " on "]) -def test_the_lane_switch_reads_the_usual_spellings(monkeypatch, value): - from raven.home import machine_lane_enabled - - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", value) - assert machine_lane_enabled() - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "0") - assert not machine_lane_enabled() + if value is None: + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + else: + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", value) + target = remote.machine("box") + assert target.id == "box" and target.display_name == "Lab box" # --- one command, and a file sent ---------------------------------------------------- diff --git a/tests/test_deliver_files_tool.py b/tests/test_deliver_files_tool.py index 7e17ead28..360d33c02 100644 --- a/tests/test_deliver_files_tool.py +++ b/tests/test_deliver_files_tool.py @@ -378,7 +378,6 @@ def machines(tmp_path, monkeypatch): registry = tmp_path / "connections.json" registry.write_text(json.dumps({"connections": [row()]}), encoding="utf-8") monkeypatch.setenv("RAVEN_CONNECTIONS", str(registry)) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") monkeypatch.setattr(remote_path, "_HOMES", {}) monkeypatch.setattr(remote_path, "_READ", {}) monkeypatch.setattr( diff --git a/tests/test_node_client.py b/tests/test_node_client.py index a373607a8..deec328bc 100644 --- a/tests/test_node_client.py +++ b/tests/test_node_client.py @@ -129,8 +129,6 @@ def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch, rows: list[dict[ self.registry = root / "connections.json" self.rows(rows or [row()]) monkeypatch.setenv("RAVEN_CONNECTIONS", str(self.registry)) - # The lane is debug-only; these tests are about what it does when on. - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") path = f"{self.bin}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}" monkeypatch.setattr(backend_env, "_LOGIN_ENV", {**os.environ, "PATH": path}) monkeypatch.setattr("raven.config.paths.get_logs_dir", lambda: root / "logs") @@ -370,16 +368,13 @@ async def test_a_call_that_runs_out_of_time_closes_the_node(box, pool): assert pool.status("box").state == "error" -async def test_with_the_lane_off_no_node_is_started(box, pool, monkeypatch): +@pytest.mark.slow +async def test_a_node_starts_without_a_debug_switch(box, pool, monkeypatch): box.node() - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") - - with pytest.raises( - NodeError, match="the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1" - ): - await pool.hello("box") - - assert box.launches() == [] and pool.live() == [] + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + hello = await pool.hello("box") + assert hello["protocol"] == protocol.PROTOCOL + assert len(box.launches()) == 1 async def test_a_machine_that_is_not_registered_is_refused_before_any_ssh(box, pool): diff --git a/tests/test_node_install.py b/tests/test_node_install.py index 3d8e93952..a806bb757 100644 --- a/tests/test_node_install.py +++ b/tests/test_node_install.py @@ -43,7 +43,6 @@ def write(**extra: Any) -> None: write() monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") return write @@ -186,13 +185,16 @@ def test_a_look_that_failed_past_ssh_says_so(registry): install.plan("box", run=lambda target, command: (2, "df: weird")) -def test_with_the_lane_off_a_plan_is_refused_before_any_ssh(registry, monkeypatch): - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") +def test_an_install_plan_needs_no_debug_switch(registry, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + asked = [] - with pytest.raises( - InstallError, match="the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1" - ): - install.plan("box", run=lambda target, command: pytest.fail("no ssh with the lane off")) + def run(target, command): + asked.append(target.id) + return 0, "PYTHON=/usr/bin/python3.12\nUV=/usr/bin/uv\nFREE_KB=1048576\n" + + plan = install.plan("box", run=run) + assert plan.target.id == "box" and asked == ["box"] def test_a_machine_that_is_not_registered_is_refused_before_any_ssh(registry): diff --git a/tests/test_ops_remote_path.py b/tests/test_ops_remote_path.py index 00997b300..6552abe72 100644 --- a/tests/test_ops_remote_path.py +++ b/tests/test_ops_remote_path.py @@ -233,7 +233,6 @@ def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): path = tmp_path / "connections.json" path.write_text(json.dumps({"connections": ROWS}), encoding="utf-8") monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") @pytest.fixture @@ -322,14 +321,11 @@ def test_the_fence_is_the_root_a_path_is_under_else_what_was_approved(registry, assert remote_path.fence("read_file", where, "/id", "/root") == "/" -def test_with_the_lane_off_no_path_names_a_machine(registry, agents, monkeypatch): - # Off, the file tools read every path as they always did: a name with a - # colon is a local file, scp:// is a URL, and no registry is consulted. - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") - - assert remote_path.target("box2:/root/x") is None - assert remote_path.target("scp://root@203.0.113.7:49273//root/x") is None - assert remote_path.reading("read_file", {"path": "box2:.ssh/id_rsa"}) is None +def test_registered_paths_are_remote_without_a_debug_switch(registry, agents, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + assert remote_path.target("box2:/root/x").machine == "box2" + assert remote_path.target("scp://root@203.0.113.7:49273//root/x").machine == "box2" + assert remote_path.reading("read_file", {"path": "box2:.ssh/id_rsa"}) is not None def test_the_node_pool_is_whatever_assembly_handed_over(): diff --git a/tests/test_permissions_remote_read.py b/tests/test_permissions_remote_read.py index 9c3159b6f..e981c9795 100644 --- a/tests/test_permissions_remote_read.py +++ b/tests/test_permissions_remote_read.py @@ -35,7 +35,6 @@ def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): path = tmp_path / "connections.json" path.write_text(json.dumps({"connections": ROWS}), encoding="utf-8") monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") monkeypatch.setattr(remote_path, "_HOMES", {}) monkeypatch.setattr( "raven.config.loader.load_config", @@ -66,11 +65,13 @@ def test_a_remote_read_is_free_in_the_roots_and_asks_elsewhere(tool, path, tier) assert default_tier(tool, {"path": path}) is tier -def test_with_the_lane_off_a_read_is_tiered_as_it_always_was(monkeypatch): - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") - - assert default_tier("read_file", {"path": "box:.ssh/id_rsa"}) is Tier.ALLOW - assert session_keys("read_file", {"path": "box:.ssh/id_rsa"}) != session_keys("read_file", {"path": "box:.ssh/x"}) +def test_remote_permissions_apply_without_a_debug_switch(monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + assert default_tier("read_file", {"path": "box:.ssh/id_rsa"}) is Tier.ASK + assert default_tier("write_file", {"path": "box:raven-work/report.md", "content": "x"}) is Tier.ASK + assert session_keys("read_file", {"path": "box:.ssh/id_rsa"}) == session_keys( + "read_file", {"path": "box:.ssh/config"} + ) def test_an_absolute_path_reads_freely_once_the_machine_s_home_is_known(): diff --git a/tests/test_raven_config_tool.py b/tests/test_raven_config_tool.py index e6dfcc6f7..aabe0f2b6 100644 --- a/tests/test_raven_config_tool.py +++ b/tests/test_raven_config_tool.py @@ -1830,7 +1830,7 @@ async def test_a_refused_add_on_a_machine_sends_the_user_to_that_machine_and_run @pytest.mark.asyncio -async def test_describe_names_the_registered_machines_only_while_the_lane_is_on(config_file, tmp_path, monkeypatch): +async def test_describe_names_registered_machines_without_a_debug_switch(config_file, tmp_path, monkeypatch): from raven.ops import connections registry = tmp_path / "connections.json" @@ -1857,9 +1857,6 @@ async def test_describe_names_the_registered_machines_only_while_the_lane_is_on( tool.set_rpc_caller(Calls({"subagents.list": {"rows": []}})) monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - assert "machines a preset can be added on" not in await _run(tool, action="describe", path="subagents") - - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") said = await _run(tool, action="describe", path="subagents") line = next(ln for ln in said.splitlines() if ln.strip().startswith("machines a preset can be added on")) assert line.split("on:", 1)[1].split("--", 1)[0].strip() == "ravenx2 (RavenX 2)", "the local row is no machine" diff --git a/tests/test_remote_file_tools.py b/tests/test_remote_file_tools.py index c4cc57753..050a10992 100644 --- a/tests/test_remote_file_tools.py +++ b/tests/test_remote_file_tools.py @@ -43,7 +43,6 @@ def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: path = tmp_path / "connections.json" path.write_text(json.dumps({"connections": [row()]}), encoding="utf-8") monkeypatch.setenv("RAVEN_CONNECTIONS", str(path)) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") return path @@ -201,13 +200,12 @@ async def test_a_node_that_cannot_answer_is_one_sentence(registry, monkeypatch, assert HOST not in got -async def test_with_the_lane_off_a_remote_path_is_read_as_it_always_was(registry, pool, tmp_path, monkeypatch): - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") - +async def test_a_remote_path_reaches_its_node_without_a_debug_switch(registry, pool, tmp_path, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) got = await _tools(tmp_path)["read_file"].execute("box:raven-work/t1/a.md") - - assert got.startswith("Error: File not found: box:raven-work/t1/a.md") - assert pool.calls == [] + assert got == "ok" + assert pool.calls[0][0:2] == ("box", "fs/read") + assert pool.calls[0][2]["path"] == f"{HOME}/raven-work/t1/a.md" async def test_a_local_path_is_read_here_as_before(registry, pool, tmp_path): diff --git a/tests/test_remote_resume.py b/tests/test_remote_resume.py index b36196e78..8ce593710 100644 --- a/tests/test_remote_resume.py +++ b/tests/test_remote_resume.py @@ -23,7 +23,6 @@ def _machines(tmp_path, monkeypatch): registry = tmp_path / "connections.json" registry.write_text(json.dumps({"connections": [row()]}), encoding="utf-8") monkeypatch.setenv("RAVEN_CONNECTIONS", str(registry)) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") monkeypatch.setenv("RAVEN_HOME", str(tmp_path / "home")) monkeypatch.setattr(remote_path, "_NODES", None) diff --git a/tests/test_resume_remote.py b/tests/test_resume_remote.py index 83c8c2403..1cf94d9cf 100644 --- a/tests/test_resume_remote.py +++ b/tests/test_resume_remote.py @@ -448,7 +448,6 @@ async def test_a_cli_run_is_picked_up_by_its_cli_entry_and_an_acp_record_is_not_ from raven.agent.subagent.backends import build_third_party_backend from raven.config.schema import ThirdPartyCliSubagentConfig - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") cli = build_third_party_backend( ThirdPartyCliSubagentConfig(name="sh@box", command="cat", machine="box", remote_cwd="~/raven-work") ) @@ -473,3 +472,36 @@ async def pick_up(run, *, pool=None): _run(task="stub@box-2", agent="sh@box") assert await resume_remote.resume_remote_runs(manager) == 1 assert "no longer on the agent table as an acp agent" in manager.announced[-1]["result"] + + +@pytest.mark.parametrize("kind", ["acp", "cli"]) +@pytest.mark.parametrize("value", [None, "0"]) +async def test_pending_machine_runs_resume_without_a_debug_switch(tmp_path, monkeypatch, kind, value): + from raven.agent.subagent.registry import AgentRegistry + from raven.config.schema import ThirdPartyAcpSubagentConfig, ThirdPartyCliSubagentConfig + + if value is None: + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + else: + monkeypatch.setenv("RAVEN_DEBUG_MACHINES", value) + cls = ThirdPartyAcpSubagentConfig if kind == "acp" else ThirdPartyCliSubagentConfig + cfg = cls(name="stub@box", command="cat", machine="box") + registry = AgentRegistry(build_builtin=lambda row, narrowed: ("backend", row.name)) + registry.apply([cfg]) + manager = _Manager(None, tmp_path / "s1") + manager.registry = registry + run = _run(kind=kind) + picked = [] + + async def pick_up(saved, backend): + picked.append(saved.task) + assert backend.machine == "box" + return remote_resume.PickedUp("ok", "recovered answer") + + monkeypatch.setattr(resume_remote, "_pick_up", pick_up) + monkeypatch.setattr(resume_remote, "get_registry", lambda: _Registry()) + assert await resume_remote.resume_remote_runs(manager) == 1 + assert picked == [run.task] + assert manager.announced[0]["status"] == "ok" + assert manager.announced[0]["result"] == "recovered answer" + assert remote_runs.pending() == [] diff --git a/tests/test_rpc_subagents.py b/tests/test_rpc_subagents.py index 22687f05f..966b5c74d 100644 --- a/tests/test_rpc_subagents.py +++ b/tests/test_rpc_subagents.py @@ -3149,13 +3149,12 @@ async def _no_record(cfg: object) -> None: @pytest.fixture def registry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """A machine registry with one usable row, and the machine lane on.""" + """A machine registry with one usable row.""" from raven.ops import connections path = tmp_path / "connections.json" path.write_text(json.dumps({"connections": _MACHINES}), encoding="utf-8") monkeypatch.setattr(connections, "store_path", lambda: path) - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") return path @@ -3195,25 +3194,20 @@ async def test_add_on_a_machine_keeps_the_name_the_caller_chose( assert entry["machine"] == "ravenx2" and not entry.get("remoteCwd"), "unset: the node's default applies" -async def test_add_on_a_machine_is_refused_with_the_lane_off_and_nothing_is_stored_or_pinged( - config_path: Path, registry: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("RAVEN_DEBUG_MACHINES") - pinged: list[object] = [] +async def test_add_on_a_machine_needs_no_debug_switch(config_path, registry, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + pinged = [] - async def _ping(cfg: object) -> object: + async def ping(cfg): pinged.append(cfg) return await _pings_ok(cfg) - monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", _ping) - before = _stored(config_path) - - with pytest.raises(ConfigValidationError) as caught: - await subagents_add({"preset": "claude_code", "machine": "ravenx2"}) - - assert "machine lane is debug-only" in str(caught.value) and "RAVEN_DEBUG_MACHINES=1" in str(caught.value) - assert caught.value.data["field"] == "machine" - assert pinged == [] and _stored(config_path) == before + monkeypatch.setattr("raven.rpc.methods.subagents.ping_agent", ping) + monkeypatch.setattr("raven.rpc.methods.subagents.record_capabilities", _no_record) + out = await subagents_add({"preset": "claude_code", "machine": "ravenx2"}) + assert out == {"added": True, "name": "claude_code@ravenx2"} + assert [cfg.machine for cfg in pinged] == ["ravenx2"] + assert any(row.get("machine") == "ravenx2" for row in _stored(config_path)) @pytest.mark.parametrize(("machine", "said"), [("nowhere", "is not registered"), (" ", "must not be blank")]) diff --git a/tests/test_shell_machine_channel.py b/tests/test_shell_machine_channel.py index e19b968bf..531a1ec6e 100644 --- a/tests/test_shell_machine_channel.py +++ b/tests/test_shell_machine_channel.py @@ -950,18 +950,9 @@ def test_on_a_windows_host_the_words_are_read_without_their_quotes(monkeypatch): @pytest.mark.asyncio -async def test_typed_ssh_s_refusal_leaves_copies_alone_and_names_the_file_tools_only_with_the_lane_on( - registry, tmp_path, monkeypatch -): - # With the lane off the file tools read a remote spelling as a path here, - # so naming them would send the model to write on this computer. +async def test_typed_ssh_s_refusal_names_file_tools_without_a_debug_switch(registry, tmp_path, monkeypatch): tool = ExecTool(working_dir=str(tmp_path)) monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - - off = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") - on = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") - - assert "scp and rsync to it are still fine here" in off and "read_file" not in off - assert "scp and rsync to it are still fine here" in on - assert "read_file, write_file and edit_file on the path the way scp writes it" in on + said = await tool.execute(command="ssh -p 58717 root@203.0.113.7 ls") + assert "scp and rsync to it are still fine here" in said + assert "read_file, write_file and edit_file on the path the way scp writes it" in said diff --git a/tests/test_subagent_acp_remote.py b/tests/test_subagent_acp_remote.py index 3ebf8593c..f739f322f 100644 --- a/tests/test_subagent_acp_remote.py +++ b/tests/test_subagent_acp_remote.py @@ -122,8 +122,6 @@ def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch) -> None: encoding="utf-8", ) monkeypatch.setenv("RAVEN_CONNECTIONS", str(registry)) - # The lane is debug-only; these tests are about what it does when on. - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") path = f"{self.bin}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}" # Both ways ssh is started: the one-shot runner inherits this process's # PATH, and an agent's launch takes the captured login-shell one. @@ -881,41 +879,30 @@ def get_default_model(self) -> str: assert mgr._machine_workdir("nobody", "t1") == ("", "") -# ---- the lane is debug-only -------------------------------------------------------------- +# ---- registered machines ---------------------------------------------------------------- -def _lane_off(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - - -def test_with_the_lane_off_a_machine_row_stays_off_the_table(box, monkeypatch): +def test_machine_and_local_acp_rows_are_on_the_table_without_a_debug_switch(box, monkeypatch): from raven.agent.subagent.registry import AgentRegistry - local = ThirdPartyAcpSubagentConfig(name="here", command=f"{sys.executable} {_STUB}") + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) registry = AgentRegistry(build_builtin=lambda row, narrowed: ("backend", row.name)) - - _lane_off(monkeypatch) + local = ThirdPartyAcpSubagentConfig(name="here", command="stub") registry.apply([remote_config(), local]) - assert "stub@box" not in registry.names() and "here" in registry.names() + assert "stub@box" in registry.names() and "here" in registry.names() + assert box.calls() == [], "constructing the table does not connect to a machine" - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") - registry.apply([remote_config(), local]) - assert "stub@box" in registry.names() - - -async def test_with_the_lane_off_nothing_reaches_the_machine_and_each_door_says_so(box, monkeypatch, tmp_path): - _lane_off(monkeypatch) - said = "the machine lane is debug-only and off; start raven with RAVEN_DEBUG_MACHINES=1" - - with pytest.raises(AcpConnectionError, match=said): - await build_third_party_backend(remote_config()).run("hi", task_id="t1", workspace=tmp_path, executor=None) - probed = await probe.probe_one(remote_config(), source="config") - assert probed.status == "attention" and said in probed.detail - verified = await verify_agent(remote_config()) - assert verified.status == "attention" and said in verified.detail - tested = await probe.run_test(remote_config(), source="config") - assert tested.ok is False and said in tested.detail and tested.remedy is None - assert box.calls() == [], "off is off: no ssh for a dispatch, a probe, a verify or a test" + +async def test_acp_dispatch_and_checks_need_no_debug_switch(box, monkeypatch, tmp_path): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) + cfg = remote_config() + probed = await probe.probe_one(cfg, source="config") + assert box.calls() == [], "the free probe must not open a connection" + assert "debug-only" not in probed.detail + reply = await build_third_party_backend(cfg).run("hi", task_id="default-on", workspace=tmp_path, executor=None) + assert "pong" in reply + assert (await verify_agent(cfg)).status == "ready" + assert (await probe.run_test(cfg, source="config")).ok async def test_the_boot_backfill_never_logs_in_to_a_machine(box): diff --git a/tests/test_subagent_cli_remote.py b/tests/test_subagent_cli_remote.py index 064070d8c..e493019e1 100644 --- a/tests/test_subagent_cli_remote.py +++ b/tests/test_subagent_cli_remote.py @@ -679,25 +679,19 @@ def test_an_openai_row_drops_a_remote_cwd_with_a_warning_and_the_write_door_refu reject_unsupported_openai_fields([row]) -def test_with_the_lane_off_a_cli_machine_row_stays_off_the_table(box, monkeypatch): +def test_machine_and_local_cli_rows_are_on_the_table_without_a_debug_switch(box, monkeypatch): + monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) registry = AgentRegistry(build_builtin=lambda row, narrowed: ("backend", row.name)) local = ThirdPartyCliSubagentConfig(name="here", command="cat") - - monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - registry.apply([cli_config("cat"), local]) - assert "where@box" not in registry.names() and "here" in registry.names() - - monkeypatch.setenv("RAVEN_DEBUG_MACHINES", "1") registry.apply([cli_config("cat"), local]) - assert "where@box" in registry.names() + assert "where@box" in registry.names() and "here" in registry.names() + assert box.calls() == [] -async def test_with_the_lane_off_a_cli_dispatch_says_so_and_reaches_nothing(box, monkeypatch, tmp_path): +async def test_a_cli_dispatch_needs_no_debug_switch(box, monkeypatch, tmp_path): monkeypatch.delenv("RAVEN_DEBUG_MACHINES", raising=False) - - with pytest.raises(RuntimeError, match="the machine lane is debug-only and off"): - await _run(cli_config("cat"), "hi", tmp_path) - assert box.calls() == [] + reply = await _run(cli_config("cat"), "hi", tmp_path) + assert reply.startswith("hi") and "Ran on machine" in reply async def test_the_announce_names_a_cli_row_s_directory_on_the_machine(box): From b184d82d9a4591faf652b1d59eee3e658d6498e5 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:26:41 +0800 Subject: [PATCH 42/44] test(agent): the direct chat's gating stand-in forwards only what the fake behind it declares Its `**kw` told the manager it takes every keyword, so the manager handed it the record keyword a run on a machine takes (`reported_by_caller`), which the fake behind it does not declare, and the queued spawn failed instead of completing. Co-authored-by: Claude (claude-fable-5-1) --- tests/test_subagent_direct_chat.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_subagent_direct_chat.py b/tests/test_subagent_direct_chat.py index 1ce680117..8e61bade1 100644 --- a/tests/test_subagent_direct_chat.py +++ b/tests/test_subagent_direct_chat.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import inspect import json from typing import Any @@ -1730,7 +1731,12 @@ async def test_a_spawn_queued_behind_a_direct_chat_stays_pending_and_leaves_the_ async def gated(task, **kw): if task == "hold": await release.wait() - return await original(task, **kw) + # ``**kw`` tells the manager this stand-in takes every keyword, so it + # is handed the ones the fake behind it does not declare (the record + # keyword a run on a machine takes): forwarded only as far as the + # fake's own paper, the way the manager hands them over. + taken = inspect.signature(original).parameters + return await original(task, **{key: value for key, value in kw.items() if key in taken}) monkeypatch.setattr(manager.registry.backend("raven"), "run", gated) statuses: list[str] = [] From 32532eceedaed8e14c740a932d2dfa8baf66569b Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:27:39 +0800 Subject: [PATCH 43/44] test(agent): accept the result ownership keyword in the direct chat stub The queued-spawn test wraps the backend with a keyword-forwarding gate. Match the backend call's result ownership keyword in its fake so the test reaches the queue completion assertion instead of a signature error. Co-authored-by: Codex --- tests/test_subagent_direct_chat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_subagent_direct_chat.py b/tests/test_subagent_direct_chat.py index 1ce680117..78d9ecb42 100644 --- a/tests/test_subagent_direct_chat.py +++ b/tests/test_subagent_direct_chat.py @@ -40,6 +40,7 @@ async def fake_run( model=None, mode=None, authored_task=None, + reported_by_caller=False, history=None, on_messages=None, on_delta=None, From 2052eb3888e1b44cdc5e00f55602ac44e7ccb559 Mon Sep 17 00:00:00 2001 From: Handsome-wzw <68996445+Handsome-wzw@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:30:06 +0800 Subject: [PATCH 44/44] docs(*): the pick-up's docstrings say the turn is replayed whole from the prompt Two docstrings still said the daemon replays only what this computer never read; the record's count now stops at the prompt, so the turn is read again from its first frame. Co-authored-by: Claude (claude-fable-5-1) --- raven/acp_client/remote_resume.py | 7 ++++--- raven/node/client.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/raven/acp_client/remote_resume.py b/raven/acp_client/remote_resume.py index 847540ac0..0f3039de9 100644 --- a/raven/acp_client/remote_resume.py +++ b/raven/acp_client/remote_resume.py @@ -2,9 +2,10 @@ The run's record (:mod:`raven.acp_client.remote_runs`) says which machine, which task its daemon runs the agent under, which ACP session the turn is in, -which prompt is awaited and how many frames that process had read. This -module turns that into the reply the turn owes: the machine's daemon replays -what was never read and hands the rest over as it comes (``agent/attach``), +which prompt is awaited and how many frames that process had read when the +prompt went out. This module turns that into the reply the turn owes: the +machine's daemon replays the turn from there, whole, and hands the rest over +as it comes (``agent/attach``), a connection over that channel is taken into the pool as if started here, the turn's collector reads the agent's messages off it, and the answer to the prompt -- sent by the other process, awaited by this one -- ends the wait. What diff --git a/raven/node/client.py b/raven/node/client.py index 512d0882e..5bf99f2be 100644 --- a/raven/node/client.py +++ b/raven/node/client.py @@ -563,9 +563,10 @@ async def attach( """``task`` on the machine, onto a channel of its link, from the frame past the ``after``-th. What a process that comes after the one that started the agent uses - to pick the run up: the daemon replays what this computer never read - and hands the rest over as it comes, or, for a task that has ended, - replays its record and says so. The endpoint returned is what an + to pick the run up: the daemon replays the frames past the record's + count -- the turn whole, from where its prompt went out -- and hands + the rest over as it comes, or, for a task that has ended, replays its + record and says so. The endpoint returned is what an :class:`AcpClient` runs over; the answer says the pid, the frames and the exit code (``None`` while the agent runs). """