feat(*): fold a large tool catalog by default, with the meta-pair reserved

CHANGELOG.md

@@ -22,6 +22,19 @@ All notable changes to Raven are documented here.
 
 ### Changed
 
+- Progressive tool disclosure ships on (`tools.toolSearch.enabled`). Below
+  `compactionThreshold` (50) nothing changes: the strategy drops `tool_search`
+  from every request while the catalog fits. Above it, most tool schemas are
+  withheld and reached through `tool_search` / `tool_call`, so context stops
+  scaling with tool count and an MCP connect no longer moves the cached prompt
+  prefix. Set `enabled: false` to keep the old shape.
+
+- `tools.disabledTools` can no longer take `tool_search` or `tool_call` away.
+  Their absence from a request is how the fold reads "no search route", so an
+  entry there did not slim a folded request, it unfolded it. An entry naming
+  either is reported once at startup and ignored; `tools.toolSearch.enabled` is
+  the switch that speaks for the pair.
+
 - `ppt_generate_image` no longer answers a repeat ask from the file the last
   ask wrote. The tool exists for "that picture is not what I wanted, do it
   again", and that ask carries the same words as the first one, so the cache

agents/raven-code/config.json

@@ -39,8 +39,6 @@
       "spawn",
       "message",
       "hub",
-      "tool_search",
-      "tool_call",
       "deep_research",
       "image_generate",
       "text_to_speech",

agents/raven-oncall/config.json

@@ -40,8 +40,6 @@
       "read_skill",
       "use_skill",
       "hub",
-      "tool_search",
-      "tool_call",
       "deep_research",
       "image_generate",
       "text_to_speech",

agents/raven-ppt/config.json

@@ -95,8 +95,6 @@
       "run_subagent_dag",
       "spawn",
       "text_to_speech",
-      "tool_call",
-      "tool_search",
       "video_generate",
       "browser_click",
       "browser_navigate",

agents/raven-research/config.json

@@ -131,8 +131,6 @@
       "read_skill",
       "use_skill",
       "find_skill",
-      "tool_search",
-      "tool_call",
       "image_generate",
       "text_to_speech",
       "video_generate",

raven/agent/loop/mcp_glue.py

@@ -172,7 +172,7 @@ def _mcp_tool_notices(self) -> list[str]:
         """Host facts about MCP tools the definitions cannot carry.
 
         One line per enabled server whose tools are absent from this turn for a
-        reason the model cannot see, and the two reasons need different lines:
+        reason the model cannot see, and the three reasons need different lines:
 
         * ``auth_required`` -- the wait is on a person, and the answer names who
           can end it. Without this the model reads an unauthorized plugin as a
@@ -181,6 +181,24 @@ def _mcp_tool_notices(self) -> list[str]:
           that overtakes it, which since ``prewarm_mcp`` is the ordinary shape of
           a first turn rather than a rarity: the turn no longer waits, so it can
           be assembled while servers are still coming up.
+        * ``connected`` while the catalog is folded -- nothing else in the prompt
+          names an installed server, so a model that does not think to search
+          reports a capability it holds as missing. ``tool_search``'s own
+          description says a catalog exists; only this says what is in it.
+
+          This line states an inventory and promises no route, which is what keeps
+          it true. This block is rendered once per turn, while the fold is decided
+          per model call against an array a ``before_iteration`` hook may narrow
+          first (``HookDecision.modified_tools``) -- so a turn-level sentence about
+          what this request folded, or about reaching it through ``tool_search`` /
+          ``tool_call``, can be falsified after it is written. "This server is
+          connected and offers N tools" cannot.
+
+          The count is the server's tools this turn's array carries, not the
+          manager's ``tool_count``: that one counts every wrapper the server
+          registered, including one the operator switched off, which no surface
+          offers. The fold gate is only about noise -- below it every schema is in
+          the array and the line tells the model nothing it cannot see.
 
         Rendered into the runtime-context block, not the system prompt -- the
         set changes turn to turn and must never be cached with the prefix.
@@ -195,6 +213,11 @@ def _mcp_tool_notices(self) -> list[str]:
         # that way -- an imperative here would be either ignored or a fence
         # violation. The fact alone is enough to stop it reporting a missing
         # capability.
+        controller = getattr(self, "tool_search_controller", None)
+        folded = controller is not None and controller.search_visible()
+        offered: set[str] = set()
+        if folded:
+            offered = {d["function"]["name"] for d in self.tools.get_definitions()}
         notices = []
         for snap in mgr.status():
             if not snap.get("enabled", True):
@@ -211,6 +234,10 @@ def _mcp_tool_notices(self) -> list[str]:
                     f"from this turn's definitions and register themselves when the handshake "
                     f"finishes, so they are available from a later turn without anyone acting."
                 )
+            elif folded and snap["connected"]:
+                mine = [n for n in self.tools.names_from(snap["name"]) if n in offered]
+                if mine:
+                    notices.append(f"MCP plugin '{snap['name']}': connected, offering {len(mine)} tool(s).")
         return notices
 
     def prewarm_mcp(self) -> None:

raven/agent/loop/wiring.py

@@ -82,12 +82,14 @@ def _report_reserved_disabled_tools(self) -> None:
         preference expressed by destroying its subject is one that cannot be
         reversed: nothing remembered what to put back.
 
-        The MCP meta-tools are the one exemption, and it is theirs by ownership
-        rather than by policy: :meth:`_sync_mcp_meta_tools` registers them while
-        some connected server serves resources or prompts and withdraws them when
-        none does, so it owns those five names for the life of the loop. An entry
-        naming one is a preference nothing can act on -- reported here once,
-        ignored where the array is built.
+        Two groups are exempt, and both are theirs by ownership rather than by
+        policy. :meth:`_sync_mcp_meta_tools` registers the five MCP meta-tools
+        while some connected server serves resources or prompts and withdraws them
+        when none does, so it owns those names for the life of the loop.
+        ``tool_search`` and ``tool_call`` are owned the same way by
+        :meth:`_register_default_tools`, and ``tools.tool_search.enabled`` is the
+        switch that speaks for them. An entry naming either group is a preference
+        nothing can act on -- reported here once, ignored where the array is built.
 
         Run after :meth:`_register_default_tools` and after MCP connect, so an
         entry naming a tool from either group is resolvable by the time it is
@@ -98,42 +100,73 @@ def _report_reserved_disabled_tools(self) -> None:
         is about the entry the operator can actually see -- most of them are in
         the config file, which is no longer copied into ``_disabled_tools``.
         """
+        from raven.agent.tools.tool_search import META_TOOL_NAMES, TOOL_SEARCH_NAME
         from raven.config.live import disabled_tool_names
         from raven.mcp.prompts import PROMPT_TOOL_NAMES
         from raven.mcp.resources import RESOURCE_TOOL_NAMES
 
         entries = set(disabled_tool_names(self._live_config)) | self._disabled_tools
         if not entries:
             return
-        reserved = RESOURCE_TOOL_NAMES | PROMPT_TOOL_NAMES
+        mcp_reserved = RESOURCE_TOOL_NAMES | PROMPT_TOOL_NAMES
         for entry in sorted(entries):
             if entry in self._disabled_tools_reserved_warned:
                 continue
-            if any(name in reserved for name in self.tools.resolve_configured(entry)):
+            resolved = self.tools.resolve_configured(entry)
+            if any(name in mcp_reserved for name in resolved):
                 self._disabled_tools_reserved_warned.add(entry)
                 logger.warning(
                     "tools.disabled_tools names '{}', which raven registers and withdraws on its "
                     "own as MCP servers serving resources or prompts come and go. The entry has "
                     "no effect; remove it to keep the config honest.",
                     entry,
                 )
+            elif any(name in META_TOOL_NAMES for name in resolved):
+                self._disabled_tools_reserved_warned.add(entry)
+                # One remedy per name, because the two are not owned the same
+                # way: tools.tool_search.enabled decides whether tool_search is
+                # registered at all, while tool_call is registered whatever that
+                # switch says -- it is the only route to a schema-hidden tool.
+                # A shared "turn the fold off" line sent an operator who wrote
+                # tool_call here to a setting that leaves it exactly where it was.
+                remedy = (
+                    "turn the fold off with tools.tool_search.enabled, which is what registers it"
+                    if TOOL_SEARCH_NAME in resolved
+                    else "this deploy has no switch for it: it is the only route to a tool whose "
+                    "schema is not in the array, so it is registered whatever the fold is doing"
+                )
+                logger.warning(
+                    "tools.disabled_tools names '{}', which raven owns for the life of the loop. "
+                    "The entry has no effect; remove it, or {}.",
+                    entry,
+                    remedy,
+                )
 
     def _withheld_tool_names(self) -> frozenset[str]:
         """Which tools are not on offer right now, read live.
 
         The registry asks this when it assembles a tool array. Reserved names are
-        removed here rather than at the switch: ``_sync_mcp_meta_tools`` owns those
-        five for the life of the loop (it registers them while some connected
-        server serves resources or prompts and withdraws them when none does), so
-        an entry naming one is a preference the loop cannot honour -- reported
-        above, ignored here.
+        removed here rather than at the switch: ``_sync_mcp_meta_tools`` owns the
+        five MCP meta-tools for the life of the loop (it registers them while some
+        connected server serves resources or prompts and withdraws them when none
+        does), so an entry naming one is a preference the loop cannot honour --
+        reported above, ignored here.
+
+        ``tool_search`` and ``tool_call`` are reserved for that reason and one
+        more. Their absence from the array is how ``ToolSearchStrategy`` reads
+        "this request has no search route", and it answers by shipping every
+        schema instead -- so an off switch here would not slim the array, it would
+        unfold it, and mid-turn at that, because this source is read once per
+        assembly. The switch that speaks for the fold is
+        ``tools.tool_search.enabled``.
 
         Constructor-supplied names are unioned in because an eval harness passes
         them directly rather than through a config file; a file-less run would
         otherwise lose its blacklist entirely. Nothing copies the config's own
         list in there -- see the note in ``__init__`` on why that made the switch
         one-way.
         """
+        from raven.agent.tools.tool_search import META_TOOL_NAMES
         from raven.config.live import disabled_tool_names
         from raven.mcp.prompts import PROMPT_TOOL_NAMES
         from raven.mcp.resources import RESOURCE_TOOL_NAMES
@@ -143,7 +176,7 @@ def _withheld_tool_names(self) -> frozenset[str]:
         for entry in configured:
             withheld.update(self.tools.resolve_configured(entry))
         withheld.update(self._unconfigured_tool_names())
-        return frozenset(withheld - (RESOURCE_TOOL_NAMES | PROMPT_TOOL_NAMES))
+        return frozenset(withheld - (RESOURCE_TOOL_NAMES | PROMPT_TOOL_NAMES | META_TOOL_NAMES))
 
     def _unconfigured_tool_names(self) -> set[str]:
         """Registered tools whose config asks for nothing right now, read live.

raven/agent/tools/tool_search.py

@@ -262,15 +262,18 @@ def resolve_target(self, name: Any) -> _Target:
             return _Target(None, f"Error: '{name}' cannot be invoked via tool_call.")
         tool = self._registry.get(name)
         if tool is None or not self._registry.offers(tool):
-            tail = " -- tool_search lists what is currently loaded" if self._search_visible() else ""
+            tail = " -- tool_search lists what is currently loaded" if self.search_visible() else ""
             return _Target(None, absent_tool_error(name, tail=tail))
         return _Target(tool, None)
 
-    def _search_visible(self) -> bool:
+    def search_visible(self) -> bool:
         """Whether ``tool_search`` reaches the model's tool list this turn.
 
-        The old shape of :meth:`tool_call_available`, kept but moved to the
-        meta-tool it is actually true of. ``tool_search`` is registered only
+        Public because it is also the honest gate on anything the host says
+        about the fold: ``_mcp_tool_notices`` tells the model a connected
+        server's schemas are withheld, and that sentence is true only while
+        this is. The old shape of :meth:`tool_call_available`, kept but moved to
+        the meta-tool it is actually true of. ``tool_search`` is registered only
         where the fold is configured on, withheld by the same off switch as
         anything else, and dropped from the request again below the threshold --
         so a pointer at it is dishonest in three separate states, not one.
@@ -435,6 +438,23 @@ async def execute(self, name: str, arguments: dict[str, Any] | None = None) -> s
         return await self._ctrl.call(name, arguments)
 
 
+def _schema_name(entry: Any) -> str | None:
+    """The name a tool definition carries, or ``None`` for a shape this strategy
+    cannot read.
+
+    The arrays raven assembles are always well formed; replay is what is not.
+    It feeds a recorded array back through this same chain, so a corrupt
+    recording arrives here as an entry whose ``function`` is a string -- and
+    that has to end at replay's own backstop as an unreplayable recording,
+    not as a TypeError thrown from a strategy that assumed the shape. An entry
+    this cannot read is counted as a tool and passed through untouched: the
+    fold may only withhold a schema it recognised.
+    """
+    function = entry.get("function") if isinstance(entry, dict) else None
+    name = function.get("name") if isinstance(function, dict) else None
+    return name if isinstance(name, str) else None
+
+
 class ToolSearchStrategy(TokenStrategy):
     """``before_llm_call`` hook that compacts the tool list for large catalogs.
 
@@ -470,16 +490,16 @@ async def before_llm_call(
         if not tools:
             return messages, tools, model
         self._ctrl.refresh()
-        catalog_size = sum(1 for t in tools if t["function"]["name"] not in META_TOOL_NAMES)
+        catalog_size = sum(1 for t in tools if _schema_name(t) not in META_TOOL_NAMES)
         if catalog_size <= self._compaction_threshold:
-            out = [t for t in tools if t["function"]["name"] != TOOL_SEARCH_NAME]
+            out = [t for t in tools if _schema_name(t) != TOOL_SEARCH_NAME]
             return messages, out, model
-        present = {t["function"]["name"] for t in tools}
+        present = {name for t in tools if (name := _schema_name(t)) is not None}
         if not META_TOOL_NAMES <= present:
             # Meta-tools unavailable (e.g. removed via disabled_tools): expose
             # everything rather than strand the cataloged tools behind a search
             # the model cannot invoke.
             return messages, tools, model
         visible = self._ctrl.visible_names()
-        out = [t for t in tools if t["function"]["name"] in visible]
+        out = [t for t in tools if (name := _schema_name(t)) is None or name in visible]
         return messages, out, model

raven/config/schema.py

@@ -1258,9 +1258,17 @@ class ToolSearchConfig(Base):
     there (``ToolRegistry.hide_from_schema``), and that has nothing to do with
     catalog size. Turning this off folds nothing; it does not take the name
     route away.
+
+    On by default, which costs a deploy under the threshold nothing: the
+    strategy drops ``tool_search`` from every request while the catalog fits, so
+    the switch only starts deciding anything once a deploy has more tools than a
+    request should carry. Neither name can be taken away through
+    ``tools.disabled_tools`` -- their absence is how the fold reads "this request
+    has no search route", so an off switch there would unfold the array rather
+    than slim it. This setting is the one that speaks for both.
     """
 
-    enabled: bool = False
+    enabled: bool = True
     compaction_threshold: int = 50
     """Tool-catalog size that triggers compaction: at or below this many tools
     everything is exposed directly; above it, schemas are withheld."""

raven/trajectory/replay.py

@@ -1070,6 +1070,7 @@ async def run_replay(bundle_dir: Path, mode: str = "warn") -> ReplayReport:
 
     from raven.agent.loop import AgentLoop
     from raven.agent.loop.bundles import ToolWiring
+    from raven.config.schema import ToolSearchConfig
     from raven.session.manager import SessionManager
     from raven.spine.message import ChatType, Source
     from raven.spine.turn import Origin, TurnRequest
@@ -1109,6 +1110,17 @@ async def _drop_delta(_text: str) -> None:
                 session_manager=sessions,
                 tools=ToolWiring(
                     restrict_to_workspace=True,
+                    # A replay reproduces a request; it does not assemble one.
+                    # The fold ships on, and a recording made above the
+                    # threshold already IS the folded array -- core plus the
+                    # meta-pair -- so serving it through the strategy again
+                    # counts a catalog under the threshold and drops
+                    # ``tool_search`` from what the replay sends. That reads as
+                    # a divergence in the offered tool names, which halts every
+                    # strict regression case before the first recorded reply.
+                    # Spelled out rather than inherited: this harness is a
+                    # deliberate bypass, not a deployment.
+                    tool_search_config=ToolSearchConfig(enabled=False),
                 ),
             )
             # The loop's ContextBuilder starts the skill file watcher, a