diff --git a/AGENTS.md b/AGENTS.md index 14e0c90..3c3f44e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,12 +44,12 @@ The wizard prompts (`pond init`, source discovery, etc.) go through cliclack/dia ## Search scope is intentional - do not "fix" it -- `search_text` (the column the FTS index, embeddings, and `pond_search` all use, built in `sessions.rs::search_text`) carries ONLY user- and assistant-role conversational text (plus file metadata). Tool calls, tool results, reasoning, and system/tool-role messages are deliberately excluded. This is a deliberate signal-quality choice, NOT a gap or a bug: that excluded content almost never holds anything findable beyond what the conversational text already says, and indexing it would pollute both results and the caller's context. Never "improve" search by folding tool/reasoning/system content into `search_text`. Tool-body archaeology goes through `parts.variant_data` via `pond_sql_query`. -- Subagent sessions are excluded from default `pond_search` results for the same reason (low unique signal, high pollution); they remain reachable on purpose via `pond_sql_query` (`parent_session_id`). Don't surface them in the main search. +- `search_text` (the column the FTS index, embeddings, and `pond_search` all use, built in `sessions.rs::search_text`) carries ONLY user- and assistant-role conversational text (plus file metadata). Tool calls, tool results, reasoning, and system/tool-role messages are deliberately excluded. This is a deliberate signal-quality choice, NOT a gap or a bug: that excluded content almost never holds anything findable beyond what the conversational text already says, and indexing it would pollute both results and the caller's context. Never "improve" search by folding tool/reasoning/system content into `search_text`. Tool-body archaeology goes through `parts.variant_data` via `pond_sql`. +- Subagent sessions are excluded from default `pond_search` results for the same reason (low unique signal, high pollution); they remain reachable on purpose via `pond_sql` (`parent_session_id`). Don't surface them in the main search. ## MCP surfaces are hard-enforced read-only -- Every pond MCP action is read-only, hard-enforced (durable user constraint) - any new MCP tool that reaches the store must clear the same bar; never expose a write path through MCP. `pond_sql_query` proves it in two layers: (1) a pre-parse gate requiring exactly one `Statement::Query` - this also catches `EXPLAIN ANALYZE` / `DESCRIBE`, which DataFusion's `SQLOptions` alone misses; (2) `sql_with_options` with `allow_ddl` / `allow_dml` / `allow_statements` all false (only a bare `EXPLAIN` of a SELECT flips `allow_statements`). Fresh `SessionContext` per query. +- Every pond MCP action is read-only, hard-enforced (durable user constraint) - any new MCP tool that reaches the store must clear the same bar; never expose a write path through MCP. `pond_sql` proves it in two layers: (1) a pre-parse gate requiring exactly one `Statement::Query` - this also catches `EXPLAIN ANALYZE` / `DESCRIBE`, which DataFusion's `SQLOptions` alone misses; (2) `sql_with_options` with `allow_ddl` / `allow_dml` / `allow_statements` all false (only a bare `EXPLAIN` of a SELECT flips `allow_statements`). Fresh `SessionContext` per query. ## Sync change-detection oracle (S3 perf, measured) @@ -83,6 +83,11 @@ The wizard prompts (`pond init`, source discovery, etc.) go through cliclack/dia ## Repo layout - One flat crate. `src/` holds the `adapter/` module folder alongside top-level files (`handlers.rs`, `sessions.rs`, `substrate.rs`, `transport.rs`, `wire.rs`, `embed.rs`, `config.rs`, `main.rs`, `lib.rs`). Unit tests live in `#[cfg(test)] mod tests` inside the file they test; `tests/` is for cross-module integration only. +- Root `SKILL.md` is compiled into the binary via `include_str!` (`pond skill` prints it; `pond init` installs it to `~/.claude/skills/pond/`), so it MUST stay out of Cargo.toml's `exclude` list - excluding it breaks the crates.io publish build. + +## MCP tool routing is deliberate + +- Clients (Claude Code, Codex) defer MCP tool descriptions behind tool search; the always-loaded pond-authored surfaces are the server `instructions` (transport.rs `get_info`), the tool NAMES, and the installed skill. The instructions therefore carry all routing: "analyze / review / summarize a session" binds to `pond_get(session_id)`, and `pond_sql` self-describes as the escape hatch (never as the "analytic" tool - that word routes "analyze this session" to SQL). Keep cookbook detail in the `schema://` resources and recovery guidance in error messages (sql.rs `enrich`, the timeout text) - those are the surfaces agents reliably see. Don't fatten tool descriptions back up. ## Documentation diff --git a/Cargo.toml b/Cargo.toml index e7ee9fc..421460a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,8 @@ exclude = [ "moon.yml", "AGENTS.md", "CLAUDE.md", - "SKILL.md", + # SKILL.md is NOT excluded: `include_str!` in main.rs/init.rs compiles it + # into the binary, so the published package must carry it. "flake.nix", "nix/", "**/.DS_Store", @@ -59,7 +60,7 @@ anyhow = "1.0.102" # Lock-free read of the loaded `RowKeyMap` on the hot search path; replaced # atomically when `ensure_rowmap` rebuilds. Already in the tree via lance. arc-swap = "1" -# ndjson export writer for `pond_sql_query` (already in the tree via the arrow +# ndjson export writer for `pond_sql` (already in the tree via the arrow # stack; named directly here for the `LineDelimitedWriter`). arrow-json = "58.3" # Already in the tree via the arrow stack; named directly for @@ -111,7 +112,7 @@ hf-hub = { version = "0.5", default-features = false, features = ["ureq"] } indicatif = "0.18" lance = { version = "8.0.0", features = ["protoc"] } lance-arrow = "8.0.0" -# `pond_sql_query` registers lance's JSON / contains_tokens UDFs on the SQL +# `pond_sql` registers lance's JSON / contains_tokens UDFs on the SQL # SessionContext via `lance_datafusion::udf::register_functions`. Already in # the tree transitively; named here to reach that one function. lance-datafusion = "8.0.0" @@ -136,7 +137,7 @@ memmap2 = "0.9" # drive a conditional put (`PutMode::Create`) through `ObjectStore::inner` and # classify the typed errors (AlreadyExists / Unauthenticated / NotImplemented). object_store = { version = "0.13", default-features = false } -# Parquet export writer for `pond_sql_query`. The only genuinely new compiled +# Parquet export writer for `pond_sql`. The only genuinely new compiled # crate the SQL tool adds (lance reads/writes its own format, not parquet). # `arrow` feature only; results are written uncompressed via `ArrowWriter`. parquet = { version = "58.3", default-features = false, features = ["arrow"] } diff --git a/README.md b/README.md index 593f649..fa251e1 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,11 @@ Install, run guided setup, and ingest your local sessions: ```sh brew install tenequm/tap/pond -pond init # guided setup: storage, adapters, MCP registration, optional schedule +pond init # guided setup: storage, adapters, MCP + agent skill, optional schedule pond sync # ingest, embed, update indexes - every enabled adapter ``` -`pond init` registers pond as an MCP server for detected clients (by hand: `claude mcp add -s user pond -- pond mcp`, `codex mcp add pond -- pond mcp`). Then ask your agent - real prompts from daily use: +`pond init` registers pond as an MCP server for detected clients and installs the bundled pond skill for Claude Code (by hand: `claude mcp add -s user pond -- pond mcp`, `codex mcp add pond -- pond mcp`; skill: save `pond skill` output to `~/.claude/skills/pond/SKILL.md`). Then ask your agent - real prompts from daily use: ``` check in pond how we solved this before, then apply the same fix here @@ -123,7 +123,7 @@ pond copy --from snapshot.pond --to local ### Read-only SQL -Ask structured questions with read-only SQL (the same surface as the `pond_sql_query` MCP tool): +Ask structured questions with read-only SQL (the same surface as the `pond_sql` MCP tool): ```sh pond sql "SELECT project, count(*) FROM messages GROUP BY project ORDER BY 2 DESC" @@ -193,7 +193,7 @@ The full contract is in [`docs/spec.md`](docs/spec.md). Key choices: - **Index lifecycle decoupled from writes.** Writes commit data (embeddings included, computed inline at ingest) without folding the search indexes. `pond sync` runs index maintenance by default, and `pond optimize --only index` runs it on demand; Lance merges index results with a flat scan over unindexed fragments, so reads stay correct. - **Single-arm retrieval.** Each query runs one retriever - `vector` (cosine, with a gentle recency tiebreaker) or `fts` (BM25) - chosen per query; no server-side fusion. The vector arm falls back to full-text when the store has no embeddings, and `--sort-by recency` returns newest-first. Results group to one summary per session, keyed on `session_root`. - **Language-neutral full-text.** Word-level `simple` tokenizer with English stemming (ascii-folding on); tokens the stemmer does not recognize pass through unchanged and stay exact-matchable, so pond indexes sessions in any language alike. -- **Two transports, one handler set.** HTTP+JSON (axum) and MCP (rmcp) both dispatch into the same handlers. Wire ops: `pond_search`, `pond_get`, `pond_ingest`. MCP additionally exposes the read-only `pond_sql_query` tool and the `schema://pond`, `schema://pond-sql`, and `stats://pond` resources. +- **Two transports, one handler set.** HTTP+JSON (axum) and MCP (rmcp) both dispatch into the same handlers. Wire ops: `pond_search`, `pond_get`, `pond_ingest`. MCP additionally exposes the read-only `pond_sql` tool and the `schema://pond`, `schema://pond-sql`, and `stats://pond` resources. - **Opaque-string multi-tenancy.** Each tenant is a `namespace` string the integrator supplies; pond does not authenticate, authorize, or model identity. The object store's IAM is the storage boundary. - **Encryption is operational.** Bucket SSE plus filesystem encryption; pond holds no keys and adds no application-level crypto. diff --git a/SKILL.md b/SKILL.md index 94b779e..c38f88a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,33 +1,49 @@ --- name: pond -description: Recall past AI agent sessions (Claude Code, Codex, opencode, and more) - lossless storage with semantic and full-text search, served over MCP. +description: Recall and analyze past AI agent sessions (Claude Code, Codex, opencode, and more). Find prior work and decisions, read/review/summarize a past session transcript, or run SQL analytics over session history. Use whenever the user references past sessions, prior work, "check pond", or asks what was done or decided before. --- # pond pond is your memory across every AI coding session you have run - stored -losslessly, searchable. If a task needs context you lack, pond likely has it: -recall first, then answer. Before you say "I don't know" or re-derive something -that sounds prior, search pond. +losslessly, searchable over MCP. If a task needs context you lack, pond likely +has it: recall first, then answer. Before you say "I don't know" or re-derive +something that sounds prior, search pond. ## Which tool -- Past work, by meaning ("what did we decide", "last week", "that other repo") - -> `pond_search` (`mode=vector` default; `mode=fts` for exact words/symbols). -- A known session or message -> `pond_get` (transcript, or one message in context). -- Counts, filters, trends across sessions -> `pond_sql_query` (read-only SQL over - `sessions` / `messages` / `parts`). +- Find past work by meaning ("what did we decide", "have we hit this before") + -> `pond_search` (`mode=vector` default; `mode=fts` for exact whole words). +- Read, analyze, review, or summarize a session -> `pond_get(session_id)` - + one call, full readable transcript. `pond_get(message_id)` expands one + message with its full tool bodies. +- Corpus-wide aggregation, exact strings inside tool bodies, subagent + sessions, bulk export -> `pond_sql` (read-only SQL). Read resource + `schema://pond-sql` first - do not guess columns or JSON paths. -Params and response shapes live in each tool's MCP description and the -`schema://pond`, `schema://pond-sql`, `stats://pond` resources - read them at -call time, don't guess. +## Rules that prevent wrong conclusions + +- Long sessions supersede their own early conclusions. For "what did we + decide / latest state", read the end (`pond_get(session_id, + session_from="end")`) or `pond_search` with `sort_by=recency` - relevance + rank favors the early, confident, possibly overturned phrasing. +- Search covers only user/assistant conversational text - tool output is + excluded by design. A weak search result is NOT proof of absence: verify + exact strings with `pond_sql` `contains_tokens(search_text, '...')` before + concluding something never happened. +- Tool bodies in SQL: tool_call is `{call_id, name, params}` (a Bash command + is `json_extract(variant_data, '$.params.command')`); tool_result is + `{call_id, name, is_failure, result}`. +- On a remote store, SQL over `parts` costs seconds per round-trip: scope by + `session_id` / `tool_name`, and raise `timeout_seconds` when a broad scan + is genuinely needed. ## Setup `brew install tenequm/tap/pond` (or `cargo binstall pond-db`, or `nix profile -add github:tenequm/pond#pond`), then `pond init`, then `claude mcp add -s -user pond -- pond mcp`. Keep current with `pond sync`; `pond --help` for the rest. -Claude.ai chats are not synced automatically - request a data export +add github:tenequm/pond#pond`), then `pond init` - it registers the MCP server +and installs this skill. Keep current with `pond sync`; `pond --help` for the +rest. Claude.ai chats are not synced automatically - request a data export (claude.ai Settings -> Privacy -> Export data, arrives as an emailed `.zip`), then `pond sync claude-ai-export --path `. Docs: https://pond.locker/ diff --git a/benches/serve_mem_bench.rs b/benches/serve_mem_bench.rs index 73f4f16..328c1b3 100644 --- a/benches/serve_mem_bench.rs +++ b/benches/serve_mem_bench.rs @@ -23,7 +23,7 @@ //! - vector_first : first vector query (the cold E5 model-load spike) //! - vector_steady : N vector-arm queries (default arm; needs the embedder) //! - get_steady : N `pond_get` hydration calls on prior hits -//! - sql_steady : N `pond_sql_query` calls (the analytic tool) +//! - sql_steady : N `pond_sql` calls (the analytic tool) //! - idle/drained : resting footprint; drained drops the model (cache stays) //! //! Run: @@ -85,7 +85,7 @@ const QUERIES: &[&str] = &[ "schema evolution add column", ]; -/// `pond_sql_query` workload: one metadata-only count (manifest, no data read), +/// `pond_sql` workload: one metadata-only count (manifest, no data read), /// two column scans, a token filter (FTS-accelerated), and a group-by. Mirrors /// the analytic shapes the MCP tool actually serves. const SQL_QUERIES: &[&str] = &[ @@ -1201,7 +1201,7 @@ async fn main() -> Result<()> { .map(|s| phases.push(s))?; } - // ---- Phase: sql_steady (the pond_sql_query analytic tool) ---- + // ---- Phase: sql_steady (the pond_sql analytic tool) ---- run_sql_phase("sql_steady", &store, &sampler, SQL_QUERIES) .await .map(|s| phases.push(s))?; diff --git a/benches/sync_oracle_bench.rs b/benches/sync_oracle_bench.rs index bd19490..85d994e 100644 --- a/benches/sync_oracle_bench.rs +++ b/benches/sync_oracle_bench.rs @@ -124,7 +124,7 @@ async fn fetch_tables(store: &Store) -> Result { } /// Run a SQL query through pond's own `sql::run` path - so the bench exercises -/// the exact DataFusion + Lance pushdown wiring the MCP `pond_sql_query` tool +/// the exact DataFusion + Lance pushdown wiring the MCP `pond_sql` tool /// uses. Returns the row count from the result; the inline rendering cost is /// kept tiny by passing `inline_rows = 0`. async fn run_sql(tables: &Tables, sql: &str) -> Result { diff --git a/docs/site/src/pages/get-started/connect-your-agents.mdx b/docs/site/src/pages/get-started/connect-your-agents.mdx index a4f4e4d..bf52c13 100644 --- a/docs/site/src/pages/get-started/connect-your-agents.mdx +++ b/docs/site/src/pages/get-started/connect-your-agents.mdx @@ -1,19 +1,19 @@ # Connect your agents -pond is an MCP server. `pond init` registers it for detected clients. To add it by hand: +pond is an MCP server. `pond init` registers it for detected clients and installs the bundled pond skill into Claude Code's user skills dir. To add the server by hand: ```sh claude mcp add -s user pond -- pond mcp # Claude Code codex mcp add pond -- pond mcp # Codex ``` -Other clients: use `pond mcp` (alias for `pond serve --transport stdio`) as the server command. +Other clients: use `pond mcp` (alias for `pond serve --transport stdio`) as the server command. The skill installs by hand too: save `pond skill` output to `~/.claude/skills/pond/SKILL.md`. ## Tools the agent gets -- `pond_search` - search across all sessions (vector or full-text) -- `pond_get` - fetch a session or message by id -- `pond_sql_query` - read-only SQL over the corpus +- `pond_search` - find relevant messages across all sessions (vector or full-text) +- `pond_get` - read, analyze, or summarize a session; or one message with its full tool bodies +- `pond_sql` - the escape hatch: read-only SQL over the corpus ## Agent setup prompt diff --git a/docs/site/src/pages/reference/cli.mdx b/docs/site/src/pages/reference/cli.mdx index 21fc641..cbb6716 100644 --- a/docs/site/src/pages/reference/cli.mdx +++ b/docs/site/src/pages/reference/cli.mdx @@ -5,7 +5,7 @@ pond --help # full flags for any command ``` ## Setup -- `pond init` - guided setup-and-repair: storage, adapters, MCP, schedule +- `pond init` - guided setup-and-repair: storage, adapters, MCP + agent skill, schedule - `pond adapters list|discover|enable|disable` - manage which adapters sync ingests - `pond storage check|use` - probe a destination; switch the active store - `pond creds add|list|delete` - manage URL-scoped credential sets diff --git a/docs/site/src/pages/reference/mcp-tools.mdx b/docs/site/src/pages/reference/mcp-tools.mdx index a829226..f12c7f7 100644 --- a/docs/site/src/pages/reference/mcp-tools.mdx +++ b/docs/site/src/pages/reference/mcp-tools.mdx @@ -2,9 +2,11 @@ What your agent gets when pond is registered as an MCP server ([Connect your agents](/get-started/connect-your-agents)). The wire schemas ship with the server; this page is the human-facing summary. +Two tools cover almost every request: `pond_search` finds, `pond_get` reads. `pond_sql` is the advanced escape hatch for what those two cannot express. + ## pond_search -Semantic or keyword search over all stored sessions. Returns a readable transcript of ranked hits grouped by session, best session first. +Find relevant messages across all stored sessions - semantic or keyword. Returns a readable transcript of ranked hits grouped by session, best session first. | Parameter | Meaning | |---|---| @@ -18,20 +20,20 @@ Semantic or keyword search over all stored sessions. Returns a readable transcri Every response reports how many searchable messages the filters left in scope, so an empty result distinguishes "nothing relevant exists" from "my filters excluded everything". By default subagent (child) sessions are excluded; an explicit `session_id` or source-agent filter opts back in deliberately. -Search indexes only conversational text (user and assistant turns). Tool calls, tool results, and reasoning are deliberately not indexed - reach them with `pond_sql_query`. +Search indexes only conversational text (user and assistant turns). Tool calls, tool results, and reasoning are deliberately not indexed - reach them with `pond_sql`. ## pond_get -Fetch stored content by id, as a readable transcript. Pass exactly one of: +Read stored conversations - the tool for analyzing, reviewing, or summarizing a past session, as a readable transcript. Pass exactly one of: - `session_id` - the whole session: user/assistant text plus one-line tool and file references. Page with `session_limit` (default 20 messages), `session_from: "start" | "end"` (`end` reads the most recent page first - e.g. recovering context after compaction), and `session_after_message_id` / `session_before_message_id` to page onward. - `message_id` - one message with its full parts, including tool call and result bodies, plus conversational neighbors (`message_context_before` / `message_context_after`, default 3 each side - like `grep -B/-A`). A session response lists its subagent sessions in a footer so each can be opened in turn. Not for bulk export - that is [`pond copy`](/reference/cli). -## pond_sql_query +## pond_sql -One read-only SQL query (SELECT/WITH only, DataFusion-planned) over the three tables: `sessions`, `messages`, `parts`. This is the escape hatch search deliberately doesn't cover: exact strings, identifiers, counts, joins, group-bys, tool-call archaeology through `parts.variant_data`, and subagent sessions (`parent_session_id`). +The advanced escape hatch: one read-only SQL query (SELECT/WITH only, DataFusion-planned) over the three tables: `sessions`, `messages`, `parts`. Not for finding or reading conversations - that is search and get. It covers what they deliberately don't: exact strings, identifiers, counts, joins, group-bys, tool-call archaeology through `parts.variant_data` (a tool_call body is `{call_id, name, params}` - a Bash command lives at `json_extract(variant_data, '$.params.command')`), and subagent sessions (`parent_session_id`). Two useful full-text primitives: diff --git a/docs/site/src/pages/why.mdx b/docs/site/src/pages/why.mdx index e89227d..8b38d7d 100644 --- a/docs/site/src/pages/why.mdx +++ b/docs/site/src/pages/why.mdx @@ -73,7 +73,7 @@ pond's premise: a session should outlive the tool that made it. Three rules foll ## Two audiences, one source - **Humans:** one binary, one searchable archive across every client; ask any agent about your past work. Backend switches and rollbacks are one command; the local copy stays as a backup. -- **Agents:** pond is an MCP server - `pond_search` / `pond_get` / `pond_sql_query` over the user's full cross-client history, no per-client integration. Every command is exit-code-gated for CI. +- **Agents:** pond is an MCP server - `pond_search` / `pond_get` / `pond_sql` over the user's full cross-client history, no per-client integration. Every command is exit-code-gated for CI. ## Next diff --git a/docs/spec.md b/docs/spec.md index bc57cb5..11b7a5a 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -105,7 +105,7 @@ These are stable positions. pond will not: - **Authenticate, authorize, or model identity or tenancy.** An integrator decides who may reach which namespace before any pond call; `namespace` on the wire is an opaque routing string pond does not interpret. On hosted deployments the object store's IAM is the storage boundary and the integrator's gateway is the application boundary. - **Encrypt at the application layer.** Encryption is bucket server-side encryption plus filesystem encryption; pond holds no keys and adds no cryptography of its own. pond is not a zero-knowledge store - an operator with bucket and key access can read everything. - **Act as a runtime.** pond does not execute tools, run an agent loop, compact context, render output, or emit telemetry. It stores what those systems produce. -- **Become a SQL database, a UI, or a sidecar daemon.** The primary query surface is the search and filter API of Section 8, which compiles to Lance scalar predicates and search calls. Alongside it sits one read-only SQL escape hatch - the `pond_sql_query` tool (Section 7.7) and the `pond sql` verb (Section 7.8): a single SELECT planned by embedded DataFusion over the same Lance datasets, never a write path and never a second storage engine. The only engine is embedded Lance; there is no UI and no daemon beyond `pond serve`. +- **Become a SQL database, a UI, or a sidecar daemon.** The primary query surface is the search and filter API of Section 8, which compiles to Lance scalar predicates and search calls. Alongside it sits one read-only SQL escape hatch - the `pond_sql` tool (Section 7.7) and the `pond sql` verb (Section 7.8): a single SELECT planned by embedded DataFusion over the same Lance datasets, never a write path and never a second storage engine. The only engine is embedded Lance; there is no UI and no daemon beyond `pond serve`. ### 2.4 Platform @@ -655,7 +655,7 @@ Retryability is conveyed by the code; there is no separate field. `conflict` is 2. **`pond_get`** (`POST /v1/get`) - fetch a whole session, or one message with surrounding context. Session mode takes a `response_mode`: `conversational` (the default - human/model text, with a compact `parts_summary` per message), `complete` (all messages including system/tool carriers, with `parts_summary`), or `verbatim` (all messages with full Parts inline). Session mode also takes `session_from`: `start` (the default - oldest messages first) or `end` (the most recent messages, still in chronological order - e.g. recovering recent context after compaction). Message mode returns the target's full Parts (paginated) plus `context_depth` sibling messages each side; the siblings follow `response_mode` - conversational by default, so system/tool carriers do not crowd the conversation out of the window - while the target itself returns regardless of role. Pages are bounded by a size budget and never cut mid-message; when `messages_remaining` (session) or `target_parts_remaining` (message) is non-zero, the caller pages on by passing the last returned id as `after_id`. Not for bulk export - that is the restore/export path. 3. **`pond_ingest`** (`POST /v1/ingest`) - accept a batch of canonical events. Always batched, bounded by an event count and a body-size cap. Events are grouped by session and applied per session; partial success across sessions is normal and reported per row. -Three resources - `schema://pond`, `schema://pond-sql`, and `stats://pond` - expose the search-field documentation, the SQL surface's table schemas, and dataset statistics. The read-only SQL query surface is not an HTTP operation: it is exposed as the `pond_sql_query` MCP tool (7.7) and the `pond sql` verb (7.8). +Three resources - `schema://pond`, `schema://pond-sql`, and `stats://pond` - expose the search-field documentation, the SQL surface's table schemas, and dataset statistics. The read-only SQL query surface is not an HTTP operation: it is exposed as the `pond_sql` MCP tool (7.7) and the `pond sql` verb (7.8). ### 7.6 Ingest events @@ -663,19 +663,21 @@ A `pond_ingest` event is one canonical object - a Session, a Message, or a Part ### 7.7 MCP surface -The MCP transport exposes the read operations - `pond_search`, `pond_get`, and `pond_sql_query` - as tools, plus the resources. Ingest stays HTTP-and-CLI only. Why: MCP's role is read access for an agent; ingest is an operator action. +The MCP transport exposes the read operations - `pond_search`, `pond_get`, and `pond_sql` - as tools, plus the resources. Ingest stays HTTP-and-CLI only. Why: MCP's role is read access for an agent; ingest is an operator action. + +Tool metadata is a routing surface, designed for clients that defer tool descriptions behind tool search (Claude Code, Codex): the server `instructions` carry all routing (search finds, get reads - including analyze/review/summarize a session - and `pond_sql` is the self-demoting escape hatch), tool names alone must route correctly, descriptions lead with the verbs they claim, cookbook detail lives in the `schema://` resources, and error messages teach the correct next query at the point of failure. ### 7.8 CLI verbs The same handlers back a set of command-line verbs. The storage destination and config file are `global = true` selectors (`--storage-path` / `POND_STORAGE_PATH`, `--config-file` / `POND_CONFIG_FILE`), resolved at the root or after any subcommand alike; `pond init` is the one exception - it ignores an env-sourced `--storage-path`, since writing ephemeral env state into config would be a silent surprise. -- `pond init` - idempotent setup-and-repair wizard: storage destination (with an end-to-end probe for remote URLs), adapter selection, MCP registration, and an opt-in sync schedule, written to `config.toml` in a single pass at the end. Every section is flag-answerable for non-interactive use: the destination via the global `--storage-path`, the rest via `--adapters`, `--every`, `--skip-mcp`, `--yes`, and `--force` (start from built-in defaults, ignoring existing config). `--yes` alone never schedules. The embedding model is deliberately not an init concern: overriding it is an advanced, re-embed-forcing act that lives only under `[embeddings]` in config. After the config write, an interactive init offers to run the first sync in the foreground (full progress UI) and registers the schedule only once that sync completes - a fresh systemd timer fires immediately on registration, and two concurrent syncs only slow each other down. An opted-in schedule survives Ctrl-C during that first sync: the interrupt path registers it on the way out, so "the next sync resumes where this one stopped" stays true unattended. +- `pond init` - idempotent setup-and-repair wizard: storage destination (with an end-to-end probe for remote URLs), adapter selection, MCP registration (one consent also installs the bundled agent skill - the same bytes `pond skill` prints - into Claude Code's user skills dir, kept in sync on re-runs; a hand-edited copy is never overwritten without an explicit confirm), and an opt-in sync schedule, written to `config.toml` in a single pass at the end. Every section is flag-answerable for non-interactive use: the destination via the global `--storage-path`, the rest via `--adapters`, `--every`, `--skip-mcp`, `--yes`, and `--force` (start from built-in defaults, ignoring existing config). `--yes` alone never schedules. The embedding model is deliberately not an init concern: overriding it is an advanced, re-embed-forcing act that lives only under `[embeddings]` in config. After the config write, an interactive init offers to run the first sync in the foreground (full progress UI) and registers the schedule only once that sync completes - a fresh systemd timer fires immediately on registration, and two concurrent syncs only slow each other down. An opted-in schedule survives Ctrl-C during that first sync: the interrupt path registers it on the way out, so "the next sync resumes where this one stopped" stays true unattended. - `pond sync` - the additive ingest verb: import fresh sessions - each message embedded inline in its ingest commit - then fold the search indexes and run the `[maintenance]` compaction pass (amortizing the version-cleanup walk over several runs since it is round-trip-bound on object stores) so the lake is queryable on exit. A positional `` syncs just that one enabled adapter; `--path ` (requires ``) is a one-off path override that bypasses `[adapters.]` and never writes config. Sync ingests only already-enabled `[adapters.*]`; it never discovers, enables, or writes adapter state - that is the explicit job of `pond adapters` and `pond init`. With no enabled adapters it does nothing but name the fix. Per host and store, sync is single-flight: a local flock in the state dir (never on the Lance store - cross-host writers stay pure OCC) makes a second sync wait, naming the holder; `--no-wait` skips instead (exit 0), which is what the scheduled run passes so ticks never queue. `--dry-run` prints the freshness gate's per-adapter verdict (sessions, fresh, pending) and writes nothing to the store (it may build the local freshness cache - the same one-time scan a real sync starts with, and what makes the preview accurate). `--format json` emits one machine-readable summary document on stdout for every outcome - ok, skipped, or error (progress stays on stderr). The embedding-model preload is best-effort: a caught-up sync embeds nothing, so an offline host with no cached weights still completes as a no-op - only a run that actually needs to embed fails. Every phase that can run long has a live face - the freshness-map build, the embedding-model download, the per-adapter import bar (with an ETA from the bar's recent-rate estimator, never a whole-run average an early fresh-skip burst would poison), and the inline-embed counter inside each commit - and off-TTY (cron, agents) the bar line is emitted as a plain heartbeat every ~30s, including through the embed/commit phase. A run that ingested rows re-extends the local freshness cache before exiting, so `pond status` pending counts are as-of sync end. Each run finishes by writing a per-host last-sync record (outcome, deltas, duration) to the state dir for `pond status`. - `pond optimize` - the maintenance verb: embed the un-embedded backlog, then fold the text + semantic indexes (the index stage also runs the `[maintenance]` compaction / version-cleanup pass). `--only ` runs exactly one stage; `--skip ` omits one. Stages are `embed` and `index`. `--force-embed` re-embeds rows whose `embedding_model` differs from the current model via conditional merge. `pond sync` embeds inline at ingest, folds indexes, and compacts on every run (cleaning versions only periodically); `optimize` is the on-demand catch-up - backlog embed (when embedding was off at ingest) and model-swap re-embed - and cleans versions every run. - `pond adapters list|discover|enable|disable` - manage which adapters `pond sync` ingests (the `[adapters.*]` entries) and nothing else. `list` shows configured adapters (enabled state, path) plus detected-but-unconfigured adapters; `discover` probes the machine and enables the interactive picks; `enable`/`disable` flip one by name, `enable` discovering its default path when it has no entry yet. Enabling an adapter is therefore always an explicit action - `pond sync` reads these entries but never writes them. - `pond search [--explain]` - search from the command line. `--mode vector|fts` picks the retrieval arm and `--sort-by relevance|recency` the order. `--explain` returns Lance's `analyze_plan` output instead of results. - `pond get` - fetch a session, or one message with context, from the command line. -- `pond sql` - one read-only SQL query over the corpus: a DataFusion-planned SELECT/WITH over the three session datasets, with the same two-layer read-only enforcement as the `pond_sql_query` tool (Section 7.7). Results render inline (row-capped) or export to parquet/ndjson. +- `pond sql` - one read-only SQL query over the corpus: a DataFusion-planned SELECT/WITH over the three session datasets, with the same two-layer read-only enforcement as the `pond_sql` tool (Section 7.7). Results render inline (row-capped) or export to parquet/ndjson. - `pond status` - row counts, dataset statistics, embedding coverage, index health, and the sync-schedule state, plus this host's relationship to the store: what each enabled adapter sees on local disk, how many sessions are pending sync (classified against the locally cached freshness map - never a remote scan; unknown until the host's first sync), the last sync's outcome (including a surfaced FAILURE from a scheduled run), and the estimated next scheduled run. `--hosts` adds the fleet view of a shared store - sessions and latest activity per ingest-host stamp. It prints the cheap storage summary first, then completes the longer checks. - `pond serve --transport http|stdio` - run HTTP by default, or MCP over stdio. - `pond mcp` - alias for `pond serve --transport stdio`. diff --git a/src/handlers.rs b/src/handlers.rs index 085855c..c493fe1 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -1787,7 +1787,7 @@ mod search_handler { /// spec.md#search: subagents are excluded from `pond_search` results - /// always, except when the caller scopes to one `session_id` (which may /// itself be a subagent session, so the exclusion would fight the filter). - /// Subagents are reachable only via `pond_sql_query` (`parent_session_id`). + /// Subagents are reachable only via `pond_sql` (`parent_session_id`). pub fn default_excludes_subagents(filters: &SearchFilters) -> bool { filters.session_id.is_none() } diff --git a/src/init.rs b/src/init.rs index f1ee56f..c9d3bb3 100644 --- a/src/init.rs +++ b/src/init.rs @@ -1,7 +1,8 @@ //! `pond init`: the idempotent setup-and-repair wizard. //! -//! One pass over four sections - storage, adapters, MCP registration, sync -//! schedule - then a single `config.toml` write at the end. Every section is +//! One pass over four sections - storage, adapters, MCP registration (which +//! also installs the bundled agent skill), sync schedule - then a single +//! `config.toml` write at the end. Every section is //! answerable by a flag for non-interactive use; re-running against an //! existing setup proposes only what would change. Bin-only module: the //! wizard is a CLI surface, not library behavior. @@ -29,7 +30,7 @@ pub(crate) struct InitArgs { /// Register `pond sync` on a schedule. Opt-in: `--yes` alone never schedules. #[arg(long = "every", value_enum, value_name = "EVERY")] schedule: Option, - /// Skip MCP registration. + /// Skip MCP registration and the skill install. #[arg(long)] skip_mcp: bool, /// Accept defaults for everything not covered by a flag (non-interactive). @@ -833,10 +834,10 @@ fn pick_adapters(args: &InitArgs, rows: &[AdapterRow], prompts: bool) -> Result< wiz(picker.interact()) } -/// Detect agent CLIs and offer MCP registration. claude has an idempotent -/// CLI surface (`mcp get` / `mcp add`), so pond drives it directly; codex -/// gets the exact command to run instead - pond never edits another tool's -/// config files behind the user's back. +/// Detect agent CLIs and offer MCP registration plus the bundled skill. +/// claude has an idempotent CLI surface (`mcp get` / `mcp add`), so pond +/// drives it directly; codex gets the exact command to run instead - pond +/// never edits another tool's config files behind the user's back. fn mcp_section(prompts: bool, auto: bool) -> Result<()> { let claude = crate::find_on_path("claude"); let codex = crate::find_on_path("codex"); @@ -854,15 +855,18 @@ fn mcp_section(prompts: bool, auto: bool) -> Result<()> { .status() .map(|status| status.success()) .unwrap_or(false); + // One consent covers registration and the skill; a fresh install + // that says yes here is not asked again for the skill write. + let mut skill_consented = false; if registered { cliclack::log::success("mcp: pond is already registered in Claude Code")?; } else { let add = if prompts { - wiz( - cliclack::confirm("Register pond as an MCP server in Claude Code?") - .initial_value(true) - .interact(), - )? + wiz(cliclack::confirm( + "Register pond in Claude Code (MCP server + the pond skill)?", + ) + .initial_value(true) + .interact())? } else { auto }; @@ -880,12 +884,16 @@ fn mcp_section(prompts: bool, auto: bool) -> Result<()> { String::from_utf8_lossy(&output.stderr).trim(), ))?; } + skill_consented = true; } else { cliclack::log::info( "mcp: skipped - register later with `claude mcp add -s user pond -- pond mcp`", )?; } } + if registered || skill_consented { + skill_section(prompts, auto, skill_consented)?; + } } if codex.is_some() { cliclack::note( @@ -896,6 +904,70 @@ fn mcp_section(prompts: bool, auto: bool) -> Result<()> { Ok(()) } +/// The skill pond installs for Claude Code, embedded so the installed copy +/// always matches the running binary (the same bytes `pond skill` prints). +const SKILL_MD: &str = include_str!("../SKILL.md"); + +const SKILL_DISPLAY_PATH: &str = "~/.claude/skills/pond/SKILL.md"; + +/// Sync the bundled skill into Claude Code's user skills dir. +fn skill_section(prompts: bool, auto: bool, consented: bool) -> Result<()> { + let Some(home) = std::env::var_os("HOME").map(PathBuf::from) else { + cliclack::log::info(format!( + "skill: HOME not set - install later by saving `pond skill` output to {SKILL_DISPLAY_PATH}", + ))?; + return Ok(()); + }; + let path = home + .join(".claude") + .join("skills") + .join("pond") + .join("SKILL.md"); + skill_sync(&path, prompts, auto, consented) +} + +/// Three states: current (no-op), absent (install), differs (an older pond's +/// copy or a user edit - overwriting is asked about explicitly in prompt +/// mode, even when the combined registration consent already said yes). +fn skill_sync(path: &Path, prompts: bool, auto: bool, consented: bool) -> Result<()> { + let existing = std::fs::read_to_string(path).ok(); + let (question, done) = match existing.as_deref() { + Some(current) if current == SKILL_MD => { + cliclack::log::success(format!("skill: up to date ({SKILL_DISPLAY_PATH})"))?; + return Ok(()); + } + Some(_) => ( + format!("Update the pond skill? ({SKILL_DISPLAY_PATH} differs from this pond version)"), + "skill: updated", + ), + None => ( + format!("Install the pond skill for Claude Code? ({SKILL_DISPLAY_PATH})"), + "skill: installed", + ), + }; + let write = if prompts && (existing.is_some() || !consented) { + wiz(cliclack::confirm(question).initial_value(true).interact())? + } else if prompts { + true + } else { + auto || consented + }; + if !write { + cliclack::log::info(format!( + "skill: skipped - install later by saving `pond skill` output to {SKILL_DISPLAY_PATH}", + ))?; + return Ok(()); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + std::fs::write(path, SKILL_MD) + .with_context(|| format!("failed to write {}", path.display()))?; + cliclack::log::success(format!("{done} ({SKILL_DISPLAY_PATH})"))?; + Ok(()) +} + struct LegacyStorage { access_key_id: Option, secret_access_key: Option, @@ -1076,6 +1148,38 @@ mod tests { use super::*; + #[test] + fn skill_sync_installs_updates_and_respects_declines() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("skills").join("pond").join("SKILL.md"); + + // Absent + non-interactive without consent: nothing written. + skill_sync(&path, false, false, false).unwrap(); + assert!(!path.exists()); + + // Absent + --yes: installed with the bundled bytes. + skill_sync(&path, false, true, false).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SKILL_MD); + + // Current: idempotent no-op. + skill_sync(&path, false, true, false).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SKILL_MD); + + // Differs (user edit) + non-interactive without consent: preserved. + std::fs::write(&path, "user edit").unwrap(); + skill_sync(&path, false, false, false).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "user edit"); + + // Differs + --yes: refreshed to the bundled version. + skill_sync(&path, false, true, false).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SKILL_MD); + + // Absent + combined-registration consent: installed without --yes. + std::fs::remove_file(&path).unwrap(); + skill_sync(&path, false, false, true).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), SKILL_MD); + } + #[test] fn legacy_rewrite_moves_keys_and_prefills_the_url() { let mut doc: DocumentMut = r#" diff --git a/src/main.rs b/src/main.rs index a1349ab..c0ea1a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -98,7 +98,7 @@ impl From for wire::SortBy { } /// CLI surface for `pond sql --format`. Maps to `sql::Mode` / `sql::Format`. -/// Mirrors the MCP `pond_sql_query` `format` arg (text|ndjson|parquet). +/// Mirrors the MCP `pond_sql` `format` arg (text|ndjson|parquet). #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] enum CliSqlFormat { Text, @@ -330,10 +330,10 @@ struct StoreArgs { enum Command { /// Set up pond (idempotent: safe to re-run). /// - /// Walks through storage, adapters, MCP registration, and an - /// optional sync schedule, then writes config.toml in one pass at the - /// end. Re-running repairs or updates an existing setup; flags answer - /// sections non-interactively. + /// Walks through storage, adapters, MCP registration (with the bundled + /// agent skill), and an optional sync schedule, then writes config.toml + /// in one pass at the end. Re-running repairs or updates an existing + /// setup; flags answer sections non-interactively. #[command(after_long_help = "Examples: pond init interactive setup (or repair) pond init --yes accept defaults, no prompts @@ -555,7 +555,7 @@ enum Command { /// /// DataFusion / PostgreSQL-compatible SELECT/WITH over the sessions, /// messages, and parts tables; writes and side-effecting statements are - /// rejected. Same surface as the `pond_sql_query` MCP tool - the MCP + /// rejected. Same surface as the `pond_sql` MCP tool - the MCP /// resource `schema://pond-sql` documents columns, indexed predicates, /// and pagination patterns. #[command(after_long_help = "Examples: @@ -617,7 +617,7 @@ enum Command { /// Serve the MCP tools over stdio (for agent clients). /// /// Equivalent to `pond serve --transport stdio`. Register once per - /// client; the tools are pond_search, pond_get, and pond_sql_query, with + /// client; the tools are pond_search, pond_get, and pond_sql, with /// resources schema://pond, schema://pond-sql, and stats://pond. #[command(after_long_help = "Examples: claude mcp add -s user pond -- pond mcp register in Claude Code @@ -5496,14 +5496,14 @@ fn render_get_envelope( } /// Rewrite the SQL module's canonical (MCP-vocabulary) error text for the CLI. -/// `pond::sql` is shared with the `pond_sql_query` tool and names the MCP +/// `pond::sql` is shared with the `pond_sql` tool and names the MCP /// tools/resources; a terminal user runs `pond sql`/`pond search`/`pond get` /// and has no `schema://` resource, so translate those tokens at the boundary. fn sql_error_for_cli(message: &str) -> String { message .replace("resource schema://pond-sql", "`pond sql --help`") .replace("schema://pond-sql", "`pond sql --help`") - .replace("pond_sql_query", "`pond sql`") + .replace("pond_sql", "`pond sql`") .replace("pond_search", "`pond search`") .replace("pond_get", "`pond get`") } diff --git a/src/render.rs b/src/render.rs index 233256a..600c3fe 100644 --- a/src/render.rs +++ b/src/render.rs @@ -11,7 +11,7 @@ use crate::wire::{ /// Which surface a transcript renders for. The format is identical; only the /// follow-up vocabulary differs - the MCP tools are `pond_get` / -/// `pond_sql_query` with `key=value` args, the CLI verbs are +/// `pond_sql` with `key=value` args, the CLI verbs are /// `pond get --message-id ` / `pond sql`. Without this a human at the /// terminal is told to run tool syntax their shell rejects. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -89,7 +89,7 @@ pub fn render_search_transcript( let subagent_note = match (default_excludes_subagents(&request.filters), surface) { (false, _) => "", (true, Surface::Mcp) => { - " Subagent sessions excluded; reach them via pond_sql_query (parent_session_id)." + " Subagent sessions excluded; reach them via pond_sql (parent_session_id)." } (true, Surface::Cli) => { " Subagent sessions excluded; reach them via `pond sql` (parent_session_id)." @@ -132,7 +132,7 @@ pub fn render_search_transcript( } let fts_hint = match surface { Surface::Mcp => { - " For exact strings or identifiers, try pond_sql_query: SELECT \ + " For exact strings or identifiers, try pond_sql: SELECT \ message_id, session_id, search_text FROM messages WHERE \ contains_tokens(search_text, '...')." } @@ -339,7 +339,10 @@ pub fn render_get_transcript( "session {} | {} | {}", session.id, session.source_agent, session.project, ); - // Bottom marker: later messages follow this page (page down). + // Bottom marker: later messages follow this page (page down). The + // supersession note guards the field-tested failure where an agent + // reads an early page of a long session and reports a + // since-revised conclusion as current. if *after_remaining > 0 && let Some(last) = messages.last() { @@ -347,9 +350,14 @@ pub fn render_get_transcript( Surface::Mcp => format!("pass session_after_message_id={}", last.id), Surface::Cli => format!("pass --session-after-message-id {}", last.id), }; + let latest = match surface { + Surface::Mcp => "session_from=\"end\"", + Surface::Cli => "--session-from end", + }; let _ = writeln!( out, - "... {after_remaining} later messages; {page_down} to page down", + "... {after_remaining} later messages; {page_down} to page down \ + (conclusions may have been revised - {latest} reads the latest turns)", ); } } diff --git a/src/sessions.rs b/src/sessions.rs index 3f600e0..ee94d19 100644 --- a/src/sessions.rs +++ b/src/sessions.rs @@ -1794,7 +1794,7 @@ impl Store { } /// A point-in-time `Arc` for `table`, for registering as a - /// DataFusion `LanceTableProvider` in `pond_sql_query`. Goes through the + /// DataFusion `LanceTableProvider` in `pond_sql`. Goes through the /// handle's freshness gate, so each query sees a current snapshot. pub async fn dataset(&self, table: Table) -> Result> { Ok(Arc::new(self.handle.dataset(table).await?)) @@ -2161,12 +2161,12 @@ impl Store { Ok(keys) } - /// Write a `pond_sql_query` export artifact. + /// Write a `pond_sql` export artifact. pub async fn export_write(&self, name: &str, bytes: &[u8]) -> Result<()> { self.handle.export_write(name, bytes).await } - /// Read a `pond_sql_query` export artifact back. + /// Read a `pond_sql` export artifact back. pub async fn export_read(&self, name: &str) -> Result> { self.handle.export_read(name).await } diff --git a/src/snapshots/pond__tests__help_init.snap b/src/snapshots/pond__tests__help_init.snap index 8ad9709..9048a94 100644 --- a/src/snapshots/pond__tests__help_init.snap +++ b/src/snapshots/pond__tests__help_init.snap @@ -4,7 +4,7 @@ expression: sub.render_long_help().to_string() --- Set up pond (idempotent: safe to re-run). -Walks through storage, adapters, MCP registration, and an optional sync schedule, then writes config.toml in one pass at the end. Re-running repairs or updates an existing setup; flags answer sections non-interactively. +Walks through storage, adapters, MCP registration (with the bundled agent skill), and an optional sync schedule, then writes config.toml in one pass at the end. Re-running repairs or updates an existing setup; flags answer sections non-interactively. Usage: pond init [OPTIONS] @@ -18,7 +18,7 @@ Options: [possible values: 5m, 15m, 1h, 6h, 1d] --skip-mcp - Skip MCP registration + Skip MCP registration and the skill install -y, --yes Accept defaults for everything not covered by a flag (non-interactive) diff --git a/src/snapshots/pond__tests__help_mcp.snap b/src/snapshots/pond__tests__help_mcp.snap index 3aea3b5..a9da962 100644 --- a/src/snapshots/pond__tests__help_mcp.snap +++ b/src/snapshots/pond__tests__help_mcp.snap @@ -4,7 +4,7 @@ expression: sub.render_long_help().to_string() --- Serve the MCP tools over stdio (for agent clients). -Equivalent to `pond serve --transport stdio`. Register once per client; the tools are pond_search, pond_get, and pond_sql_query, with resources schema://pond, schema://pond-sql, and stats://pond. +Equivalent to `pond serve --transport stdio`. Register once per client; the tools are pond_search, pond_get, and pond_sql, with resources schema://pond, schema://pond-sql, and stats://pond. Usage: pond mcp [OPTIONS] diff --git a/src/snapshots/pond__tests__help_sql.snap b/src/snapshots/pond__tests__help_sql.snap index a8c3889..280f8db 100644 --- a/src/snapshots/pond__tests__help_sql.snap +++ b/src/snapshots/pond__tests__help_sql.snap @@ -4,7 +4,7 @@ expression: sub.render_long_help().to_string() --- Run one read-only SQL query over the corpus. -DataFusion / PostgreSQL-compatible SELECT/WITH over the sessions, messages, and parts tables; writes and side-effecting statements are rejected. Same surface as the `pond_sql_query` MCP tool - the MCP resource `schema://pond-sql` documents columns, indexed predicates, and pagination patterns. +DataFusion / PostgreSQL-compatible SELECT/WITH over the sessions, messages, and parts tables; writes and side-effecting statements are rejected. Same surface as the `pond_sql` MCP tool - the MCP resource `schema://pond-sql` documents columns, indexed predicates, and pagination patterns. Usage: pond sql [OPTIONS] diff --git a/src/sql.rs b/src/sql.rs index 705d1bc..f319822 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -1,4 +1,4 @@ -//! `pond_sql_query`: read-only DataFusion SQL over the three Lance tables +//! `pond_sql`: read-only DataFusion SQL over the three Lance tables //! (`sessions` / `messages` / `parts`), registered as `LanceTableProvider`s //! (behind plan-time views that rename `id` to `message_id` / `session_id`) //! on a fresh per-call `SessionContext`. Read-only is enforced in two layers - a @@ -98,7 +98,7 @@ impl Format { } } -/// How `pond_sql_query` returns results. +/// How `pond_sql` returns results. #[derive(Debug, Clone, Copy)] pub enum Mode { /// Render a row-capped table into the tool result. @@ -174,7 +174,7 @@ pub async fn run( } if projection_mentions_vector(parsed.projection_query()) { return Err(SqlError::Query( - "the `vector` column is not selectable from pond_sql_query (it is a \ + "the `vector` column is not selectable from pond_sql (it is a \ FixedSizeList embedding, ~600 bytes per row and not useful in a result). \ For semantic search use pond_search. Filtering on it is allowed in WHERE \ (e.g. `vector IS NOT NULL`)." @@ -234,9 +234,12 @@ pub async fn run( .map_err(|_| { SqlError::Query(format!( "query exceeded the {}s limit; add a narrower WHERE or a LIMIT, or raise \ - the per-query timeout (`timeout_seconds` on pond_sql_query, `--timeout` \ + the per-query timeout (`timeout_seconds` on pond_sql, `--timeout` \ on pond sql; max {MAX_QUERY_TIMEOUT_SECS}s) if it legitimately needs \ - longer. For tool analytics use the narrow native columns (tool_name, \ + longer. On a remote object store, queries over parts cost seconds per \ + round-trip - scope by session_id / tool_name, and to reconstruct one \ + session use pond_get(session_id), not SQL. For tool analytics use the \ + narrow native columns (tool_name, \ call_id, is_failure) instead of json_get_* over variant_data. If you were \ substring-scanning variant_data (json_extract + LIKE), there is no \ substring index on tool bodies yet: filter parts by type and tool_name \ @@ -332,7 +335,7 @@ fn parse_and_gate(sql: &str) -> Result { .map_err(|error| SqlError::Query(format!("SQL parse error: {error}")))?; if statements.len() != 1 { return Err(SqlError::Query( - "pond_sql_query runs exactly one statement; submit a single SELECT".to_owned(), + "pond_sql runs exactly one statement; submit a single SELECT".to_owned(), )); } let Some(front) = statements.front() else { @@ -361,7 +364,7 @@ fn parse_and_gate(sql: &str) -> Result { } fn read_only_rejection() -> SqlError { - // Surface-neutral wording: this message reaches both the pond_sql_query + // Surface-neutral wording: this message reaches both the pond_sql // MCP tool and the `pond sql` CLI, so it names neither. SqlError::Query( "pond's SQL surface is read-only: only a single SELECT/WITH (or EXPLAIN of one) is \ @@ -1041,9 +1044,11 @@ fn enrich(message: &str) -> String { embedding_model, options) | sessions(session_id, parent_session_id, \ parent_message_id, source_agent, created_at, project, options) | \ parts(session_id, message_id, id, ordinal, type, provenance, tool_name, \ - call_id, is_failure, variant_data, options). Part bodies (tool params/results, \ - text) live in parts.variant_data - \ - read them with json_extract(variant_data, '$.field'). For text search use \ + call_id, is_failure, variant_data, options). Part bodies live in \ + parts.variant_data (JSONB) and nest by part type: tool_call is {call_id, \ + name, params} - a Bash command is json_extract(variant_data, \ + '$.params.command') - tool_result is {call_id, name, is_failure, result}, \ + text/reasoning carry {text}. For text search use \ contains_tokens(search_text, '...') in WHERE, or the fts('messages', ...) \ table function in FROM for ranked results; to read a transcript use pond_get. \ Full doc: resource schema://pond-sql.", @@ -1134,22 +1139,43 @@ fn is_displayable(data_type: &DataType) -> bool { ) } -/// One physical line per row: embedded newlines in cell values (markdown, -/// multi-line commands) otherwise explode a row across many table lines that -/// hard-wrap unreadably in narrow clients. The literal two-char `\n` matches -/// the JSON escaping agents already read, and keeps row boundaries -/// unambiguous. Inline table mode only - json and export modes keep raw data. -fn collapse_newlines(batches: &[RecordBatch]) -> Result, ArrowError> { +/// Per-cell character cap for the inline rendered table. Without it a single +/// fat cell (tool bodies reach 56KB) defeats the row-halving byte budget in +/// [`render_inline`] - which cannot drop below one row - and the table +/// renderer amplifies it ~5x by padding every row to the widest cell. +/// Measured: one 42KB cell rendered a 214KB response. Export modes +/// (parquet/ndjson) are never clipped. +const CELL_CLIP_CHARS: usize = 1_000; + +/// Bound each cell for the inline table: clip long values to +/// [`CELL_CLIP_CHARS`] with a marker naming the unclipped path, and collapse +/// embedded newlines (markdown, multi-line commands) that otherwise explode a +/// row across many table lines that hard-wrap unreadably in narrow clients. +/// The literal two-char `\n` matches the JSON escaping agents already read, +/// and keeps row boundaries unambiguous. Inline table mode only - json and +/// export modes keep raw data. +fn bound_cells(batches: &[RecordBatch]) -> Result, ArrowError> { fn escape(array: &GenericStringArray) -> ArrayRef { let escaped: GenericStringArray = array.iter().map(|value| value.map(escape_cell)).collect(); Arc::new(escaped) } fn escape_cell(text: &str) -> std::borrow::Cow<'_, str> { - if text.contains(['\n', '\r']) { - std::borrow::Cow::Owned(text.replace("\r\n", "\\n").replace(['\n', '\r'], "\\n")) + let clipped = match text.char_indices().nth(CELL_CLIP_CHARS) { + Some((cut, _)) => { + let omitted = text[cut..].chars().count(); + std::borrow::Cow::Owned(format!( + "{} [+{omitted} chars - full cell via format=ndjson, or project a \ + narrower json_extract path]", + &text[..cut] + )) + } + None => std::borrow::Cow::Borrowed(text), + }; + if clipped.contains(['\n', '\r']) { + std::borrow::Cow::Owned(clipped.replace("\r\n", "\\n").replace(['\n', '\r'], "\\n")) } else { - std::borrow::Cow::Borrowed(text) + clipped } } batches @@ -1201,7 +1227,7 @@ fn render_inline( )); } let render = |shown: usize| -> Result { - let limited = collapse_newlines(&limit_batches(display, shown))?; + let limited = bound_cells(&limit_batches(display, shown))?; Ok(pretty_format_batches(&limited)?.to_string()) }; let mut shown = total.min(max_rows); @@ -1514,6 +1540,57 @@ mod tests { } } + #[test] + fn render_inline_clips_fat_cells() { + let schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Utf8, true)])); + let fat = "x".repeat(42_000); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from(vec![Some(fat.as_str())]))], + ) + .expect("single-column batch"); + let out = render_inline(&[batch], 10, Duration::from_millis(1)).expect("render succeeds"); + assert!( + out.len() < INLINE_BUDGET_BYTES, + "one fat cell stays within the inline budget: {} bytes", + out.len() + ); + assert!( + out.contains("[+41000 chars - full cell via format=ndjson"), + "clip marker names the omitted count and the full path: {out}" + ); + } + + #[test] + fn cell_clip_respects_multibyte_boundaries_and_collapses_newlines() { + let schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Utf8, true)])); + // Multibyte chars around the cut plus embedded newlines in the kept + // prefix: the clip must cut on a char boundary and still collapse. + let fat = format!("é\ný{}", "λ".repeat(2_000)); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from(vec![Some(fat.as_str())]))], + ) + .expect("single-column batch"); + let out = render_inline(&[batch], 10, Duration::from_millis(1)).expect("render succeeds"); + assert!( + out.contains("é\\ný") && out.contains("[+1003 chars"), + "char-boundary clip with newline collapse: {out}" + ); + // Short cells are untouched. + let exactly_cap = "a".repeat(CELL_CLIP_CHARS); + let schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Utf8, true)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from(vec![Some( + exactly_cap.as_str(), + )]))], + ) + .expect("single-column batch"); + let out = render_inline(&[batch], 10, Duration::from_millis(1)).expect("render succeeds"); + assert!(!out.contains("chars -"), "cap-sized cell not clipped"); + } + #[test] fn render_inline_collapses_newlines_in_cells() { let schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Utf8, true)])); diff --git a/src/substrate.rs b/src/substrate.rs index f140886..0044566 100644 --- a/src/substrate.rs +++ b/src/substrate.rs @@ -1934,7 +1934,7 @@ impl Handle { &self.storage_options } - /// Object-store URI for a `pond_sql_query` export artifact: + /// Object-store URI for a `pond_sql` export artifact: /// `/exports/`. A sibling of the `*.lance` table dirs; /// the Directory namespace tracks tables in its `__manifest` table rather /// than by listing prefixes, so this prefix is never seen as a table @@ -1960,7 +1960,7 @@ impl Handle { } } - /// Write a `pond_sql_query` export artifact, reusing the handle's + /// Write a `pond_sql` export artifact, reusing the handle's /// storage_options so S3 installs inherit the same credentials. pub(crate) async fn export_write(&self, name: &str, bytes: &[u8]) -> Result<()> { let uri = self.export_uri(name); @@ -1976,7 +1976,7 @@ impl Handle { Ok(()) } - /// Read a `pond_sql_query` export artifact back (for the + /// Read a `pond_sql` export artifact back (for the /// `pond-sql-export://` MCP resource). pub(crate) async fn export_read(&self, name: &str) -> Result> { let uri = self.export_uri(name); diff --git a/src/transport.rs b/src/transport.rs index 2ef5204..c79f2a7 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -4,7 +4,7 @@ //! //! HTTP exposes `POST /v1/search`, `POST /v1/get`, and `POST /v1/ingest`. MCP //! exposes `pond_search` / `pond_get` (the kb-parity surface) plus -//! `pond_sql_query` (read-only SQL); ingest stays HTTP-only and CLI-only. +//! `pond_sql` (read-only SQL); ingest stays HTTP-only and CLI-only. use std::sync::Arc; @@ -167,7 +167,7 @@ pub mod http { } pub mod mcp { - //! The rmcp MCP layer: `pond_search` / `pond_get` / `pond_sql_query` tools + //! The rmcp MCP layer: `pond_search` / `pond_get` / `pond_sql` tools //! and `schema://pond` / `schema://pond-sql` / `stats://pond` (plus //! `pond-sql-export://` export artifacts) resources, transport-agnostic. //! Mounted on stdio (via `pond mcp`) and on the `/mcp` HTTP route (via @@ -215,7 +215,7 @@ sort_by (relevance default | recency), limit (returned sessions; default 10, \ max 200 - also the want-more knob, there is no pagination), project (path \ substring), session_id (exact session match - search within one session), \ from_date / to_date (YYYY-MM-DD). Subagents are excluded; reach them via \ -pond_sql_query (parent_session_id). +pond_sql (parent_session_id). pond_search response: a transcript. The first line states totals \ (`matched_total` is the message count before `limit` and byte-budget \ @@ -250,7 +250,7 @@ bulk export - use `pond copy --to `."; /// Static documentation served as the `schema://pond-sql` resource: the /// table/column schema, dialect, function set, output modes, pagination - /// pattern, drilling pattern, and worked examples for `pond_sql_query`. + /// pattern, drilling pattern, and worked examples for `pond_sql`. /// Loaded on demand so the tool description stays tight. /// /// TODO(#47): when the lance v8 FM-Index on parts.variant_data lands, @@ -259,7 +259,7 @@ bulk export - use `pond copy --to `."; /// no substring index (yet)" framing) and the timeout message in /// src/sql.rs. const SQL_SCHEMA_DOC: &str = "\ -pond_sql_query runs ONE read-only SELECT (DataFusion SQL, PostgreSQL-compatible) \ +pond_sql runs ONE read-only SELECT (DataFusion SQL, PostgreSQL-compatible) \ over three registered tables. Read-only is hard-enforced: anything other than a \ single SELECT/WITH (or EXPLAIN of one) is rejected (no INSERT/UPDATE/DELETE/\ CREATE/DROP/COPY/SET). @@ -395,7 +395,9 @@ slow queries without leaving SQL. Output modes (the `format` arg): - text (default): a row-capped rendered ASCII table with a header showing \ `{total_rows} in {elapsed_ms} ms; showing {shown}` and, on truncation, a \ -keyset-pagination hint. +keyset-pagination hint. Long text cells clip at ~1000 chars with a \ +`[+N chars ...]` marker - the full value is one format=ndjson call away, or \ +project a narrower json_extract path. - parquet | ndjson: write the FULL result set to a file and return a \ `pond-sql-export://` resource link; read it via MCP resources/read. On a \ local/stdio install the response also names the on-disk path so you can open it \ @@ -564,31 +566,27 @@ Examples (4 patterns the agent should recognize): message_context_after: Option, } - /// `pond_sql_query` MCP tool parameters. + /// `pond_sql` MCP tool parameters. #[derive(Debug, Deserialize, schemars::JsonSchema)] struct McpSqlParams { - /// One read-only SQL statement (DataFusion / PostgreSQL-compatible). - /// SELECT/WITH only (or EXPLAIN of one); writes and side-effecting - /// statements are rejected. Exact columns - messages(session_id, - /// message_id, timestamp, role, source_agent, project, content - /// [system-role only], search_text [the conversational text], - /// embedding_model, options) | sessions(session_id, - /// parent_session_id, parent_message_id, source_agent, created_at, - /// project, options) | parts(session_id, message_id, id, ordinal, - /// type, provenance, tool_name, call_id, is_failure, variant_data, - /// options). parts.type enums use underscores: 'tool_call', - /// 'tool_result', 'text', 'reasoning', 'file'. Tool analytics: use the - /// narrow native columns (tool_name, call_id, is_failure), not - /// json_get_* over variant_data. JSON columns (variant_data, options) - /// are JSONB: read - /// fields with json_extract(col, '$.a.b') or json_get_string(col, - /// 'key', ...), never CAST them. Text search: WHERE - /// contains_tokens(search_text, 'words') to filter, FROM - /// fts('messages', '{...}') for BM25-ranked results. Control row count - /// with SQL `LIMIT`; inline output is capped at 100 rows (use - /// format=parquet|ndjson to get every row). See the `schema://pond-sql` - /// resource for joins, JSON/FTS functions, pagination + drilling - /// patterns, and worked examples. + /// One read-only SQL statement (SELECT/WITH only; writes rejected). + /// Exact columns - messages(session_id, message_id, timestamp, role, + /// source_agent, project, content [system-role only], search_text + /// [the conversational text], embedding_model, options) | + /// sessions(session_id, parent_session_id, parent_message_id, + /// source_agent, created_at, project, options) | parts(session_id, + /// message_id, id, ordinal, type, provenance, tool_name, call_id, + /// is_failure, variant_data, options). parts.type enums use + /// underscores: 'tool_call', 'tool_result', 'text', 'reasoning', + /// 'file'. Tool bodies live in JSONB variant_data - tool_call is + /// {call_id, name, params} (a Bash command is + /// json_extract(variant_data, '$.params.command')), tool_result is + /// {call_id, name, is_failure, result}; never CAST JSON columns. Tool + /// analytics: prefer the narrow native columns (tool_name, call_id, + /// is_failure). Text search: WHERE contains_tokens(search_text, + /// 'words'), or FROM fts('messages', '{...}') for BM25 ranking. + /// Joins, indexed columns, JSON functions, pagination, worked + /// examples: resource schema://pond-sql. #[serde(alias = "sql")] query: String, /// Output format: "text" (default; rendered ASCII table with metrics @@ -647,27 +645,17 @@ Examples (4 patterns the agent should recognize): } #[tool( - description = "Semantic search over stored conversation history. Pick the arm per \ - query with `mode`: \"vector\" (default) matches on meaning - use it \ - for concepts and paraphrases; \"fts\" matches exact whole words \ - (BM25) - use it when you know the literal words. For symbols, \ - substrings, identifiers, cross-session analytics, or subagent \ - sessions, use pond_sql_query instead (this tool excludes subagents). \ - Returns a readable transcript: a leading `key:` line explains the \ - format and the first line states totals plus how many searchable \ - messages the filters left in scope (the absence signal; searchable text \ - is user/assistant conversational text by design - tool calls/results and \ - reasoning are excluded as low-signal noise, so a gap there is expected, \ - not a failure - reach tool output via pond_sql_query over \ - parts.variant_data), then results are \ - grouped by session, best session first; within a session, matching \ - messages are newest-first. Each hit is a `--- [n] score | role | time \ - | message_id | project | agent | session ---` rule followed by the \ - matched text. Pass a returned `message_id` to `pond_get` for full \ - text. Args: query (semantic - concepts, not project names), mode, \ - sort_by (\"relevance\" default | \"recency\"), project / session_id / \ - from_date / to_date to scope, limit to widen (no pagination - raise \ - limit for more). Scores are relative within one response.", + description = "Find relevant messages in past sessions - the entry point for \ + recall: \"have we worked on X\", \"what did we decide about Y\", \ + \"find the session where...\". mode=\"vector\" (default) matches \ + meaning; mode=\"fts\" matches exact whole words (BM25). Scope with \ + project / session_id / from_date / to_date; keep the query semantic \ + (concepts, not project names). Returns scored hits grouped by \ + session, best session first; pass a hit's message_id or session_id \ + to pond_get to read it. Searches conversational text only (tool \ + calls/results and reasoning are excluded by design - a gap there is \ + expected, not a failure) and excludes subagent sessions; reach both \ + via pond_sql. Response format details: resource schema://pond.", annotations(read_only_hint = true, idempotent_hint = true, open_world_hint = false) )] async fn pond_search( @@ -724,22 +712,21 @@ Examples (4 patterns the agent should recognize): } #[tool( - description = "Retrieve stored conversation content as a readable transcript \ - (a leading `key:` line explains the format). Pass exactly one of: \ - session_id (the whole session as a conversational transcript - \ - user/assistant text plus one-line tool/file refs; never inlines \ - tool bodies) OR message_id (that one message with its full parts, \ - incl. tool_call/tool_result bodies, marked `>`, plus its \ - conversational neighbors). Params are prefixed by scope. session_*: \ - session_limit (cap, default 20), session_from (\"start\"|\"end\"; \ - \"end\" = most recent, e.g. to recover context after compaction), \ - session_after_message_id / session_before_message_id (page down/up - \ - pass the id a page marker shows). message_*: message_context_before / \ - message_context_after (conversational neighbors each side, like \ - grep -B/-A, default 3). A session_id response lists the session's \ - subagents in a footer so you can open each. Tool/result lines render \ - as `-> name [call_id]` / `<- name [call_id] (ok|failed)`. Not for \ - bulk export - use `pond copy --to `.", + description = "Read stored conversations - the tool for analyzing, reviewing, or \ + summarizing a past session. Pass exactly one of: session_id (the \ + whole session as a readable chronological transcript - \ + user/assistant text plus one-line tool/file refs) OR message_id \ + (that one message expanded with its full tool_call/tool_result \ + bodies, plus conversational neighbors). Use it after pond_search, \ + or directly when you already have an id. Paging: session_limit \ + (default 20), session_from=\"end\" reads the most recent turns \ + first (post-compaction recovery), session_after_message_id / \ + session_before_message_id to continue; message_context_before / \ + message_context_after size the neighbor window (default 3, like \ + grep -B/-A). A session response lists its subagent sessions in a \ + footer - pass a listed id back as session_id to open one. Not for \ + bulk export - use `pond copy --to `. Response format \ + details: resource schema://pond.", annotations(read_only_hint = true, idempotent_hint = true, open_world_hint = false) )] async fn pond_get( @@ -789,36 +776,23 @@ Examples (4 patterns the agent should recognize): } #[tool( - description = "Run ONE read-only SQL query (DataFusion / PostgreSQL-compatible) \ - over the stored corpus as three tables: sessions, messages, parts. \ - For filtering, joins, and aggregation (counts, group-by, time \ - buckets) - the analytic complement to pond_search's semantic \ - recall. SELECT/WITH only (or EXPLAIN of one); writes and side- \ - effecting statements are rejected. The exact column lists are in \ - the `query` parameter description - use those names, do not guess \ - (column discovery also works: SELECT column_name FROM \ - information_schema.columns WHERE table_name = 'messages'). \ - Routing: metadata analytics -> SQL on messages/sessions; tool-call \ - analytics -> parts WHERE type = 'tool_call' on the narrow native \ - columns (tool_name, call_id, is_failure); text search -> WHERE \ - contains_tokens(search_text, 'words') to filter or FROM \ - fts('messages', '{...json...}') for BM25-ranked results, or \ - pond_search for semantic recall; reading a transcript -> pond_get, \ - not SQL. The embedding `vector` column is never returned (explicit \ - projection is rejected; filtering in WHERE is fine). Control row \ - count with SQL `LIMIT`; inline text output is capped at 100 rows. \ - Queries are wall-clock-capped at 30s by default; set \ - timeout_seconds (max 600) for a genuinely long-running query. \ - format defaults to text (a row-capped rendered table); set \ - format=parquet|ndjson to write the full result to a file returned as \ - a pond-sql-export:// resource (ndjson for machine-readable JSON). \ - Read resource schema://pond-sql \ - for joins, indexed columns, JSON access rules, the function \ - quick-reference, pagination + drilling patterns, and worked \ - examples.", + description = "Advanced escape hatch: run ONE read-only SQL statement \ + (SELECT/WITH, DataFusion / PostgreSQL-compatible) over the \ + sessions / messages / parts tables. NOT for finding or reading \ + conversations - pond_search and pond_get cover almost all recall. \ + Reach for SQL only for: aggregation (counts, group-by, joins, time \ + buckets), exact strings or identifiers in conversational text \ + (contains_tokens / fts), tool-call analytics and tool bodies, \ + subagent sessions, bulk export (format=parquet|ndjson). Read \ + resource schema://pond-sql FIRST - exact columns, indexed \ + predicates, JSON access rules, worked examples; do not guess \ + column names or JSON paths. Inline text output is row-capped \ + and long cells clip with a +N chars marker (full values via \ + format=parquet|ndjson); queries are wall-clock-capped (raise \ + via timeout_seconds).", annotations(read_only_hint = true, idempotent_hint = true, open_world_hint = false) )] - async fn pond_sql_query( + async fn pond_sql( &self, Parameters(params): Parameters, ) -> Result { @@ -931,33 +905,37 @@ Examples (4 patterns the agent should recognize): // rmcp's default `from_build_env` reports the rmcp crate (name + // version) - clients display the server's own identity, so set it. .with_server_info(Implementation::new("pond", env!("CARGO_PKG_VERSION"))) + // The one pond-authored text eagerly loaded into every connected + // session (clients defer tool descriptions behind tool search), + // so it carries ALL the routing; cookbook detail lives in the + // schema:// resources. Wording is deliberate: "analyze / review / + // summarize a session" must bind to pond_get(session_id), and + // pond_sql must never read as the general "analyze" tool. .with_instructions( "pond recalls past agent sessions (Claude Code and others) - prior work, \ - decisions, and context across sessions, not the live conversation. \ - Workflow: pond_search to find relevant messages, then pond_get to read \ - full text by message_id or a whole session by session_id; both return \ - readable transcripts, not JSON. Scope with filters, not the query: project \ - (path substring), session_id, source_agent, from_date / to_date - \ - keep query semantic (concepts, not project names). Scores are relative \ - within one response; there is no min_score. Subagents are stored as their \ - own sessions (source_agent like \"claude-code/general-purpose\"); pond_get \ - on a parent session lists them in a footer so you can open each. Recover \ - context lost to compaction: find this session via pond_search (a distinctive \ - recent topic + project + from_date=today), then pond_get(session_id, \ - session_from=\"end\") for the recent pre-compaction turns. Deeper \ - reference on demand: resource schema://pond (all filters + response format), \ - stats://pond (corpus + embedding stats). For structured/analytic queries \ - (filtering, joins, counts, group-by) use pond_sql_query: read-only SQL \ - (SELECT only) over the sessions/messages/parts tables, with optional \ - parquet/ndjson export; see resource schema://pond-sql. Search indexes only \ - user/assistant conversational text by design (tool calls/results and \ - reasoning are excluded as low-signal noise, not a bug), and a \ - zero/weak result is not proof of absence - for exact strings, \ - identifiers, or error messages run pond_sql_query with WHERE \ - contains_tokens(search_text, 'words') (all words must match; \ - index-accelerated), or FROM fts('messages', \ - '{\"match\":{\"column\":\"search_text\",\"terms\":\"...\"}}') for \ - BM25-ranked results; both cover subagent sessions too.", + decisions, and context across sessions, not the live conversation. Two \ + tools cover almost every request: pond_search finds relevant messages, \ + pond_get reads them; both return readable transcripts. To analyze, \ + review, or summarize a past session: pond_get(session_id) - one call, the \ + whole transcript. For \"what did we decide / latest state\" questions, \ + early conclusions in a long session are often superseded later - read \ + from the end (pond_get session_from=\"end\") or re-sort with pond_search \ + sort_by=\"recency\"; relevance rank favors the early, confident, possibly \ + overturned phrasing. To recover context lost to compaction: pond_search (a \ + distinctive recent topic + project + from_date=today), then \ + pond_get(session_id, session_from=\"end\"). pond_sql is the advanced \ + escape hatch for what search+get cannot express: corpus-wide aggregation \ + (counts, group-by, joins), exact strings or identifiers inside tool \ + bodies, subagent sessions, bulk export (parquet/ndjson) - read resource \ + schema://pond-sql before writing SQL. Scope with filters (project, \ + session_id, from_date / to_date), keep queries semantic (concepts, not \ + project names). Search covers only user/assistant conversational text by \ + design (tool calls/results and reasoning are excluded as low-signal \ + noise, not a bug) - a zero/weak result is not proof of absence: verify \ + exact strings with pond_sql WHERE contains_tokens(search_text, '...') \ + before concluding something never happened. Deeper reference: resources \ + schema://pond (search/get params + response format), schema://pond-sql \ + (SQL columns, functions, worked examples), stats://pond (corpus stats).", ) } @@ -991,7 +969,7 @@ Examples (4 patterns the agent should recognize): SQL_SCHEMA_DOC, request.uri, )])), - // `pond_sql_query` export artifacts: read the file pond wrote + // `pond_sql` export artifacts: read the file pond wrote // (parquet -> base64 blob, ndjson -> text). The filename is // validated to a minted `.` so the URI can't traverse. uri if uri.starts_with("pond-sql-export://") => { @@ -1100,7 +1078,7 @@ Examples (4 patterns the agent should recognize): let chars = match tool.name.as_ref() { "pond_search" => 80_000, "pond_get" => 200_000, - "pond_sql_query" => 80_000, + "pond_sql" => 80_000, _ => continue, }; let mut meta = serde_json::Map::new(); @@ -1133,7 +1111,7 @@ Examples (4 patterns the agent should recognize): CallToolResult::success(vec![Content::text(transcript)]) } - /// Build the `pond_sql_query` export result: a text summary plus a + /// Build the `pond_sql` export result: a text summary plus a /// `resource_link` to the artifact (the spec-canonical way to hand back a /// tool-produced file - the bytes ride `resources/read`, not the tool /// result, so they don't load into context unless the host fetches them). diff --git a/tests/integration/sql.rs b/tests/integration/sql.rs index c1650ba..275d355 100644 --- a/tests/integration/sql.rs +++ b/tests/integration/sql.rs @@ -1,6 +1,6 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] -//! `pond_sql_query` over the stdio-MCP transport (spec.md#protocol): an +//! `pond_sql` over the stdio-MCP transport (spec.md#protocol): an //! in-process rmcp client drives read-only SQL against a synthetic corpus. //! Asserts inline aggregation, the hard read-only gate (DROP/INSERT rejected as //! tool errors), `vector`-column omission, the `fts()` UDTF, and the @@ -193,7 +193,7 @@ fn resource_link_uri(result: &CallToolResult) -> Option { } #[tokio::test(flavor = "multi_thread")] -async fn pond_sql_query_over_mcp() -> anyhow::Result<()> { +async fn pond_sql_over_mcp() -> anyhow::Result<()> { let temp = TempDir::new()?; let state = synthetic_state(&temp).await?; @@ -208,7 +208,7 @@ async fn pond_sql_query_over_mcp() -> anyhow::Result<()> { let call = async |args: serde_json::Value| { client .call_tool( - CallToolRequestParams::new("pond_sql_query") + CallToolRequestParams::new("pond_sql") .with_arguments(args.as_object().unwrap().clone()), ) .await @@ -218,11 +218,11 @@ async fn pond_sql_query_over_mcp() -> anyhow::Result<()> { let tools = client.list_all_tools().await?; let cap = tools .iter() - .find(|tool| tool.name == "pond_sql_query") + .find(|tool| tool.name == "pond_sql") .and_then(|tool| tool.meta.as_ref()) .and_then(|meta| meta.0.get("anthropic/maxResultSizeChars")) .and_then(serde_json::Value::as_i64); - assert_eq!(cap, Some(80_000), "pond_sql_query carries a size cap"); + assert_eq!(cap, Some(80_000), "pond_sql carries a size cap"); // 1. Inline aggregation: GROUP BY over the role column. let result = call(json!({