[codex] Fix critical search indexing and performance

AGENTS.md

@@ -8,9 +8,11 @@ This repo is a prototype OpenCode TUI plugin that replaces the built-in session
 
 - Use `bun install` for dependencies.
 - Use `bun run typecheck` for TypeScript validation.
-- Use `bun run test` for the repo check suite. Right now this intentionally runs `tsc --noEmit`.
+- Use `bun run test` for the repo check suite: `tsc --noEmit` plus the integration tests under `test/integration`.
 - `bunfig.toml` ignores `upstream/**` for bare Bun test discovery so submodule tests are not collected accidentally.
 - Use `bun run dev:opencode -- <workspace>` to launch the plugin in upstream OpenCode without touching the user's real OpenCode config/state.
+- Use `bun run test:perf` for deterministic fuzzy-search performance checks.
+- Use `bun run test:perf:live` for opt-in readonly fuzzy-search benchmarks against the real local OpenCode database.
 
 ## Disposable OpenCode Plugin Testing
 
@@ -42,6 +44,18 @@ Current one-time local disposable theme copied for this machine: `lucent-orng` i
 
 Because the dev run isolates XDG data/state too, it will not show the user's real OpenCode sessions. That is intentional for safe plugin testing. Do not remove that isolation unless a task explicitly asks to test against real local OpenCode data.
 
+## Real Local OpenCode Performance Testing
+
+Only use the user's real OpenCode install when a task explicitly asks for real local database or live TUI comparison. Treat the user's global OpenCode configuration and status/state as read-only: do not edit `~/.config/opencode`, do not change installed plugin config, and do not mutate real session data to set up a test.
+
+Allowed explicit workflows:
+
+- Run readonly performance tests with `bun run test:perf:live`. The live benchmark requires `OPENCODE_SMART_PICKER_PERF_WORKSPACE` to point at a busy local workspace; it reads the real OpenCode SQLite database in readonly mode, filters to that workspace, and writes benchmark sidecar DBs only under the system temp directory.
+- Use an existing tmux session/window or create a new tmux window to launch real `opencode <busy-workspace>` for manual TUI timing when the plugin is already installed. Keep any manual interaction limited to opening the session picker and running searches needed for performance measurement.
+- For disposable plugin isolation, prefer `bun run dev:opencode -- <busy-workspace>`. For real installed-plugin comparison, do not copy or rewrite config; launch the existing real OpenCode setup as-is.
+
+The live fuzzy-search benchmark should choose terms from the local OpenCode DB at runtime to cover high-hit and low-hit behavior. Do not commit real workspace paths, record IDs, session IDs, customer/project identifiers, or sampled query terms into this public repo.
+
 ## Boundaries
 
 - Do not run tests inside `upstream/opencode` or `upstream/opentui` as part of this repo's normal checks.
@@ -60,4 +74,6 @@ Because the dev run isolates XDG data/state too, it will not show the user's rea
 - Search implementation lives under `src/search/`. Keep `src/tui.tsx` as a thin OpenTUI dialog wrapper around `searchSessions`, the mode selector, dependency status chips, and the session result list.
 - The only OpenCode override is the built-in session picker command path and its related dialog. Do not add extra command palette entries, routes, keybind overrides, storage mutations, or config reads unless a task explicitly asks for them.
 - Use OpenCode SDK/plugin types for OpenCode-owned data. Local types are allowed only for plugin-owned sidecar documents, ranking diagnostics, dependency health, and search configuration.
+- The sidecar cache uses one shared connection (`openSharedSidecar`) with WAL + busy_timeout, and incremental session-level reindexing through `indexDelta`/`upsertSessions`. Do not reintroduce per-search `SearchSidecar.open` calls, per-keystroke meta writes, or full-corpus rebuilds on every invalidation event.
+- External-content FTS5 rows must mirror the `document` content table exactly; clearing goes through the FTS5 `delete-all` command. Mismatched values corrupt the index (see the rebuild regression test).
 - Keep `src/tui.tsx` lean: local code should exist only where OpenCode does not expose the native picker internals through the plugin API, or where the semantic search feature needs plugin-owned sidecar behavior.

docs/session-indexing-notes.md

@@ -59,7 +59,8 @@ Start conservative:
   to search reasoning.
 - `part.data.type = "tool"`: index the tool name, completed title, completed
   output, and error text if present. Down-weight verbose outputs.
-- `part.data.type = "file"`: index filename, URL, MIME type, and source fields.
+- `part.data.type = "file"`: index filename, non-`data:` URL, MIME type, and
+  source fields. Do not index base64 payloads from pasted image/PDF data URLs.
 - `part.data.type = "patch"`: index patch summary/content, but chunk carefully.
 - `part.data.type = "subtask"`: index task description/prompt if present.
 

docs/upstream-verification-2026-06-13.md

@@ -0,0 +1,60 @@
+# Upstream Verification - 2026-06-13
+
+Verified this plugin's API usage against a fresh shallow clone of
+`anomalyco/opencode` (HEAD `73dbd8a`, 2026-06-12) and current OpenCode/OpenTUI
+docs. Clone lives under the system temp directory only; do not vendor it.
+
+## Plugin API surface
+
+Every member this plugin uses exists on current main with the same shape:
+
+- `api.keymap.registerLayer` with `commands` (namespace/name/title/category/
+  suggested/slashName/slashAliases/run are open-ended command props read by
+  the palette and slash handling) and `bindings`.
+- `api.ui.DialogSelect` props: `title`, `placeholder`, `options`, `skipFilter`,
+  `onFilter`, `onMove`, `onSelect` (plus `flat`, `current` we do not use).
+  Internal-only props (`actions`, `footerHints`, `gutter`) are stripped by the
+  plugin adapter, so per-row actions like the native picker's rename/delete
+  are not expressible through the public `DialogSelect`.
+- `api.ui.dialog.replace/clear/setSize`, `api.ui.toast`.
+- `api.event.on`: all seven invalidation event names used here exist exactly
+  (`session.updated`, `session.deleted`, `message.updated`, `message.removed`,
+  `message.part.updated`, `message.part.removed`, `session.compacted`).
+  Handlers also receive an undocumented second `{ directory, workspace }`
+  metadata argument at runtime.
+- `api.client.app.log({ service, level, message, extra })`.
+- `api.client.session.list({ roots, search })` and
+  `api.client.session.messages({ sessionID })`.
+- `api.state.ready/session.count/session.messages/part`,
+  `api.lifecycle.signal/onDispose`, `api.tuiConfig.keybinds.get`,
+  `api.route.navigate("session", { sessionID })`, and all theme tokens used in
+  `src/tui.tsx`.
+
+The legacy `api.command` path is deprecated ("Remove in v2");
+`keymap.registerLayer` is the current recommended path. `@opencode-ai/plugin`
+versions in lockstep with opencode.
+
+## Storage facts
+
+- OpenCode core has no FTS5 anywhere. The sidecar FTS index is plugin-owned
+  with no upstream pattern to mirror.
+- `session.list`'s `search` parameter is a `LIKE %term%` match on the session
+  title only, ordered by `time_updated desc`. Message/part content is never
+  searched server-side, which is why this plugin keeps its own index.
+- Session/message/part SQLite schema matches the assumptions in
+  `src/search/source-db.ts` (the real `session` table has more columns than we
+  read; all the ones we read exist).
+- Channel DB naming: `opencode.db` for `latest`/`beta`/`prod` or when
+  `OPENCODE_DISABLE_CHANNEL_DB` is set, else `opencode-<sanitized-channel>.db`.
+  Caveat: `OPENCODE_CHANNEL` is a compile-time define in real builds; setting
+  it at runtime only matters for source runs, which default to `local` anyway.
+
+## OpenTUI notes
+
+- All OpenTUI usages in `src/tui.tsx` are valid on current versions, but
+  `scrollbarOptions={{ visible: false }}`, `wrapMode`, span `style={{ fg, bg }}`,
+  and `KeyEvent.preventDefault/stopPropagation` are source-supported yet
+  undocumented surfaces - re-verify them on OpenTUI upgrades.
+- 0.2.x -> 0.4.x: no breaking changes flagged for scrollbox/box/text/
+  useKeyboard; 0.4.0 swapped to native yoga-layout, so do a visual smoke test
+  when bumping past 0.3.x. `viewportCulling` defaults to true since ~0.3.2.

package.json

@@ -10,14 +10,17 @@
   "scripts": {
     "dev:opencode": "bun run scripts/dev-opencode.ts",
     "typecheck": "tsc --noEmit",
-    "test": "bun run typecheck && bun test test/integration"
+    "test": "bun run typecheck && bun test test/integration",
+    "test:perf": "bun test test/performance",
+    "test:perf:live": "OPENCODE_SMART_PICKER_LIVE_PERF=1 bun test test/performance"
   },
   "dependencies": {
-    "@opencode-ai/plugin": "^1.14.50",
-    "@opencode-ai/sdk": "^1.14.50",
-    "@opentui/core": "^0.2.9",
-    "@opentui/keymap": "^0.2.9",
-    "@opentui/solid": "^0.2.9",
+    "@opencode-ai/plugin": "^1.17.8",
+    "@opencode-ai/sdk": "^1.17.8",
+    "@opentui/core": "^0.3.4",
+    "@opentui/keymap": "^0.3.4",
+    "@opentui/solid": "^0.3.4",
+    "opentui-spinner": "0.0.7",
     "solid-js": "^1.9.12",
     "sqlite-vec": "^0.1.9"
   },

src/search/dependencies.ts

@@ -82,7 +82,9 @@ export async function checkEmbeddingServer(config: SearchConfig) {
   const healthUrls = ["/health", "/v1/health"]
   for (const healthUrl of healthUrls) {
     try {
-      const response = await fetch(new URL(healthUrl, config.embedBaseUrl))
+      const response = await fetch(new URL(healthUrl, config.embedBaseUrl), {
+        signal: AbortSignal.timeout(1_500),
+      })
       if (response.ok) return { state: "available" as const }
     } catch {
       continue

src/search/embedding.ts

@@ -8,13 +8,30 @@ type EmbeddingResponse = {
   model?: string
 }
 
+const HEALTH_TIMEOUT_MS = 1_500
+const EMBED_TIMEOUT_MS = 30_000
+const EMBED_BATCH_SIZE = 64
+const HEALTH_CACHE_TTL_MS = 10_000
+
+const healthCache = new Map<string, { healthy: boolean; checkedAt: number }>()
+
 export class LlamaEmbeddingClient {
   constructor(private readonly config: SearchConfig) {}
 
   async health() {
+    const cached = healthCache.get(this.config.embedBaseUrl)
+    if (cached && Date.now() - cached.checkedAt < HEALTH_CACHE_TTL_MS) return cached.healthy
+    const healthy = await this.checkHealth()
+    healthCache.set(this.config.embedBaseUrl, { healthy, checkedAt: Date.now() })
+    return healthy
+  }
+
+  private async checkHealth() {
     for (const endpoint of ["/health", "/v1/health"]) {
       try {
-        const response = await fetch(new URL(endpoint, this.config.embedBaseUrl))
+        const response = await fetch(new URL(endpoint, this.config.embedBaseUrl), {
+          signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS),
+        })
         if (response.ok) return true
       } catch {
         continue
@@ -28,10 +45,16 @@ export class LlamaEmbeddingClient {
   }
 
   async embedDocuments(documents: string[]) {
-    return this.embed(documents.map((document) => this.config.documentPrefix + document))
+    const inputs = documents.map((document) => this.config.documentPrefix + document)
+    const out: Float32Array[] = []
+    for (let index = 0; index < inputs.length; index += EMBED_BATCH_SIZE) {
+      out.push(...(await this.embed(inputs.slice(index, index + EMBED_BATCH_SIZE))))
+    }
+    return out
   }
 
   private async embed(inputs: string[]) {
+    if (!inputs.length) return []
     const response = await fetch(new URL("/v1/embeddings", this.config.embedBaseUrl), {
       method: "POST",
       headers: { "content-type": "application/json" },
@@ -40,6 +63,7 @@ export class LlamaEmbeddingClient {
         input: inputs,
         encoding_format: "float",
       }),
+      signal: AbortSignal.timeout(EMBED_TIMEOUT_MS),
     })
     if (!response.ok) throw new Error(`Embedding request failed: ${response.status}`)
 

src/search/extractor.ts

@@ -2,6 +2,8 @@ import { createHash } from "node:crypto"
 import type { Message, Part, Session } from "@opencode-ai/sdk/v2"
 import type { SearchDocument } from "./types"
 
+export const SEARCH_EXTRACTOR_VERSION = "4"
+
 function hash(value: unknown) {
   return createHash("sha256").update(JSON.stringify(value)).digest("hex")
 }
@@ -21,16 +23,11 @@ function sourceText(part: Part) {
     case "reasoning":
       return
     case "tool":
-      return joinText([
-        part.tool,
-        part.state.status === "completed" ? part.state.title : undefined,
-        part.state.status === "completed" ? part.state.output : undefined,
-        part.state.status === "error" ? part.state.error : undefined,
-      ])
+      return
     case "file":
       return joinText([
         part.filename,
-        part.url,
+        searchableFileUrl(part.url),
         part.mime,
         part.source?.type === "file" || part.source?.type === "symbol" ? part.source.path : undefined,
         part.source?.type === "symbol" ? part.source.name : undefined,
@@ -39,15 +36,22 @@ function sourceText(part: Part) {
     case "patch":
       return part.files.join("\n")
     case "subtask":
-      return joinText([part.prompt, part.description, part.agent, part.command])
+      return
     case "agent":
-      return joinText([part.name, part.source?.value])
+      return
     default:
       return
   }
 }
 
+function searchableFileUrl(url: string | undefined) {
+  if (!url) return
+  if (/^data:/i.test(url)) return
+  return url
+}
+
 export function extractSearchDocuments(session: Session, message: Message, part: Part): SearchDocument[] {
+  if (message.role !== "user") return []
   const text = sourceText(part)
   if (!text) return []
 
@@ -61,12 +65,7 @@ export function extractSearchDocuments(session: Session, message: Message, part:
     sessionTimeUpdated: session.time.updated,
     messageTimeCreated: message.time.created,
   }
-  const indexedText = joinText([
-    `Title: ${session.title}`,
-    `Role: ${message.role}`,
-    session.path ? `Path: ${session.path}` : session.directory ? `Directory: ${session.directory}` : undefined,
-    text,
-  ])
+  const indexedText = text
 
   return [
     {
@@ -93,7 +92,31 @@ export function extractSessionDocuments(
     parts: Part[]
   }>,
 ) {
-  return messages.flatMap((message) =>
+  const title = session.title?.trim()
+  const titleDocument: SearchDocument[] = title
+    ? [
+        {
+          docID: `opencode:${session.id}:title:0`,
+          sessionID: session.id,
+          chunkIndex: 0,
+          synthetic: true,
+          ignored: false,
+          text: title,
+          metadata: {
+            title: session.title,
+            directory: session.directory,
+            path: session.path,
+            projectID: session.projectID,
+            workspaceID: session.workspaceID,
+            parentID: session.parentID,
+            sessionTimeUpdated: session.time.updated,
+          },
+          sourceHash: hash({ session: session.id, title, time: session.time.updated }),
+        },
+      ]
+    : []
+
+  return titleDocument.concat(messages.flatMap((message) =>
     message.parts.flatMap((part) => extractSearchDocuments(session, message.info, part)),
-  )
+  ))
 }

src/search/fzf.ts

@@ -11,12 +11,7 @@ export type FzfResult =
   | { status: "error"; sessionIDs: []; message: string }
 
 function candidateLine(candidate: FzfCandidate) {
-  const parts = [
-    candidate.session.title,
-    candidate.session.path,
-    candidate.session.directory,
-    candidate.snippet?.replace(/\s+/g, " "),
-  ].filter(Boolean)
+  const parts = [candidate.session.title, candidate.snippet?.replace(/\s+/g, " ")].filter(Boolean)
   return `${candidate.session.id}\t${parts.join(" ")}`
 }
 
@@ -31,7 +26,7 @@ export async function runFzfSearch(input: { bin: string; query: string; candidat
       "--scheme=history",
       "--delimiter",
       "\t",
-      "--with-nth",
+      "--nth",
       "2..",
       "--accept-nth",
       "1",

src/search/logging.ts

@@ -7,6 +7,14 @@ const SERVICE = "smart-session-picker"
 
 let sequence = 0
 
+function boolEnv(value: string | undefined) {
+  return value === "1" || value === "true" || value === "yes"
+}
+
+function debugLoggingEnabled() {
+  return boolEnv(process.env.OPENCODE_SMART_PICKER_DEBUG) || boolEnv(process.env.OPENCODE_SMART_PICKER_PERF)
+}
+
 export function nextLogID(prefix: string) {
   sequence += 1
   return `${prefix}-${sequence}`
@@ -75,12 +83,16 @@ export function logEvent(
   message: string,
   extra: Record<string, unknown> = {},
 ) {
+  // Debug events fire per keystroke; only ship them over HTTP when debug
+  // logging is explicitly enabled.
+  if (level === "debug" && !debugLoggingEnabled()) return
+  const effectiveLevel = level === "debug" && debugLoggingEnabled() ? "info" : level
   void api.client.app
     .log({
       service: SERVICE,
-      level,
+      level: effectiveLevel,
       message,
-      extra,
+      extra: effectiveLevel === level ? extra : { ...extra, originalLevel: level },
     })
     .catch(() => {
       // Logging must never break the picker.