Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions docs/MEMORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,11 @@ does not remove the search tool, and searching does not alter the session's pinn
Search sits behind the `FactSearchIndex` port (`memory/fact-search.ts`): the session service builds
an engine-independent `FactQuery` from authorized partitions and shapes the bounded response, and
`LexicalFactIndex` answers it with one escaped-`LIKE` statement ranked in SQL by
`FACT_SEARCH_FIELDS`. A different engine (Postgres full-text search, embeddings) is another adapter;
no migration or new service is needed today. Body matching still scans text; bounded results do not
guarantee constant query cost. The opt-in Node SQLite benchmark records query plans and
rare/broad-query timings at 1,000 and 10,000 facts with representative and maximum-size bodies:
`FACT_SEARCH_WEIGHTS` over `MEMORY_SEARCH_FIELDS` (`memory/search-fields.ts`). A different engine
(Postgres full-text search, embeddings) is another adapter; no migration or new service is needed
today. Body matching still scans text; bounded results do not guarantee constant query cost. The
opt-in Node SQLite benchmark records query plans and rare/broad-query timings at 1,000 and 10,000
facts with representative and maximum-size bodies:

```bash
MEMORY_SEARCH_BENCHMARK=1 MEMORY_SEARCH_BENCHMARK_OUTPUT=/tmp/memory-search-benchmark.json \
Expand Down
34 changes: 9 additions & 25 deletions packages/control-plane/src/db/lexical-fact-index.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import {
FACT_SEARCH_FIELDS,
type FactHit,
type FactQuery,
type FactSearchField,
type FactSearchIndex,
type FactSearchPartition,
} from "../memory/fact-search";
Expand All @@ -13,8 +11,10 @@ import {
type PartitionColumns,
type ScopeDisplayColumns,
} from "../memory/partition";
import { likeContains, LIKE_ESCAPE_CLAUSE } from "./like-pattern";
import { CURRENT_MEMORY } from "./memory-queries";
import { likeContains } from "./like-pattern";
import { FACT_SEARCH_WEIGHTS } from "../memory/fact-search";
import { MEMORY_SEARCH_FIELDS } from "../memory/search-fields";
import { CURRENT_MEMORY, revisionFieldMatches, revisionMatchesAllTerms } from "./memory-queries";
import type { SqlDatabase } from "./sql-database";
import { prepareSql, sql, type SqlFragment } from "./sql-fragment";

Expand All @@ -25,28 +25,12 @@ interface FactHitRow extends PartitionColumns, ScopeDisplayColumns {
description: string;
}

const COLUMNS: Record<FactSearchField, SqlFragment> = {
title: sql`lower(r.title)`,
description: sql`lower(r.description)`,
content: sql`lower(r.content)`,
};
const LIKE_ESCAPE = sql.constant(LIKE_ESCAPE_CLAUSE);

const matches = (field: FactSearchField, pattern: string) =>
sql`${COLUMNS[field]} LIKE ${pattern} ${LIKE_ESCAPE}`;

/** The term matches at least one searchable field. */
const matchesAnyField = (pattern: string) =>
sql`(${sql.join(
FACT_SEARCH_FIELDS.map(({ field }) => matches(field, pattern)),
" OR "
)})`;

/** The weight of the strongest field the term matches (fields are listed strongest first). */
const strongestFieldWeight = (pattern: string) =>
sql`(CASE ${sql.join(
FACT_SEARCH_FIELDS.map(
({ field, weight }) => sql`WHEN ${matches(field, pattern)} THEN ${weight}`
MEMORY_SEARCH_FIELDS.map(
(field) =>
sql`WHEN ${revisionFieldMatches(field, pattern)} THEN ${FACT_SEARCH_WEIGHTS[field]}`
),
" "
)} ELSE 0 END)`;
Expand All @@ -60,7 +44,7 @@ function searchable({ partition, pinnedIn }: FactSearchPartition): SqlFragment {

/**
* Portable lexical search: escaped `LIKE` over current active facts, ranked in SQL by
* {@link FACT_SEARCH_FIELDS} in a single statement. Body matching scans text, so response
* {@link FACT_SEARCH_WEIGHTS} in a single statement. Body matching scans text, so response
* bounds do not imply constant query cost.
*/
export class LexicalFactIndex implements FactSearchIndex {
Expand All @@ -77,7 +61,7 @@ export class LexicalFactIndex implements FactSearchIndex {
${CURRENT_MEMORY}
WHERE m.status = 'active' AND m.memory_type = 'fact'
AND (${sql.join(query.partitions.map(searchable), " OR ")})
AND ${sql.join(patterns.map(matchesAnyField), " AND ")}
AND ${revisionMatchesAllTerms(patterns)}
ORDER BY ${sql.join(patterns.map(strongestFieldWeight), " + ")} DESC, m.updated_at DESC, m.id
LIMIT ${query.limit + 1}`
).all<FactHitRow>();
Expand Down
24 changes: 24 additions & 0 deletions packages/control-plane/src/db/memory-queries.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { partitionPredicate, type MemoryPartition } from "../memory/partition";
import { MEMORY_SEARCH_FIELDS, type MemorySearchField } from "../memory/search-fields";
import { LIKE_ESCAPE_CLAUSE } from "./like-pattern";
import { sql, type SqlFragment } from "./sql-fragment";

/**
Expand All @@ -15,3 +17,25 @@ export function inPartitions(partitions: readonly MemoryPartition[]): SqlFragmen
" OR "
)})`;
}

const SEARCH_COLUMNS: Record<MemorySearchField, SqlFragment> = {
title: sql`lower(r.title)`,
description: sql`lower(r.description)`,
content: sql`lower(r.content)`,
};
const LIKE_ESCAPE = sql.constant(LIKE_ESCAPE_CLAUSE);

/** The current revision's `field` matches `pattern`, a lowercase `like-pattern` pattern. */
export const revisionFieldMatches = (field: MemorySearchField, pattern: string): SqlFragment =>
sql`${SEARCH_COLUMNS[field]} LIKE ${pattern} ${LIKE_ESCAPE}`;

/** The pattern matches at least one searchable field of the current revision. */
export const revisionMatchesAnyField = (pattern: string): SqlFragment =>
sql`(${sql.join(
MEMORY_SEARCH_FIELDS.map((field) => revisionFieldMatches(field, pattern)),
" OR "
)})`;

/** Every pattern matches some searchable field; no patterns means no filter. */
export const revisionMatchesAllTerms = (patterns: readonly string[]): SqlFragment =>
patterns.length ? sql`(${sql.join(patterns.map(revisionMatchesAnyField), " AND ")})` : sql`1 = 1`;
7 changes: 6 additions & 1 deletion packages/control-plane/src/db/memory-records.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ import {
import type { MemoryActor, MemoryCandidate, MemoryRecord } from "../memory/types";
import { MAX_D1_QUERY_PARAMETERS } from "./query-limits";
import type { SqlDatabase, SqlStatement } from "./sql-database";
import { CURRENT_MEMORY, inPartitions } from "./memory-queries";
import { likeContains } from "./like-pattern";
import { CURRENT_MEMORY, inPartitions, revisionMatchesAllTerms } from "./memory-queries";
import { prepareSql, sql, type SqlFragment } from "./sql-fragment";
import {
agentWriteGuard,
Expand Down Expand Up @@ -104,6 +105,8 @@ interface MemoryListOptions {
status: MemoryStatus;
offset: number;
limit: number;
/** Lowercase literal terms (see `memorySearchTerms`); each must match title, description or body. Empty means no filter. */
terms: readonly string[];
}

/** Content for a new record; the partition is resolved and authorized by the caller. */
Expand Down Expand Up @@ -139,9 +142,11 @@ export class MemoryRecordStore {

/** Read one management page in stable updated-time/ID order. */
async list(partition: MemoryPartition, options: MemoryListOptions): Promise<MemoryRecord[]> {
// Management search keeps the list's stable order (agent search ranks instead) so paging holds.
const result = await prepareSql(
this.db,
sql`${CURRENT_MEMORY_SELECT} WHERE ${partitionPredicate(partition)} AND m.status = ${options.status}
AND ${revisionMatchesAllTerms(options.terms.map(likeContains))}
ORDER BY m.updated_at DESC, m.id LIMIT ${options.limit} OFFSET ${options.offset}`
).all<MemoryRow>();
return result.results.map(memoryFromRow);
Expand Down
18 changes: 9 additions & 9 deletions packages/control-plane/src/memory/fact-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
type MemorySearchResult,
} from "@open-inspect/shared/types/memories";
import type { MemoryPartition } from "./partition";
import type { MemorySearchField } from "./search-fields";

/** A partition to search; `pinnedIn` restricts it to records pinned in that session. */
export interface FactSearchPartition {
Expand Down Expand Up @@ -33,16 +34,15 @@ export interface FactHit {
}

/**
* Ranking policy: each term scores its strongest matching field, and a hit's score is the sum
* over terms. Ties fall back to the most recently updated, then ID. Engine adapters must
* implement this ordering; listed strongest first.
* Ranking policy: each term scores its strongest matching field (see MEMORY_SEARCH_FIELDS), and a
* hit's score is the sum over terms. Ties fall back to the most recently updated, then ID. Engine
* adapters must implement this ordering.
*/
export const FACT_SEARCH_FIELDS = [
{ field: "title", weight: 5 },
{ field: "description", weight: 3 },
{ field: "content", weight: 1 },
] as const;
export type FactSearchField = (typeof FACT_SEARCH_FIELDS)[number]["field"];
export const FACT_SEARCH_WEIGHTS = {
title: 5,
description: 3,
content: 1,
} as const satisfies Record<MemorySearchField, number>;

/** Searches current active facts. Implementations decide how; the query decides what. */
export interface FactSearchIndex {
Expand Down
6 changes: 6 additions & 0 deletions packages/control-plane/src/memory/search-fields.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
/**
* The current-revision fields a memory text search matches, strongest signal first. Shared by
* agent fact search (which ranks by them) and the management list (which only filters).
*/
export const MEMORY_SEARCH_FIELDS = ["title", "description", "content"] as const;
export type MemorySearchField = (typeof MEMORY_SEARCH_FIELDS)[number];
5 changes: 5 additions & 0 deletions packages/control-plane/src/routes/memories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import {
memoryActionBodySchemas,
memoryPreferencesSchema,
memoryPreviewSchema,
memoryListSearchSchema,
memoryScopeFromSearchParams,
memorySearchTerms,
memoryStatusSchema,
reviseMemorySchema,
type MemoryAction,
Expand Down Expand Up @@ -88,6 +90,8 @@ async function list(request: Request, env: Env, _params: object, ctx: UserRouteC
});
if (!pagination.success) return error("Invalid memory pagination", 400);
const { offset, limit } = pagination.data;
const search = memoryListSearchSchema.safeParse(query.get("search") ?? "");
if (!search.success) return error("Invalid memory search", 400);
const access = await createMemoryManagementPolicy(ctx, env).authorizeScope(scope, "read");
if (access.kind === "denied") return memoryDenialResponse(access.denial);
const store = new MemoryRecordStore(ctx.db);
Expand All @@ -96,6 +100,7 @@ async function list(request: Request, env: Env, _params: object, ctx: UserRouteC
status: status.data,
offset,
limit: limit + 1,
terms: memorySearchTerms(search.data),
});
const page = records.slice(0, limit);
const supersededBy = await store.supersededByIds(page.map((record) => record.id));
Expand Down
37 changes: 37 additions & 0 deletions packages/control-plane/test/integration/memories-routes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,43 @@ describe("memory HTTP lifecycle and session boundaries", () => {
);
expect((await request("/memories?limit=10000")).status).toBe(400);
});
it("filters management records by literal terms across title, description and body", async () => {
const byTitle = await createMemory({ ...content, title: "Rspec retries" });
const byDescription = await createMemory({
...content,
description: "Flaky RSPEC suites in CI",
});
const byBody = await createMemory({ ...content, content: "Run rspec with --seed" });
const wildcard = await createMemory({ ...content, title: "Coverage at 100%_done" });
await createMemory();
const ids = async (search: string, extra = "") => {
const response = await request(`/memories?search=${encodeURIComponent(search)}${extra}`);
expect(response.status).toBe(200);
const body = (await response.json()) as { memories: { id: string }[]; nextOffset: unknown };
return { ids: body.memories.map((memory) => memory.id).sort(), nextOffset: body.nextOffset };
};

expect((await ids("RSpec")).ids).toEqual([byTitle.id, byDescription.id, byBody.id].sort());
// Every term must match somewhere in the record.
expect((await ids("rspec seed")).ids).toEqual([byBody.id]);
// `%` and `_` match themselves, not any text.
expect((await ids("100%_")).ids).toEqual([wildcard.id]);
expect((await ids("0%")).ids).toEqual([wildcard.id]);
// Search combines with paging and status.
const firstPage = await ids("rspec", "&limit=2");
expect(firstPage.ids).toHaveLength(2);
expect(firstPage.nextOffset).toBe(2);
expect((await ids("rspec", "&status=archived")).ids).toEqual([]);
// A blank search is no filter; oversized searches are rejected.
expect((await ids(" ")).ids).toHaveLength(5);
expect((await request(`/memories?search=${"x".repeat(257)}`)).status).toBe(400);
// Nonblank searches share agent search's minimum length.
expect((await request("/memories?search=a")).status).toBe(400);
expect((await request("/memories?search=%20a%20")).status).toBe(400);
expect(
(await request(`/memories?search=${encodeURIComponent("a b c d e f g h i")}`)).status
).toBe(400);
});
it("keeps personal management owner-only, including other administrators", async () => {
const record = await createMemory();
expect((await request(`/memories/${record.id}`, "GET", undefined, OTHER)).status).toBe(404);
Expand Down
2 changes: 1 addition & 1 deletion packages/control-plane/test/integration/memories.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function memory(content: Partial<MemoryContent> = {}, extra: Partial<NewMemory>
...extra,
};
}
const page = { status: "active" as const, offset: 0, limit: 50 };
const page = { status: "active" as const, offset: 0, limit: 50, terms: [] };

describe("memory persistence", () => {
beforeEach(async () => {
Expand Down
31 changes: 31 additions & 0 deletions packages/shared/src/types/memories.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {
allowedMemoryActions,
memoryScopeFromSearchParams,
memoryScopeToSearchParams,
memoryListSearchFromInput,
memoryListSearchSchema,
memorySearchSchema,
MEMORY_TRANSITIONS,
sandboxMemoryWriteSchema,
Expand Down Expand Up @@ -209,3 +211,32 @@ describe("memory lifecycle table", () => {
});
});
});

describe("management list search", () => {
it("allows blank as no filter and applies agent search's bounds otherwise", () => {
const parse = (query: string) => memoryListSearchSchema.safeParse(query);
expect(parse(" ")).toMatchObject({ success: true, data: "" });
expect(parse(" ab ")).toMatchObject({ success: true, data: "ab" });
expect(parse("a").error?.issues[0].message).toBe("Use at least 2 characters.");
expect(parse("x".repeat(257)).error?.issues[0].message).toBe("Use at most 256 characters.");
expect(parse("a1 b2 c3 d4 e5 f6 g7 h8 i9").error?.issues[0].message).toBe(
"Use at most 8 search terms."
);
expect(parse("a1 b2 c3 d4 e5 f6 g7 h8").success).toBe(true);
});

it("treats typing under the minimum as no search yet and explains over-limit queries", () => {
expect(memoryListSearchFromInput("")).toEqual({ search: "" });
expect(memoryListSearchFromInput(" r ")).toEqual({ search: "" });
expect(memoryListSearchFromInput(" rspec ")).toEqual({ search: "rspec" });
expect(memoryListSearchFromInput("a1 b2 c3 d4 e5 f6 g7 h8 i9")).toEqual({
problem: "Use at most 8 search terms.",
});
});

it("gives agent search the same limits", () => {
expect(memorySearchSchema.safeParse({ query: "a" }).error?.issues[0].message).toBe(
"Use at least 2 characters."
);
});
});
40 changes: 30 additions & 10 deletions packages/shared/src/types/memories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,35 @@ export function memorySearchTerms(query: string): string[] {
return [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))];
}

/** A literal search query: one set of bounds for agent search and the management list. */
export const memorySearchQuerySchema = z
.string()
.trim()
.min(MEMORY_SEARCH_LIMITS.queryMin, `Use at least ${MEMORY_SEARCH_LIMITS.queryMin} characters.`)
.max(MEMORY_SEARCH_LIMITS.query, `Use at most ${MEMORY_SEARCH_LIMITS.query} characters.`)
.refine(
(query) => memorySearchTerms(query).length <= MEMORY_SEARCH_LIMITS.terms,
`Use at most ${MEMORY_SEARCH_LIMITS.terms} search terms.`
);
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/** Management-list text filter (`GET /memories?search=`): blank means no filter. */
export const memoryListSearchSchema = z
.string()
.trim()
.pipe(z.literal("").or(memorySearchQuerySchema));

/**
* What a search box applies for the text typed so far. Under the minimum length is not a search
* yet (no filter, so the list stays put while typing); a query over a limit is explained instead
* of sent, since the server would reject it.
*/
export function memoryListSearchFromInput(raw: string): { search: string } | { problem: string } {
const trimmed = raw.trim();
if (trimmed.length < MEMORY_SEARCH_LIMITS.queryMin) return { search: "" };
const parsed = memorySearchQuerySchema.safeParse(trimmed);
return parsed.success ? { search: parsed.data } : { problem: parsed.error.issues[0].message };
}

/** Optional `repoOwner`/`repoName` selector shared by agent search and write inputs. */
const repositorySelectorShape = {
repoOwner: repositoryPairInputSchema.shape.repoOwner
Expand All @@ -410,16 +439,7 @@ function checkRepositorySelector(
/** Repository filters select session members; environment and personal identity are server-derived. */
export const memorySearchSchema = z
.object({
query: z
.string()
.trim()
.min(MEMORY_SEARCH_LIMITS.queryMin)
.max(MEMORY_SEARCH_LIMITS.query)
.refine(
(query) => memorySearchTerms(query).length <= MEMORY_SEARCH_LIMITS.terms,
"Too many search terms"
)
.describe("Short literal keywords; every term must match"),
query: memorySearchQuerySchema.describe("Short literal keywords; every term must match"),
scopeType: memoryScopeTypeSchema
.optional()
.describe("Restrict to one scope type; omit to search every permitted session scope"),
Expand Down
3 changes: 1 addition & 2 deletions packages/web/src/app/(app)/(sidebar)/automations/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { PlusIcon, SearchIcon } from "@/components/ui/icons";
import { useAutomationScope } from "@/hooks/use-automation-scope";

const SEARCH_DEBOUNCE_MS = 300;
import { SEARCH_DEBOUNCE_MS } from "@/lib/search";

export default function AutomationsPage() {
return (
Expand Down
2 changes: 1 addition & 1 deletion packages/web/src/app/(app)/(sidebar)/sessions/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ import {
parseSessionDiscoveryQuery,
type SessionDiscoveryQuery,
} from "@/lib/session-discovery";
import { SEARCH_DEBOUNCE_MS } from "@/lib/search";

const SEARCH_DEBOUNCE_MS = 300;
const MANAGE_ARCHIVED_SESSIONS_HREF = "/settings?tab=data-controls";

export default function SessionsPage() {
Expand Down
Loading
Loading