Reduce repeated recall database work and release 0.1.66

Author: cucupac

app/core/ports/db/concept_repositories.py

@@ -99,6 +99,7 @@ def get_concept_bundles(
         *,
         repo_id: str,
         concept_ids: Sequence[str],
+        include_evidence: bool,
         include_lifecycle_events: bool = False,
     ) -> dict[str, dict[str, Any]]:
         """Return bundles keyed by concept ID; omit missing concepts."""

app/core/ports/db/retrieval_repositories.py

@@ -60,8 +60,6 @@ def list_concept_keyword_corpus(
         self,
         *,
         repo_id: str,
-        query_terms: Sequence[str] | None = None,
-        candidate_limit: int | None = None,
     ) -> Sequence[dict[str, Any]]:
         """This method returns active concept text rows for lexical ranking."""
 
@@ -75,8 +73,8 @@ def list_structural_memory_relation_rows(
         *,
         repo_id: str,
         include_global: bool,
-        anchor_memory_id: str,
+        anchor_memory_ids: Sequence[str],
         kinds: Sequence[str] | None,
         predicates: Sequence[str],
     ) -> Sequence[dict[str, Any]]:
-        """Return structural relation rows touching an anchor plus visible participants."""
+        """Return structural relation rows touching the anchors plus visible participants."""

app/core/use_cases/retrieval/concept_seed_retrieval.py

@@ -17,15 +17,14 @@
     normalize_lexical_text,
 )
 from app.core.ports.db.retrieval_repositories import (
-    IConceptKeywordRetrievalRepo,
     IConceptSemanticRetrievalRepo,
 )
 
 
 def retrieve_concept_seeds(
     request_data: dict[str, Any],
     *,
-    concept_keyword_retrieval: IConceptKeywordRetrievalRepo | None,
+    concept_corpus: Sequence[dict[str, Any]],
     concept_semantic_retrieval: IConceptSemanticRetrievalRepo | None,
     query_vector: Sequence[float],
     query_model: str | None,
@@ -46,7 +45,7 @@ def retrieve_concept_seeds(
     )
     keyword = _keyword_concept_candidates(
         request_data,
-        concept_keyword_retrieval=concept_keyword_retrieval,
+        concept_corpus=concept_corpus,
         lexical_query=lexical_query,
         thresholds=thresholds,
         limit=limit,
@@ -81,19 +80,19 @@ def _semantic_concept_candidates(
 def _keyword_concept_candidates(
     request_data: dict[str, Any],
     *,
-    concept_keyword_retrieval: IConceptKeywordRetrievalRepo | None,
+    concept_corpus: Sequence[dict[str, Any]],
     lexical_query: LexicalQuery,
     thresholds: ThresholdSettings,
     limit: int,
 ) -> list[dict[str, Any]]:
-    if concept_keyword_retrieval is None:
-        return []
+    terms = lexical_query.terms
+    matching_rows = [
+        row
+        for row in concept_corpus
+        if any(term in str(row["text"]).lower() for term in terms)
+    ]
     candidates = _rank_concept_keyword_candidates(
-        concept_keyword_retrieval.list_concept_keyword_corpus(
-            repo_id=str(request_data["repo_id"]),
-            query_terms=lexical_query.terms,
-            candidate_limit=max(limit * 10, 100),
-        ),
+        (matching_rows or concept_corpus)[: max(limit * 10, 100)],
         lexical_query=lexical_query,
         mode=str(request_data["mode"]),
         limit=limit,

app/core/use_cases/retrieval/deterministic_graph_recall.py

@@ -143,6 +143,7 @@ def build_deterministic_graph_pack(
     concept_bundles = uow.concepts.get_concept_bundles(
         repo_id=request.repo_id,
         concept_ids=list(concept_candidates),
+        include_evidence=False,
     )
     selected_concepts, concept_trace = _select_concepts(
         request=request,
@@ -473,6 +474,22 @@ def _expand_structural_memory_relations(
 ) -> dict[str, Any]:
     """Add canonical structural memory-relation neighbors to recall candidates."""
 
+    rows_by_anchor = defaultdict(list)
+    rows = uow.read_policy.list_structural_memory_relation_rows(
+        repo_id=request.repo_id,
+        include_global=request.include_global,
+        anchor_memory_ids=tuple(memory_candidates),
+        kinds=request.kinds or list(MATURE_MEMORY_KIND_VALUES),
+        predicates=(
+            *STRUCTURAL_PROBLEM_RELATION_PREDICATES,
+            *STRUCTURAL_FACT_UPDATE_RELATION_PREDICATES,
+        ),
+    )
+    for row in rows:
+        for anchor_id in {row["subject_memory_id"], row["object_memory_id"]}:
+            if anchor_id in memory_candidates:
+                rows_by_anchor[anchor_id].append(row)
+
     discovered: list[dict[str, Any]] = []
     for anchor_memory_id in tuple(memory_candidates):
         anchor_entry = memory_candidates[anchor_memory_id]
@@ -481,13 +498,11 @@ def _expand_structural_memory_relations(
             STRUCTURAL_PROBLEM_RELATION_PREDICATES,
             STRUCTURAL_FACT_UPDATE_RELATION_PREDICATES,
         ):
-            rows = uow.read_policy.list_structural_memory_relation_rows(
-                repo_id=request.repo_id,
-                include_global=request.include_global,
-                anchor_memory_id=anchor_memory_id,
-                kinds=request.kinds or list(MATURE_MEMORY_KIND_VALUES),
-                predicates=predicates,
-            )
+            rows = [
+                row
+                for row in rows_by_anchor[anchor_memory_id]
+                if row["predicate"] in predicates
+            ]
             for row in rows:
                 key = (
                     row["subject_memory_id"],
@@ -578,11 +593,14 @@ def _discover_concepts(
             memory_entry["concept_refs"].add(concept_id)
             memory_entry["link_roles"].add(role)
 
+    concept_corpus = uow.concept_keyword_retrieval.list_concept_keyword_corpus(
+        repo_id=request.repo_id
+    )
     for lane in lanes:
         query_vector, query_model = embeddings[lane.name]
         seeds = retrieve_concept_seeds(
             _lane_request_data(request=request, query=lane.query),
-            concept_keyword_retrieval=uow.concept_keyword_retrieval,
+            concept_corpus=concept_corpus,
             concept_semantic_retrieval=uow.concept_semantic_retrieval,
             query_vector=query_vector,
             query_model=query_model,
@@ -733,6 +751,7 @@ def _traverse_selected_concepts(
             uow.concepts.get_concept_bundles(
                 repo_id=request.repo_id,
                 concept_ids=missing_ids,
+                include_evidence=False,
             )
         )
     for neighbor_id in unique_neighbor_ids:

app/infrastructure/db/runtime/repos/relational/concepts_repo.py

@@ -445,6 +445,7 @@ def get_concept_bundle(
         return self.get_concept_bundles(
             repo_id=repo_id,
             concept_ids=[concept.id],
+            include_evidence=True,
             include_lifecycle_events=include_lifecycle_events,
         ).get(concept.id)
 
@@ -453,6 +454,7 @@ def get_concept_bundles(
         *,
         repo_id: str,
         concept_ids: Sequence[str],
+        include_evidence: bool,
         include_lifecycle_events: bool = False,
     ) -> dict[str, dict[str, Any]]:
         """Fetch candidate bundles in a fixed number of repo-scoped queries."""
@@ -535,7 +537,7 @@ def rows(table, condition):
                 for concept_id in concept_ids_for_event:
                     bundles[concept_id]["lifecycle_events"].append(event)
                 owners[("lifecycle_event", event.id)].update(concept_ids_for_event)
-        if owners:
+        if include_evidence and owners:
             for row in self._unified_evidence_rows(
                 repo_id=repo_id,
                 targets=[(f"concept_{kind}", target_id) for kind, target_id in owners],

app/infrastructure/db/runtime/repos/relational/read_policy_repo.py

@@ -26,11 +26,14 @@ def list_structural_memory_relation_rows(
         *,
         repo_id: str,
         include_global: bool,
-        anchor_memory_id: str,
+        anchor_memory_ids: Sequence[str],
         kinds: Sequence[str] | None,
         predicates: Sequence[str],
     ) -> Sequence[dict[str, Any]]:
-        """Return active structural relation rows touching an anchor."""
+        """Return active structural relation rows touching the anchors in stable order."""
+
+        if not anchor_memory_ids:
+            return []
 
         rows = (
             self._session.execute(
@@ -41,23 +44,41 @@ def list_structural_memory_relation_rows(
                     structural_memory_relations.c.status,
                     structural_memory_relations.c.confidence,
                     structural_memory_relations.c.validated_at,
-                ).where(
+                )
+                .where(
                     structural_memory_relations.c.repo_id == repo_id,
                     structural_memory_relations.c.predicate.in_(list(predicates)),
                     structural_memory_relations.c.status.in_(
                         list(POSITIVE_LIFECYCLE_STATUSES)
                     ),
                     or_(
-                        structural_memory_relations.c.subject_memory_id
-                        == anchor_memory_id,
-                        structural_memory_relations.c.object_memory_id
-                        == anchor_memory_id,
+                        structural_memory_relations.c.subject_memory_id.in_(
+                            anchor_memory_ids
+                        ),
+                        structural_memory_relations.c.object_memory_id.in_(
+                            anchor_memory_ids
+                        ),
                     ),
                 )
+                .order_by(
+                    structural_memory_relations.c.subject_memory_id,
+                    structural_memory_relations.c.predicate,
+                    structural_memory_relations.c.object_memory_id,
+                )
             )
             .mappings()
             .all()
         )
+        visible_ids = self._visible_memory_ids(
+            repo_id=repo_id,
+            include_global=include_global,
+            kinds=kinds,
+            memory_ids=[
+                str(row[field])
+                for row in rows
+                for field in ("subject_memory_id", "object_memory_id")
+            ],
+        )
         return [
             {
                 "subject_memory_id": str(row["subject_memory_id"]),
@@ -68,15 +89,8 @@ def list_structural_memory_relation_rows(
                 "validated_at": row["validated_at"],
                 "visible_memory_ids": tuple(
                     sorted(
-                        self._visible_memory_ids(
-                            repo_id=repo_id,
-                            include_global=include_global,
-                            kinds=kinds,
-                            memory_ids=(
-                                str(row["subject_memory_id"]),
-                                str(row["object_memory_id"]),
-                            ),
-                        )
+                        {str(row["subject_memory_id"]), str(row["object_memory_id"])}
+                        & visible_ids
                     )
                 ),
             }

app/infrastructure/db/runtime/repos/semantic/concept_retrieval_repo.py

@@ -33,8 +33,6 @@ def list_concept_keyword_corpus(
         self,
         *,
         repo_id: str,
-        query_terms: Sequence[str] | None = None,
-        candidate_limit: int | None = None,
     ) -> Sequence[dict[str, Any]]:
         """Return one aggregate lexical document per active concept."""
 
@@ -70,15 +68,11 @@ def list_concept_keyword_corpus(
         self._append_claim_parts(repo_id=repo_id, parts_by_id=parts_by_id)
         self._append_grounding_parts(repo_id=repo_id, parts_by_id=parts_by_id)
 
-        rows = [
+        return [
             {"concept_id": concept_id, "text": " ".join(parts_by_id[concept_id])}
             for concept_id in ordered_ids
             if parts_by_id[concept_id]
         ]
-        filtered_rows = _filter_candidate_rows(rows, query_terms=query_terms)
-        if candidate_limit is not None and candidate_limit > 0:
-            return filtered_rows[: int(candidate_limit)]
-        return filtered_rows
 
     def _append_alias_parts(
         self, *, repo_id: str, parts_by_id: dict[str, list[str]]
@@ -96,7 +90,9 @@ def _append_alias_parts(
                     concepts.c.repo_id == repo_id,
                     concepts.c.status == "active",
                 )
-                .order_by(concept_aliases.c.concept_id.asc(), concept_aliases.c.alias.asc())
+                .order_by(
+                    concept_aliases.c.concept_id.asc(), concept_aliases.c.alias.asc()
+                )
             )
             .mappings()
             .all()
@@ -243,20 +239,6 @@ def _extend_parts(parts: list[str], *values: object) -> None:
             parts.append(text)
 
 
-def _filter_candidate_rows(
-    rows: list[dict[str, Any]], *, query_terms: Sequence[str] | None
-) -> list[dict[str, Any]]:
-    if not query_terms:
-        return rows
-    terms = [str(term).strip().lower() for term in query_terms if str(term).strip()]
-    if not terms:
-        return rows
-    matched = [
-        row for row in rows if any(term in str(row["text"]).lower() for term in terms)
-    ]
-    return matched or rows
-
-
 def _locator_scalars(value: object) -> tuple[str, ...]:
     scalars: list[str] = []
 

app/infrastructure/db/runtime/repos/semantic/keyword_retrieval_repo.py

@@ -66,6 +66,13 @@ def _ranked_fts_rows(
 
         ts_vector = func.to_tsvector(_ENGLISH_REGCONFIG, memories.c.text)
         ts_query = func.websearch_to_tsquery(_ENGLISH_REGCONFIG, query_string)
+        # Keep the broad visibility query from choosing a full text-parsing scan.
+        keyword_hits = (
+            select(memories.c.id)
+            .where(ts_vector.op("@@")(ts_query))
+            .cte("keyword_hits")
+            .prefix_with("MATERIALIZED")
+        )
         rank = func.ts_rank_cd(ts_vector, ts_query)
         stmt = (
             select(
@@ -79,7 +86,7 @@ def _ranked_fts_rows(
                     include_global=include_global,
                     kinds=kinds,
                 ),
-                ts_vector.op("@@")(ts_query),
+                memories.c.id.in_(select(keyword_hits.c.id)),
             )
             .order_by(desc(rank), memories.c.id.asc())
         )

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "shellbrain"
-version = "0.1.65"
+version = "0.1.66"
 description = "Repo-scoped Shellbrain CLI with explicit evidence-backed writes."
 readme = "README.md"
 requires-python = ">=3.11"

tests/operations/concepts/execution/test_concept_show_use_case.py

@@ -370,17 +370,36 @@ def count(*args):
         event.listen(connection, "before_cursor_execute", count)
         try:
             assert (
-                uow.concepts.get_concept_bundles(repo_id="repo-a", concept_ids=[]) == {}
+                uow.concepts.get_concept_bundles(
+                    include_evidence=True, repo_id="repo-a", concept_ids=[]
+                )
+                == {}
             )
             assert statements == []
             bundles = uow.concepts.get_concept_bundles(
+                include_evidence=True,
                 repo_id="repo-a",
                 concept_ids=["third", second_id, "missing", "foreign", first_id],
                 include_lifecycle_events=True,
             )
         finally:
             event.remove(connection, "before_cursor_execute", count)
         assert len(statements) == 9
+        statements.clear()
+        event.listen(connection, "before_cursor_execute", count)
+        try:
+            lean_bundles = uow.concepts.get_concept_bundles(
+                repo_id="repo-a",
+                concept_ids=list(bundles),
+                include_evidence=False,
+                include_lifecycle_events=True,
+            )
+        finally:
+            event.remove(connection, "before_cursor_execute", count)
+        assert len(statements) == 8
+        assert lean_bundles == {
+            key: {**bundle, "evidence": []} for key, bundle in bundles.items()
+        }
         assert set(bundles) == {first_id, second_id, "third"}
         assert bundles[first_id]["relations"] == bundles[second_id]["relations"]
         assert bundles[first_id]["lifecycle_events"] == []
@@ -401,6 +420,7 @@ def count(*args):
                 repo_id="repo-a", concept_ref=concept_id, include_lifecycle_events=True
             )
         assert bundles == uow.concepts.get_concept_bundles(
+            include_evidence=True,
             repo_id="repo-a",
             concept_ids=list(reversed(bundles)),
             include_lifecycle_events=True,