test(benchmark): add latency bench for envector-msa-1.4.0

agents/common/config.py

@@ -8,6 +8,7 @@
 import json
 from pathlib import Path
 from dataclasses import dataclass, field
+from typing import Optional
 
 # Default config paths
 CONFIG_DIR = Path.home() / ".rune"
@@ -59,6 +60,7 @@ class EnVectorConfig:
     """enVector Cloud credentials (cached from Vault bundle)"""
     endpoint: str = ""
     api_key: str = ""
+    secure: Optional[bool] = None  # None = pyenvector default; API key defaults to TLS
 
 
 @dataclass
@@ -146,9 +148,25 @@ def _parse_envector_config(data: dict) -> EnVectorConfig:
     return EnVectorConfig(
         endpoint=ev_data.get("endpoint", ""),
         api_key=ev_data.get("api_key", ""),
+        secure=_parse_optional_bool(ev_data.get("secure")),
     )
 
 
+def _parse_optional_bool(value) -> Optional[bool]:
+    """Parse bool-like config values while preserving None as 'use SDK default'."""
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in ("true", "1", "yes", "y", "on"):
+            return True
+        if lowered in ("false", "0", "no", "n", "off"):
+            return False
+    raise ValueError(f"invalid boolean value: {value!r}")
+
+
 def _parse_llm_config(data: dict) -> LLMConfig:
     """Parse LLM configuration with backward-compatible migration.
 
@@ -252,6 +270,11 @@ def load_config() -> RuneConfig:
         config.scribe.slack_signing_secret = os.getenv("SLACK_SIGNING_SECRET")
     if os.getenv("NOTION_SIGNING_SECRET"):
         config.scribe.notion_signing_secret = os.getenv("NOTION_SIGNING_SECRET")
+    if os.getenv("ENVECTOR_SECURE"):
+        try:
+            config.envector.secure = _parse_optional_bool(os.getenv("ENVECTOR_SECURE"))
+        except ValueError:
+            print(f"[Config] Warning: invalid ENVECTOR_SECURE value: {os.getenv('ENVECTOR_SECURE')}")
 
     # LLM env var overrides (target config.llm, track env-sourced keys)
     _env_llm_map = {
@@ -318,6 +341,7 @@ def save_config(config: RuneConfig) -> None:
         "envector": {
             "endpoint": config.envector.endpoint,
             "api_key": config.envector.api_key,
+            "secure": config.envector.secure,
         },
         "embedding": {
             "mode": config.embedding.mode,

agents/common/envector_client.py

@@ -7,6 +7,7 @@
 
 import json
 import logging
+import os
 import sys
 from pathlib import Path
 from typing import List, Dict, Any, Optional
@@ -33,6 +34,7 @@ def __init__(
         key_path: str = "~/.rune/keys",
         key_id: str = None,
         access_token: Optional[str] = None,
+        secure: Optional[bool] = None,
         auto_key_setup: bool = True,
         agent_id: Optional[str] = None,
         agent_dek: Optional[bytes] = None,
@@ -53,6 +55,7 @@ def __init__(
         self._key_path = Path(key_path).expanduser()
         self._key_id = key_id
         self._access_token = access_token
+        self._secure = secure
         self._auto_key_setup = auto_key_setup
         self._agent_id = agent_id
         self._agent_dek = agent_dek
@@ -74,9 +77,10 @@ def _ensure_initialized(self) -> None:
                 address=self._address,
                 key_id=self._key_id,
                 key_path=str(self._key_path),
-                eval_mode="rmp",
-                query_encryption=False,  # Plain queries for simplicity
+                eval_mode=os.getenv("ENVECTOR_EVAL_MODE", "mm"),
+                query_encryption="plain",  # Plain queries for simplicity
                 access_token=self._access_token,
+                secure=self._secure,
                 auto_key_setup=self._auto_key_setup,
                 agent_id=self._agent_id,
                 agent_dek=self._agent_dek,
@@ -217,4 +221,3 @@ def remind(
             indices=indices,
             output_fields=output_fields,
         )
-

benchmark/README.md

@@ -8,14 +8,17 @@ Since v0.2.0, Rune operates in **agent-delegated mode**: the calling agent (Clau
 
 This means **capture quality is determined by the scribe prompt**, not by Rune's internal modules. The benchmark reflects this:
 
-| Benchmark | What it tests | Method |
-|-----------|---------------|--------|
-| **scribe_bench** (capture) | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON |
-| **scribe_bench** (extraction) | Is the extracted JSON well-structured? | Same LLM call, score field coverage and extraction type |
-| **retriever_bench** | Can stored decisions be retrieved with various queries? | Offline embedding similarity (FHE is transparent to scores) |
+| Benchmark                     | What it tests                                                  | Method                                                                 |
+| ----------------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- |
+| **scribe_bench** (capture)    | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON      |
+| **scribe_bench** (extraction) | Is the extracted JSON well-structured?                         | Same LLM call, score field coverage and extraction type                |
+| **retriever_bench**           | Can stored decisions be retrieved with various queries?        | Offline embedding similarity (FHE is transparent to scores)            |
+| **latency_bench**             | How long does each pipeline phase take?                        | Direct adapter calls against live envector-msa-1.4.0, per-phase timing |
 
 ## Running
 
+### Quality benchmarks (scribe / retriever)
+
 ```bash
 # Default: uses claude CLI (no API key needed)
 python benchmark/runners/scribe_bench.py
@@ -43,6 +46,43 @@ python benchmark/runners/scribe_bench.py --api-key $ANTHROPIC_API_KEY --provider
 python benchmark/runners/scribe_bench.py --api-key $OPENAI_API_KEY --provider openai --model gpt-4o
 ```
 
+### Latency benchmark (envector-msa-1.4.0)
+
+Requires: Vault running (`tcp://localhost:50051`) and envector-msa-1.4.0 endpoint reachable.
+
+```bash
+# All scenarios (10 runs, 2 warmup)
+python benchmark/runners/latency_bench.py
+
+# Single feature
+python benchmark/runners/latency_bench.py --feature capture
+python benchmark/runners/latency_bench.py --feature recall
+python benchmark/runners/latency_bench.py --feature batch_capture
+python benchmark/runners/latency_bench.py --feature vault_status
+
+# Custom run count (20 total, 3 warmup → 17 effective)
+python benchmark/runners/latency_bench.py --runs 20 --warmup 3
+
+# Save Markdown report
+python benchmark/runners/latency_bench.py \
+    --report benchmark/reports/latency_results_v1.4.0.md \
+    --format md
+
+# Save JSON report
+python benchmark/runners/latency_bench.py \
+    --report benchmark/reports/latency_results_v1.4.0.json \
+    --format json
+```
+
+Pipeline phases measured per feature:
+
+| Feature         | Phases                                                   |
+| --------------- | -------------------------------------------------------- |
+| `capture`       | embed → score (FHE search) → vault_topk → insert → total |
+| `recall`        | embed → score → vault_topk → remind → total              |
+| `batch_capture` | total_batch, per_item (embed+score per item)             |
+| `vault_status`  | vault_health_check                                       |
+
 ## Scenarios (104 total)
 
 ```
@@ -100,49 +140,49 @@ When Rune doesn't behave as expected — a decision was missed, noise was captur
 
 #### Capture scenario fields
 
-| Field | Required | Description |
-|-------|----------|-------------|
-| `id` | yes | Unique identifier. Format: `{category}-{description}-{number}` (e.g., `debug-grpc-timeout-009`) |
-| `category` | yes | Directory path under `scenarios/`. Must match where the file lives (e.g., `capture/should_capture/debugging`) |
-| `language` | yes | Language of the input: `en`, `ko`, `ja`, or `mixed` |
-| `input` | yes | The actual conversation text to evaluate. Paste the real message as-is |
-| `expected_capture` | yes | `true` if this should be captured, `false` if it should be skipped |
-| `expected_fields` | no | Expected metadata when captured (see below). Use `{}` for should_not_capture scenarios |
-| `expected_fields.domain` | no | Expected domain classification: `architecture`, `security`, `product`, `debugging`, `incident`, `ops`, `process`, `qa`, `hr`, `data`, `finance`, `general`, etc. |
-| `expected_fields.status_hint` | no | Expected decision status: `accepted`, `proposed`, or `rejected` |
-| `expected_fields.title_keywords` | no | List of keywords that should appear in the extracted title (case-insensitive, any match passes) |
-| `recall_queries` | no | Optional queries that should (or should not) retrieve this decision after capture |
-| `notes` | no | Free-text annotation explaining why this scenario is interesting or tricky |
+| Field                            | Required | Description                                                                                                                                                      |
+| -------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `id`                             | yes      | Unique identifier. Format: `{category}-{description}-{number}` (e.g., `debug-grpc-timeout-009`)                                                                  |
+| `category`                       | yes      | Directory path under `scenarios/`. Must match where the file lives (e.g., `capture/should_capture/debugging`)                                                    |
+| `language`                       | yes      | Language of the input: `en`, `ko`, `ja`, or `mixed`                                                                                                              |
+| `input`                          | yes      | The actual conversation text to evaluate. Paste the real message as-is                                                                                           |
+| `expected_capture`               | yes      | `true` if this should be captured, `false` if it should be skipped                                                                                               |
+| `expected_fields`                | no       | Expected metadata when captured (see below). Use `{}` for should_not_capture scenarios                                                                           |
+| `expected_fields.domain`         | no       | Expected domain classification: `architecture`, `security`, `product`, `debugging`, `incident`, `ops`, `process`, `qa`, `hr`, `data`, `finance`, `general`, etc. |
+| `expected_fields.status_hint`    | no       | Expected decision status: `accepted`, `proposed`, or `rejected`                                                                                                  |
+| `expected_fields.title_keywords` | no       | List of keywords that should appear in the extracted title (case-insensitive, any match passes)                                                                  |
+| `recall_queries`                 | no       | Optional queries that should (or should not) retrieve this decision after capture                                                                                |
+| `notes`                          | no       | Free-text annotation explaining why this scenario is interesting or tricky                                                                                       |
 
 #### Retriever scenario fields
 
-| Field | Required | Description |
-|-------|----------|-------------|
-| `id` | yes | Format: `recall-{subcategory}-{description}-{number}` |
-| `category` | yes | One of: `recall/exact_match`, `recall/semantic_match`, `recall/cross_domain`, `recall/temporal` |
-| `language` | yes | Language of the query |
-| `seed_records` | yes | Array of records to index before querying. Each has `title`, `domain`, `content`, and optionally `tags` |
-| `query` | yes | The recall query to test |
-| `expected_match_titles` | yes | Titles of seed records that should appear in results above `min_score` |
-| `min_score` | no | Minimum cosine similarity threshold (default: 0.35). Lower for semantic/cross-domain tests |
-| `notes` | no | Free-text annotation |
+| Field                   | Required | Description                                                                                             |
+| ----------------------- | -------- | ------------------------------------------------------------------------------------------------------- |
+| `id`                    | yes      | Format: `recall-{subcategory}-{description}-{number}`                                                   |
+| `category`              | yes      | One of: `recall/exact_match`, `recall/semantic_match`, `recall/cross_domain`, `recall/temporal`         |
+| `language`              | yes      | Language of the query                                                                                   |
+| `seed_records`          | yes      | Array of records to index before querying. Each has `title`, `domain`, `content`, and optionally `tags` |
+| `query`                 | yes      | The recall query to test                                                                                |
+| `expected_match_titles` | yes      | Titles of seed records that should appear in results above `min_score`                                  |
+| `min_score`             | no       | Minimum cosine similarity threshold (default: 0.35). Lower for semantic/cross-domain tests              |
+| `notes`                 | no       | Free-text annotation                                                                                    |
 
 #### Extraction scenario fields
 
-| Field | Required | Description |
-|-------|----------|-------------|
-| `id` | yes | Format: `extract-{type}-{description}-{number}` |
-| `category` | yes | One of: `extraction/single`, `extraction/phase_chain`, `extraction/bundle` |
-| `language` | yes | Language of the input |
-| `input` | yes | The conversation text to extract from |
-| `expected_extraction_type` | yes | Expected extraction mode: `single`, `phase_chain`, or `bundle` |
-| `expected_fields.title_keywords` | no | Keywords expected in extracted title |
-| `expected_fields.status_hint` | no | Expected status: `accepted`, `proposed`, `rejected` |
-| `expected_fields.min_alternatives` | no | Minimum number of alternatives that should be extracted |
-| `expected_fields.min_trade_offs` | no | Minimum number of trade-offs that should be extracted |
-| `expected_fields.min_phases` | no | Minimum phase count (for phase_chain/bundle) |
-| `expected_fields.max_phases` | no | Maximum phase count (for phase_chain/bundle) |
-| `notes` | no | Free-text annotation |
+| Field                              | Required | Description                                                                |
+| ---------------------------------- | -------- | -------------------------------------------------------------------------- |
+| `id`                               | yes      | Format: `extract-{type}-{description}-{number}`                            |
+| `category`                         | yes      | One of: `extraction/single`, `extraction/phase_chain`, `extraction/bundle` |
+| `language`                         | yes      | Language of the input                                                      |
+| `input`                            | yes      | The conversation text to extract from                                      |
+| `expected_extraction_type`         | yes      | Expected extraction mode: `single`, `phase_chain`, or `bundle`             |
+| `expected_fields.title_keywords`   | no       | Keywords expected in extracted title                                       |
+| `expected_fields.status_hint`      | no       | Expected status: `accepted`, `proposed`, `rejected`                        |
+| `expected_fields.min_alternatives` | no       | Minimum number of alternatives that should be extracted                    |
+| `expected_fields.min_trade_offs`   | no       | Minimum number of trade-offs that should be extracted                      |
+| `expected_fields.min_phases`       | no       | Minimum phase count (for phase_chain/bundle)                               |
+| `expected_fields.max_phases`       | no       | Maximum phase count (for phase_chain/bundle)                               |
+| `notes`                            | no       | Free-text annotation                                                       |
 
 ### 1. Capture Scenarios
 
@@ -185,6 +225,8 @@ Extraction produced the wrong structure (e.g., single decision extracted as phas
 - **Paste real conversations** as input — real-world data makes better benchmarks than synthetic examples
 - **Redact sensitive data**: mask API keys, passwords, and PII before adding
 - Verify immediately after adding:
+
   ```bash
   python benchmark/runners/scribe_bench.py --category debugging -v
+
   ```