diff --git a/agents/common/config.py b/agents/common/config.py index 2be3a1ff..9cb48d7e 100644 --- a/agents/common/config.py +++ b/agents/common/config.py @@ -8,6 +8,7 @@ import json from pathlib import Path from dataclasses import dataclass, field +from typing import Optional # Default config paths CONFIG_DIR = Path.home() / ".rune" @@ -59,6 +60,7 @@ class EnVectorConfig: """enVector Cloud credentials (cached from Vault bundle)""" endpoint: str = "" api_key: str = "" + secure: Optional[bool] = None # None = pyenvector default; API key defaults to TLS @dataclass @@ -146,9 +148,25 @@ def _parse_envector_config(data: dict) -> EnVectorConfig: return EnVectorConfig( endpoint=ev_data.get("endpoint", ""), api_key=ev_data.get("api_key", ""), + secure=_parse_optional_bool(ev_data.get("secure")), ) +def _parse_optional_bool(value) -> Optional[bool]: + """Parse bool-like config values while preserving None as 'use SDK default'.""" + if value is None or value == "": + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ("true", "1", "yes", "y", "on"): + return True + if lowered in ("false", "0", "no", "n", "off"): + return False + raise ValueError(f"invalid boolean value: {value!r}") + + def _parse_llm_config(data: dict) -> LLMConfig: """Parse LLM configuration with backward-compatible migration. @@ -252,6 +270,11 @@ def load_config() -> RuneConfig: config.scribe.slack_signing_secret = os.getenv("SLACK_SIGNING_SECRET") if os.getenv("NOTION_SIGNING_SECRET"): config.scribe.notion_signing_secret = os.getenv("NOTION_SIGNING_SECRET") + if os.getenv("ENVECTOR_SECURE"): + try: + config.envector.secure = _parse_optional_bool(os.getenv("ENVECTOR_SECURE")) + except ValueError: + print(f"[Config] Warning: invalid ENVECTOR_SECURE value: {os.getenv('ENVECTOR_SECURE')}") # LLM env var overrides (target config.llm, track env-sourced keys) _env_llm_map = { @@ -318,6 +341,7 @@ def save_config(config: RuneConfig) -> None: "envector": { "endpoint": config.envector.endpoint, "api_key": config.envector.api_key, + "secure": config.envector.secure, }, "embedding": { "mode": config.embedding.mode, diff --git a/agents/common/envector_client.py b/agents/common/envector_client.py index 9afda072..11f223e9 100644 --- a/agents/common/envector_client.py +++ b/agents/common/envector_client.py @@ -7,6 +7,7 @@ import json import logging +import os import sys from pathlib import Path from typing import List, Dict, Any, Optional @@ -33,6 +34,7 @@ def __init__( key_path: str = "~/.rune/keys", key_id: str = None, access_token: Optional[str] = None, + secure: Optional[bool] = None, auto_key_setup: bool = True, agent_id: Optional[str] = None, agent_dek: Optional[bytes] = None, @@ -53,6 +55,7 @@ def __init__( self._key_path = Path(key_path).expanduser() self._key_id = key_id self._access_token = access_token + self._secure = secure self._auto_key_setup = auto_key_setup self._agent_id = agent_id self._agent_dek = agent_dek @@ -74,9 +77,10 @@ def _ensure_initialized(self) -> None: address=self._address, key_id=self._key_id, key_path=str(self._key_path), - eval_mode="rmp", - query_encryption=False, # Plain queries for simplicity + eval_mode=os.getenv("ENVECTOR_EVAL_MODE", "mm"), + query_encryption="plain", # Plain queries for simplicity access_token=self._access_token, + secure=self._secure, auto_key_setup=self._auto_key_setup, agent_id=self._agent_id, agent_dek=self._agent_dek, @@ -217,4 +221,3 @@ def remind( indices=indices, output_fields=output_fields, ) - diff --git a/benchmark/README.md b/benchmark/README.md index db4c1a2b..95b45d61 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -8,14 +8,17 @@ Since v0.2.0, Rune operates in **agent-delegated mode**: the calling agent (Clau This means **capture quality is determined by the scribe prompt**, not by Rune's internal modules. The benchmark reflects this: -| Benchmark | What it tests | Method | -|-----------|---------------|--------| -| **scribe_bench** (capture) | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON | -| **scribe_bench** (extraction) | Is the extracted JSON well-structured? | Same LLM call, score field coverage and extraction type | -| **retriever_bench** | Can stored decisions be retrieved with various queries? | Offline embedding similarity (FHE is transparent to scores) | +| Benchmark | What it tests | Method | +| ----------------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- | +| **scribe_bench** (capture) | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON | +| **scribe_bench** (extraction) | Is the extracted JSON well-structured? | Same LLM call, score field coverage and extraction type | +| **retriever_bench** | Can stored decisions be retrieved with various queries? | Offline embedding similarity (FHE is transparent to scores) | +| **latency_bench** | How long does each pipeline phase take? | Direct adapter calls against live envector-msa-1.4.0, per-phase timing | ## Running +### Quality benchmarks (scribe / retriever) + ```bash # Default: uses claude CLI (no API key needed) python benchmark/runners/scribe_bench.py @@ -43,6 +46,43 @@ python benchmark/runners/scribe_bench.py --api-key $ANTHROPIC_API_KEY --provider python benchmark/runners/scribe_bench.py --api-key $OPENAI_API_KEY --provider openai --model gpt-4o ``` +### Latency benchmark (envector-msa-1.4.0) + +Requires: Vault running (`tcp://localhost:50051`) and envector-msa-1.4.0 endpoint reachable. + +```bash +# All scenarios (10 runs, 2 warmup) +python benchmark/runners/latency_bench.py + +# Single feature +python benchmark/runners/latency_bench.py --feature capture +python benchmark/runners/latency_bench.py --feature recall +python benchmark/runners/latency_bench.py --feature batch_capture +python benchmark/runners/latency_bench.py --feature vault_status + +# Custom run count (20 total, 3 warmup → 17 effective) +python benchmark/runners/latency_bench.py --runs 20 --warmup 3 + +# Save Markdown report +python benchmark/runners/latency_bench.py \ + --report benchmark/reports/latency_results_v1.4.0.md \ + --format md + +# Save JSON report +python benchmark/runners/latency_bench.py \ + --report benchmark/reports/latency_results_v1.4.0.json \ + --format json +``` + +Pipeline phases measured per feature: + +| Feature | Phases | +| --------------- | -------------------------------------------------------- | +| `capture` | embed → score (FHE search) → vault_topk → insert → total | +| `recall` | embed → score → vault_topk → remind → total | +| `batch_capture` | total_batch, per_item (embed+score per item) | +| `vault_status` | vault_health_check | + ## Scenarios (104 total) ``` @@ -100,49 +140,49 @@ When Rune doesn't behave as expected — a decision was missed, noise was captur #### Capture scenario fields -| Field | Required | Description | -|-------|----------|-------------| -| `id` | yes | Unique identifier. Format: `{category}-{description}-{number}` (e.g., `debug-grpc-timeout-009`) | -| `category` | yes | Directory path under `scenarios/`. Must match where the file lives (e.g., `capture/should_capture/debugging`) | -| `language` | yes | Language of the input: `en`, `ko`, `ja`, or `mixed` | -| `input` | yes | The actual conversation text to evaluate. Paste the real message as-is | -| `expected_capture` | yes | `true` if this should be captured, `false` if it should be skipped | -| `expected_fields` | no | Expected metadata when captured (see below). Use `{}` for should_not_capture scenarios | -| `expected_fields.domain` | no | Expected domain classification: `architecture`, `security`, `product`, `debugging`, `incident`, `ops`, `process`, `qa`, `hr`, `data`, `finance`, `general`, etc. | -| `expected_fields.status_hint` | no | Expected decision status: `accepted`, `proposed`, or `rejected` | -| `expected_fields.title_keywords` | no | List of keywords that should appear in the extracted title (case-insensitive, any match passes) | -| `recall_queries` | no | Optional queries that should (or should not) retrieve this decision after capture | -| `notes` | no | Free-text annotation explaining why this scenario is interesting or tricky | +| Field | Required | Description | +| -------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | yes | Unique identifier. Format: `{category}-{description}-{number}` (e.g., `debug-grpc-timeout-009`) | +| `category` | yes | Directory path under `scenarios/`. Must match where the file lives (e.g., `capture/should_capture/debugging`) | +| `language` | yes | Language of the input: `en`, `ko`, `ja`, or `mixed` | +| `input` | yes | The actual conversation text to evaluate. Paste the real message as-is | +| `expected_capture` | yes | `true` if this should be captured, `false` if it should be skipped | +| `expected_fields` | no | Expected metadata when captured (see below). Use `{}` for should_not_capture scenarios | +| `expected_fields.domain` | no | Expected domain classification: `architecture`, `security`, `product`, `debugging`, `incident`, `ops`, `process`, `qa`, `hr`, `data`, `finance`, `general`, etc. | +| `expected_fields.status_hint` | no | Expected decision status: `accepted`, `proposed`, or `rejected` | +| `expected_fields.title_keywords` | no | List of keywords that should appear in the extracted title (case-insensitive, any match passes) | +| `recall_queries` | no | Optional queries that should (or should not) retrieve this decision after capture | +| `notes` | no | Free-text annotation explaining why this scenario is interesting or tricky | #### Retriever scenario fields -| Field | Required | Description | -|-------|----------|-------------| -| `id` | yes | Format: `recall-{subcategory}-{description}-{number}` | -| `category` | yes | One of: `recall/exact_match`, `recall/semantic_match`, `recall/cross_domain`, `recall/temporal` | -| `language` | yes | Language of the query | -| `seed_records` | yes | Array of records to index before querying. Each has `title`, `domain`, `content`, and optionally `tags` | -| `query` | yes | The recall query to test | -| `expected_match_titles` | yes | Titles of seed records that should appear in results above `min_score` | -| `min_score` | no | Minimum cosine similarity threshold (default: 0.35). Lower for semantic/cross-domain tests | -| `notes` | no | Free-text annotation | +| Field | Required | Description | +| ----------------------- | -------- | ------------------------------------------------------------------------------------------------------- | +| `id` | yes | Format: `recall-{subcategory}-{description}-{number}` | +| `category` | yes | One of: `recall/exact_match`, `recall/semantic_match`, `recall/cross_domain`, `recall/temporal` | +| `language` | yes | Language of the query | +| `seed_records` | yes | Array of records to index before querying. Each has `title`, `domain`, `content`, and optionally `tags` | +| `query` | yes | The recall query to test | +| `expected_match_titles` | yes | Titles of seed records that should appear in results above `min_score` | +| `min_score` | no | Minimum cosine similarity threshold (default: 0.35). Lower for semantic/cross-domain tests | +| `notes` | no | Free-text annotation | #### Extraction scenario fields -| Field | Required | Description | -|-------|----------|-------------| -| `id` | yes | Format: `extract-{type}-{description}-{number}` | -| `category` | yes | One of: `extraction/single`, `extraction/phase_chain`, `extraction/bundle` | -| `language` | yes | Language of the input | -| `input` | yes | The conversation text to extract from | -| `expected_extraction_type` | yes | Expected extraction mode: `single`, `phase_chain`, or `bundle` | -| `expected_fields.title_keywords` | no | Keywords expected in extracted title | -| `expected_fields.status_hint` | no | Expected status: `accepted`, `proposed`, `rejected` | -| `expected_fields.min_alternatives` | no | Minimum number of alternatives that should be extracted | -| `expected_fields.min_trade_offs` | no | Minimum number of trade-offs that should be extracted | -| `expected_fields.min_phases` | no | Minimum phase count (for phase_chain/bundle) | -| `expected_fields.max_phases` | no | Maximum phase count (for phase_chain/bundle) | -| `notes` | no | Free-text annotation | +| Field | Required | Description | +| ---------------------------------- | -------- | -------------------------------------------------------------------------- | +| `id` | yes | Format: `extract-{type}-{description}-{number}` | +| `category` | yes | One of: `extraction/single`, `extraction/phase_chain`, `extraction/bundle` | +| `language` | yes | Language of the input | +| `input` | yes | The conversation text to extract from | +| `expected_extraction_type` | yes | Expected extraction mode: `single`, `phase_chain`, or `bundle` | +| `expected_fields.title_keywords` | no | Keywords expected in extracted title | +| `expected_fields.status_hint` | no | Expected status: `accepted`, `proposed`, `rejected` | +| `expected_fields.min_alternatives` | no | Minimum number of alternatives that should be extracted | +| `expected_fields.min_trade_offs` | no | Minimum number of trade-offs that should be extracted | +| `expected_fields.min_phases` | no | Minimum phase count (for phase_chain/bundle) | +| `expected_fields.max_phases` | no | Maximum phase count (for phase_chain/bundle) | +| `notes` | no | Free-text annotation | ### 1. Capture Scenarios @@ -185,6 +225,8 @@ Extraction produced the wrong structure (e.g., single decision extracted as phas - **Paste real conversations** as input — real-world data makes better benchmarks than synthetic examples - **Redact sensitive data**: mask API keys, passwords, and PII before adding - Verify immediately after adding: + ```bash python benchmark/runners/scribe_bench.py --category debugging -v + ``` diff --git a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md new file mode 100644 index 00000000..1be1d167 --- /dev/null +++ b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md @@ -0,0 +1,313 @@ +# Rune × envector-msa-1.4.0 Latency Benchmark Plan + +> **[측정 환경]** envector 연결은 `secure=false`(plaintext gRPC)로 측정. Vault(`localhost:50051`)는 TLS 활성 상태. +> plaintext로 진행해도 무방하다고 판단한 이유: gRPC는 HTTP/2 커넥션을 재사용하므로 TLS 핸드셰이크 비용이 warmup 이후 측정값에 포함되지 않으며, 주요 병목이 FHE 연산에 있어 TLS 유무가 결과에 실질적 영향이 낮을 수 있다고 판단. +> 이후 envector TLS 환경(`secure=true`)에서도 동일 시나리오로 벤치마크를 진행할 예정. + +## Context + +기존 benchmark suite는 **capture 품질(정확도)과 recall 품질(hit rate)**을 측정하는 데 집중되어 있다. +이번 작업은 rune의 각 MCP 기능을 실행했을 때 **실제 응답 시간(latency)**을 측정하는 것이 목표다. + +envector-msa-1.4.0은 이전 버전(rmp)에서 크게 변경되었다: + +- **KMS**: 독립 서비스로 분리 (Port 50060) +- **Audit**: 독립 서비스로 분리 (Port 50052) +- **pyenvector 1.4.0**: + - `secure` 파라미터 추가 — TLS 활성/비활성 토글, 기본 False(`access_token` 설정 시 True). + 출처: `envector-msa-1.4.0/docs/architecture/components/sdk.md` Configuration 섹션. + - 검색(Search) 경로의 PPMM native 연산 cache-aware 최적화 추가 — query batch m=4–16에서 + 1.20–1.40x (V2: loop swap), 1.35–1.59x (V3: swap+j-tile) speedup. 단일 쿼리(m=1)에는 효과 없음. + 출처: `envector-msa-1.4.0/docs/design/ppmm-cache-optimization-analysis.md` + (Solution Dev Team, 2026-03-09). +- **gRPC-Gateway**: REST 전사(transcoding) 지원 + +rune의 현재 config는 `envector.endpoint = 146.56.178.130:50050` (원격 클라우드)를 사용하고 있으며, +vault는 `tcp://localhost:50051`에서 로컬 gRPC로 실행된다. + +--- + +## 측정 대상 기능 및 파이프라인 분해 + +### Feature 1: `capture` (MCP tool) + +전체 파이프라인: + +``` +[1] 텍스트 → Embedding (로컬 fastembed, Qwen/Qwen3-Embedding-0.6B) +[2] Novelty Check → envector inner_product (FHE 암호화 쿼리 전송) +[3] Vault TopK Decrypt (gRPC localhost:50051) +[4] FHE Encrypt → pyenvector insert_data (envector 저장) +[5] capture_log 기록 (로컬 파일) +──── +Total end-to-end +``` + +### Feature 2: `recall` (MCP tool) + +전체 파이프라인: + +``` +[1] 쿼리 → Embedding (로컬) +[2] Encrypted Search → envector inner_product +[3] Vault TopK Decrypt (gRPC) +[4] 메타데이터 조회 (로컬 JSON 파싱) +──── +Total end-to-end +``` + +### Feature 3: `batch_capture` (MCP tool) + +- batch size: 1, 5, 10, 20 +- 총 시간 + item당 평균 latency + +### Feature 4: `vault_status` / `diagnostics` (MCP tool) + +- Vault gRPC 연결 latency +- envector 헬스체크 latency + +--- + +## 구체적 테스트 시나리오 (각 기능별) + +### [T1] Capture — 짧은 영문 텍스트 (baseline) + +```json +{ + "input": "We decided to use PostgreSQL as our primary database. Team familiarity and mature ecosystem were the key reasons. Redis considered but rejected due to durability concerns.", + "domain": "architecture", + "expected_insight": "PostgreSQL chosen over Redis for primary DB due to team familiarity and durability" +} +``` + +반복: 10회 (첫 2회 warmup 제외) + +### [T2] Capture — 긴 영문 텍스트 (embedding 부하 테스트) + +```json +{ + "input": "Full ADR: Context — our monolith hit 10k RPS limits. Considered: (1) horizontal scale with read replicas, (2) CQRS split, (3) microservices decomposition. Trade-offs: read replicas cheapest but doesn't solve write bottleneck; CQRS complex but keeps single codebase; microservices highest ops overhead. Decision: CQRS with event sourcing on order-service first. Rationale: allows independent scaling of read path, event log gives audit trail for compliance (legal requirement from Q2 review). Rollback plan: feature flag, revert within 2 sprints if P99 > 500ms.", + "domain": "architecture", + "label": "long_text_~150_tokens" +} +``` + +반복: 10회 + +### [T3] Capture — 한국어 텍스트 (multi-language) + +```json +{ + "input": "Redis를 캐시 레이어로 사용하기로 결정했습니다. Memcached도 검토했지만 데이터 구조 지원(Sorted Set, List)이 필요해서 Redis로 확정. TTL은 1시간으로 설정. 담당: 백엔드팀.", + "domain": "architecture", + "label": "korean_text" +} +``` + +반복: 10회 + +### [T4] Capture — 중복 입력 (novelty check 동작 확인) + +- T1의 입력을 두 번 연속 capture +- 두 번째 호출의 novelty check 결과(near-duplicate) 및 응답 시간 비교 +- 중복 스킵 여부 확인 + +### [T5] Recall — Exact match 쿼리 + +```json +{ + "query": "Why did we choose PostgreSQL?", + "topk": 5, + "label": "exact_match" +} +``` + +반복: 10회 + +### [T6] Recall — Semantic match 쿼리 (의미 기반) + +```json +{ + "query": "데이터베이스 선택 이유", + "topk": 5, + "label": "cross_language_semantic" +} +``` + +반복: 10회 + +### [T7] Recall — topk 변화에 따른 latency + +```json +{ + "query": "architecture decisions", + "topk_variants": [1, 3, 5, 10], + "label": "topk_scaling" +} +``` + +각 topk 당 5회 반복 + +### [T8] Batch Capture — batch size 스케일링 + +```json +{ + "batch_sizes": [1, 5, 10, 20], + "inputs": ["decision_{i}: chose option A over B because ..."], + "label": "batch_scaling" +} +``` + +각 batch size 당 3회 반복 + +### [T9] Vault Status / Diagnostics 헬스체크 + +- `vault_status` 10회 반복 +- `diagnostics` 10회 반복 +- Vault gRPC cold/warm 연결 구분 + +### [T10] Memory 크기별 Recall latency (회귀 테스트) + +- 빈 index에서 recall +- 10개 capture 후 recall +- 50개 capture 후 recall +- 각각 5회 반복하여 index 크기가 latency에 미치는 영향 확인 + +--- + +## 측정 방법론 + +### 통계 기준 + +- **반복**: 최소 10회 (warmup 2회 제외하여 8회 유효) +- **보고 지표**: p50, p95, p99, mean, min, max (ms 단위) +- **타이머**: `time.perf_counter()` (Python 고정밀 타이머) +- **단계별 측정**: 각 파이프라인 단계를 개별 타이머로 계측 + +### 환경 조건 + +- envector-msa-1.4.0: 원격 클라우드 (`146.56.178.130:50050`, `secure=false`) +- Vault: 로컬 (`tcp://localhost:50051`) +- Embedding model: `Qwen/Qwen3-Embedding-0.6B` (로컬) +- 네트워크 조건: 고정 IP, 테스트 전 `ping 146.56.178.130` 기준 RTT 기록 + +--- + +## 구현할 파일 + +### 1. `benchmark/runners/latency_bench.py` (신규) + +기존 `common.py`의 `BenchmarkReport` / `ScenarioResult` 재사용. +새로 추가할 구조: + +```python +@dataclass +class LatencyMeasurement: + phase: str # "embedding", "novelty_check", "fhe_insert", "topk_decrypt", "total" + samples_ms: list[float] + p50: float + p95: float + p99: float + mean: float + +@dataclass +class LatencyBenchReport: + feature: str # "capture", "recall", "batch_capture", "vault_status" + scenario_id: str + measurements: list[LatencyMeasurement] + metadata: dict # text_length, topk, batch_size, etc. +``` + +**측정 방식**: MCP server를 직접 import하지 않고, `mcp/adapter/envector_sdk.py`와 `mcp/adapter/vault_client.py`를 직접 호출하는 방식으로 각 단계 계측. + +**실행 방법**: + +```bash +# 모든 시나리오 +python benchmark/runners/latency_bench.py + +# 특정 feature +python benchmark/runners/latency_bench.py --feature capture +python benchmark/runners/latency_bench.py --feature recall +python benchmark/runners/latency_bench.py --feature batch_capture + +# 반복 횟수 지정 +python benchmark/runners/latency_bench.py --runs 20 + +# 보고서 저장 +python benchmark/runners/latency_bench.py --report benchmark/reports/latency_results_v1.4.0.md --format md +python benchmark/runners/latency_bench.py --report benchmark/reports/latency_results_v1.4.0.json --format json +``` + +### 2. `benchmark/reports/latency_results_v1.4.0.md` (보고서) + +테스트 실행 후 자동 생성. 형식: + +```markdown +# Rune × envector-msa-1.4.0 Latency Report + +## Environment + +- Date: 2026-05-XX +- envector endpoint: 146.56.178.130:50050 +- Vault: tcp://localhost:50051 +- Embedding model: Qwen/Qwen3-Embedding-0.6B +- Network baseline: XX ms RTT + +## Feature: capture + +### T1: Short English text (~30 tokens) + +| Phase | p50 | p95 | p99 | mean | +| ------------------ | --- | --- | --- | ---- | +| embedding | | | | | +| novelty_check | | | | | +| fhe_encrypt+insert | | | | | +| vault_topk_decrypt | | | | | +| **total** | | | | | + +... +``` + +--- + +## 실행 전 사전 조건 체크 + +```bash +# 1. Vault 실행 확인 +grpc_health_probe -addr=localhost:50051 || echo "Vault not running" + +# 2. envector 연결 확인 +python -c "import pyenvector as ev; ev.init(host='146.56.178.130', port=50050)" + +# 3. Embedding model 로드 확인 +python -c "from fastembed import TextEmbedding; m = TextEmbedding('Qwen/Qwen3-Embedding-0.6B'); print('OK')" + +# 4. 네트워크 baseline +ping -c 5 146.56.178.130 +``` + +--- + +## 검증 방법 + +1. **단계별 합계 일치**: `sum(phase latencies) ≈ total` (±5% 허용) +2. **재현성**: 같은 시나리오를 서로 다른 시간에 실행했을 때 p50 변동 < 20% +3. **스케일 선형성 확인**: T8 batch_capture에서 item당 latency가 batch_size 증가에도 일정한지 확인 +4. **중복 감지 동작**: T4에서 두 번째 capture가 "near_duplicate"로 skip되는지 확인 + +--- + +## 수정할 파일 목록 + +| 파일 | 작업 | +| --------------------------------------------- | --------------------------------------------------------- | +| `benchmark/runners/latency_bench.py` | 신규 생성 | +| `benchmark/reports/latency_results_v1.4.0.md` | 실행 후 자동 생성 | +| `benchmark/runners/common.py` | `LatencyMeasurement`, `LatencyBenchReport` dataclass 추가 | + +수정하지 않는 파일: + +- `mcp/adapter/envector_sdk.py` — 직접 호출하되 수정 없음 +- `mcp/adapter/vault_client.py` — 직접 호출하되 수정 없음 +- `agents/common/embedding_service.py` — 직접 호출하되 수정 없음 diff --git a/benchmark/reports/latency_results_v1.4.0.md b/benchmark/reports/latency_results_v1.4.0.md new file mode 100644 index 00000000..fb381578 --- /dev/null +++ b/benchmark/reports/latency_results_v1.4.0.md @@ -0,0 +1,271 @@ +# Rune × envector-msa-1.4.0 Latency Report + +> **[측정 환경]** envector 연결은 `secure=false`(plaintext gRPC)로 측정. Vault(`localhost:50051`)는 TLS 활성 상태. +> plaintext로 진행해도 무방하다고 판단한 이유: gRPC는 HTTP/2 커넥션을 재사용하므로 TLS 핸드셰이크 비용이 warmup 이후 측정값에 포함되지 않으며, 주요 병목이 FHE 연산에 있어 TLS 유무가 결과에 실질적 영향이 낮을 수 있다고 판단. +> 이후 envector TLS 환경(`secure=true`)에서도 동일 시나리오로 벤치마크를 진행할 예정. + +## 실행 결과 요약 + +| 시나리오 | feature | p50 total | p95 total | n | +| ---------------------------------- | ------------- | --------- | --------- | ------ | +| T1 짧은 영문 (~35 tokens) | capture | 102.8 ms | 104.8 ms | 8 | +| T2 긴 영문 (~155 tokens) | capture | 123.2 ms | 126.8 ms | 8 | +| T3 한국어 | capture | 155.0 ms | 165.3 ms | 8 | +| T4 중복 입력 (near-duplicate path) | capture | 106.8 ms | 201.2 ms | 8 | +| T5 Recall — exact match | recall | 37.3 ms | 40.4 ms | 8 | +| T6 Recall — 한→영 cross-language | recall | 38.4 ms | 39.3 ms | 8 | +| T7 Recall — topk 1/3/5/10 | recall | ~39–42 ms | ~42–46 ms | 5 each | +| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~50 ms | ~50 ms | 3 each | +| T9 Vault health check | vault_status | 0.6 ms | 0.8 ms | 8 | + +**capture 병목**: embedding (42–70 ms) + insert (52–78 ms) 가 전체의 ~94%를 차지. +**recall 병목**: embedding (29–32 ms) + score (8–11 ms) 가 전체의 ~100%. +**batch 선형성 확인**: item당 ~50 ms 일정 (batch size 1→20 전 구간). + +--- + +## 주목할 점 + +### vault_topk / remind 가 0 ms 로 측정됨 + +`capture` 및 `recall` 전 시나리오에서 `vault_topk = 0.0 ms`, `remind = 0.0 ms`. +이는 `envector_client.score()` 가 반환하는 `encrypted_blobs` 리스트가 비어 있어 +Vault TopK 복호화 분기가 실행되지 않았음을 의미한다. + +**가능한 원인 (둘 중 하나 또는 복합):** + +1. **envector 클러스터에 FHE eval key가 로드되지 않은 상태** + `insert`는 성공(52–78 ms)하지만 `index.scoring()` 이 빈 CipherBlock 리스트를 반환. + → envector-msa-1.4.0 에서 `load_key` / `load_index` 가 완료된 상태인지 확인 필요. + +2. **`query_encryption = "plain"` 모드** + pyenvector 1.4.0 에서 plain 쿼리는 FHE ciphertext blob 대신 다른 경로로 결과를 처리할 수 있음. + → `query_encryption = "cipher"` 로 전환 후 재측정 필요. + +**결론**: embedding / FHE insert / network score call 의 latency 는 정확하게 측정되었으며, +vault_topk → remind 경로는 현재 환경에서 비활성 상태. **eval key 로드 후 재측정 권장**. + +### T3 한국어 embedding 이 영문 대비 ~65% 느림 + +T1(42 ms) → T3(69 ms): 동일 모델(Qwen3-Embedding-0.6B)에서 한국어 텍스트가 상당히 느림. +토큰 수가 비슷함에도 차이가 크므로 tokenizer subword 분절 수 차이로 추정. + +### T4 중복 입력의 p95 가 p50 대비 2배 (106 → 201 ms) + +`score` 단계에서 occasional spike (9 ms → 72 ms) 가 발생. +중복 경로 자체의 오버헤드가 아닌 네트워크 지터로 보임. 재측정 시 확인 권장. + +--- + +## Environment + +- **date**: 2026-05-04 +- **envector_endpoint**: 146.56.178.130:50050 +- **vault_endpoint**: tcp://localhost:50051 +- **embedding_model**: Qwen/Qwen3-Embedding-0.6B +- **embedding_mode**: sbert +- **index_name**: runecontext +- **key_id**: vault-key +- **network_rtt**: unknown +- **runs_per_scenario**: 8 +- **warmup_runs**: 2 + +## Feature: `capture` + +### T1_short_en + +- label: short English (~30 tokens) +- tokens_approx: 35 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 42.1 | 42.7 | 42.7 | 42.1 | +| score | 8 | 9.0 | 10.0 | 10.2 | 8.8 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 51.7 | 54.0 | 54.5 | 51.8 | +| total | 8 | 102.8 | 104.8 | 105.1 | 102.7 | + +### T2_long_en + +- label: long English (~150 tokens) +- tokens_approx: 155 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 52.8 | 53.5 | 53.6 | 53.0 | +| score | 8 | 8.5 | 10.8 | 11.4 | 8.7 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 62.0 | 63.6 | 63.6 | 61.8 | +| total | 8 | 123.2 | 126.8 | 127.5 | 123.5 | + +### T3_korean + +- label: Korean text +- tokens_approx: 50 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 69.2 | 69.7 | 69.7 | 69.1 | +| score | 8 | 8.8 | 10.1 | 10.2 | 8.8 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 77.6 | 86.3 | 89.3 | 79.1 | +| total | 8 | 155.0 | 165.3 | 168.6 | 157.0 | + +### T4_duplicate + +- label: duplicate input — tests novelty near-duplicate path +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 42.3 | 43.3 | 43.6 | 42.4 | +| score | 8 | 9.5 | 72.8 | 74.4 | 25.5 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 54.6 | 87.5 | 91.6 | 61.8 | +| total | 8 | 106.8 | 201.2 | 203.3 | 129.7 | + +## Feature: `recall` + +### T5_exact_match + +- label: exact match query +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 28.7 | 29.8 | 30.0 | 28.7 | +| score | 8 | 8.7 | 11.1 | 11.5 | 9.1 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 8 | 37.3 | 40.4 | 41.0 | 37.8 | + +### T6_cross_lang + +- label: cross-language semantic (KO→EN) +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 30.1 | 30.5 | 30.6 | 30.0 | +| score | 8 | 8.3 | 9.5 | 9.6 | 8.3 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 8 | 38.4 | 39.3 | 39.4 | 38.3 | + +### T7_topk_1 + +- topk: 1 +- label: topk scaling topk=1 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 31.6 | 33.4 | 33.4 | 31.7 | +| score | 5 | 8.1 | 12.2 | 12.6 | 9.2 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 39.8 | 45.2 | 46.0 | 40.8 | + +### T7_topk_3 + +- topk: 3 +- label: topk scaling topk=3 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 30.3 | 30.6 | 30.6 | 29.6 | +| score | 5 | 11.0 | 15.2 | 15.3 | 11.8 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 41.7 | 45.5 | 45.6 | 41.4 | + +### T7_topk_5 + +- topk: 5 +- label: topk scaling topk=5 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 29.2 | 29.7 | 29.7 | 29.1 | +| score | 5 | 9.8 | 13.2 | 13.7 | 10.1 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 39.3 | 42.3 | 42.9 | 39.2 | + +### T7_topk_10 + +- topk: 10 +- label: topk scaling topk=10 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 28.6 | 30.2 | 30.4 | 28.8 | +| score | 5 | 8.5 | 11.1 | 11.3 | 9.3 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 38.9 | 39.5 | 39.6 | 38.1 | + +## Feature: `batch_capture` + +### T8_batch_1 + +- batch_size: 1 +- label: batch size 1 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 49.9 | 50.1 | 50.2 | 49.4 | +| per_item | 3 | 49.9 | 50.1 | 50.2 | 49.4 | + +### T8_batch_5 + +- batch_size: 5 +- label: batch size 5 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 248.9 | 249.3 | 249.4 | 248.4 | +| per_item | 3 | 49.8 | 49.9 | 49.9 | 49.7 | + +### T8_batch_10 + +- batch_size: 10 +- label: batch size 10 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 499.2 | 499.8 | 499.8 | 499.2 | +| per_item | 3 | 49.9 | 50.0 | 50.0 | 49.9 | + +### T8_batch_20 + +- batch_size: 20 +- label: batch size 20 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 1018.2 | 1019.0 | 1019.0 | 1016.5 | +| per_item | 3 | 50.9 | 50.9 | 51.0 | 50.8 | + +## Feature: `vault_status` + +### T9_vault_status + +- label: Vault gRPC health check +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ------------------ | --- | ------ | ------ | ------ | ------- | +| vault_health_check | 8 | 0.6 | 0.8 | 0.8 | 0.7 | diff --git a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md new file mode 100644 index 00000000..eb9637fa --- /dev/null +++ b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md @@ -0,0 +1,268 @@ +# Rune × envector-msa-1.4.0 Latency Report + +> **[측정 환경]** envector 연결은 `secure=false`(plaintext gRPC)로 측정. Vault(`localhost:50051`)는 TLS 활성 상태. +> plaintext로 진행해도 무방하다고 판단한 이유: gRPC는 HTTP/2 커넥션을 재사용하므로 TLS 핸드셰이크 비용이 warmup 이후 측정값에 포함되지 않으며, 주요 병목이 FHE 연산에 있어 TLS 유무가 결과에 실질적 영향이 낮을 수 있다고 판단. +> 이후 envector TLS 환경(`secure=true`)에서도 동일 시나리오로 벤치마크를 진행할 예정. +> +> **[2026-05-05 재측정 노트]** 2026-05-04 리포트에서 `vault_topk = 0 ms`로 측정되어 FHE 복호화 분기가 비활성으로 의심되던 부분이 해결되었음. 이번 측정은 eval key가 정상 로드된 상태에서 **end-to-end FHE 경로(novelty check + topk decrypt)** 를 모두 거치는 결과이며, 어제 측정 대비 capture는 ~50배, recall은 ~10배 증가한 실제 FHE 연산 비용을 반영함. + +## 실행 결과 요약 + +| 시나리오 | feature | p50 total | p95 total | n | Δ(2026-05-04 대비) | +| ---------------------------------- | ------------- | ----------- | ----------- | ------ | ------------------ | +| T1 짧은 영문 (~35 tokens) | capture | 5269.5 ms | 5449.8 ms | 8 | +5167 ms (×51) | +| T2 긴 영문 (~155 tokens) | capture | 4707.0 ms | 4931.9 ms | 8 | +4584 ms (×38) | +| T3 한국어 | capture | 4595.2 ms | 4789.1 ms | 8 | +4440 ms (×30) | +| T4 중복 입력 (near-duplicate path) | capture | 4886.1 ms | 5020.2 ms | 8 | +4779 ms (×46) | +| T5 Recall — exact match | recall | 368.2 ms | 385.3 ms | 8 | +331 ms (×9.9) | +| T6 Recall — 한→영 cross-language | recall | 384.1 ms | 464.3 ms | 8 | +346 ms (×10.0) | +| T7 Recall — topk 1/3/5/10 | recall | ~367–376 ms | ~374–402 ms | 5 each | ×9.2–×9.6 | +| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~238–245 ms | ~242–246 ms | 3 each | ×4.8 | +| T9 Vault health check | vault_status | 0.8 ms | 0.9 ms | 8 | ≈ 동일 | + +**capture 병목**: `insert` (4200–5000 ms, FHE encrypt + 원격 insert) 가 전체의 **~92%**를 차지. embed/score/vault_topk 모두 합쳐도 5–10% 수준. +**recall 병목**: `score` (180–200 ms) + `vault_topk` (100–115 ms) + `remind` (45–47 ms) 의 FHE 경로가 전체의 **~80%**, 임베딩(~40 ms)은 부수적. +**batch 선형성**: per-item ~240 ms로 batch size 1→20 전 구간 선형 (FHE insert 미수행, embed+score만 측정하는 path). +**recall topk 무관**: T7에서 topk 1→10에도 total p50 367–376 ms로 거의 동일. FHE inner_product cost가 index 크기 의존적이라 topk 변화 영향 없음. + +--- + +## 주목할 점 + +### vault_topk가 정상 측정됨 — 어제 0 ms 문제 해결 + +어제 리포트에서 모든 시나리오의 `vault_topk = 0.0 ms`였던 것이 오늘은 capture 31–100 ms / recall 100–115 ms로 정상 측정됨. eval key 로드 상태가 정상화되어 FHE encrypted_blobs가 비어 있지 않게 되었고, Vault gRPC 복호화 분기가 실제로 실행됨. + +**부수적으로 확인된 인프라 변경**: `mcp/adapter/vault_client.py:33`의 `MAX_MESSAGE_LENGTH`가 256 MB → ~1.95 GB로 상향되어야 했음. EvalKey 응답 크기가 약 1.18 GB에 달해 256 MB 한도에서 `RESOURCE_EXHAUSTED`로 실패하던 것을 동일 브랜치 내 plugin cache 버전과 일치시켜 해결. + +### capture 시간 대부분이 insert에서 소비됨 (~4.5초, 전체의 92%) + +`insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, 절대값은 4.2–5.0초 수준. (참고: envector-msa-1.4.0의 PPMM cache 최적화는 Search 경로 전용이며 insert와 무관 — `envector-msa-1.4.0/docs/design/ppmm-cache-optimization-analysis.md`.) 다음 측정 라운드에서: +1. FHE encrypt 단독 시간 (로컬 CPU) +2. 원격 envector `insert_data` gRPC 시간 (네트워크 + 서버측 저장) +을 분리 계측하면 병목 위치가 더 명확해질 것으로 보임. + +### recall score 단계가 capture novelty score보다 ~2배 무거움 + +같은 `score` 호출인데도 capture 시 100 ms vs recall 시 180 ms. capture는 novelty check만 하고 단순 거리 metric을 받는 반면, recall은 그 결과를 가지고 후속 vault_topk + remind 분기로 연결되며 ScoreEntry payload가 달라지는 것으로 추정. 코드 경로 확인 필요. + +### T3 한국어 capture가 영문보다 score/vault_topk만 느림 + +| | embed | score | vault_topk | insert | +| --------- | ------------ | ------------ | ------------ | ------------- | +| T1 영문 | 92.6 | 100.4 | 31.9 | 5018.1 | +| T3 한국어 | 126.9 (+37%) | 167.5 (+67%) | 75.6 (+137%) | 4209.0 (-16%) | + +embed/score/vault_topk는 한국어가 모두 느리지만 insert는 오히려 더 빠름. 토큰 차이로 embed가 느린 것은 어제와 동일하나, score/vault_topk가 토큰 수에 영향받는 이유는 명확하지 않음 (이론적으로 FHE 벡터 길이가 동일해야 함). 토크나이저 출력의 vector dim이 다르거나, payload 길이 차이가 직렬화 오버헤드로 나타났을 가능성. + +### T4 중복 입력의 score가 다른 capture 시나리오의 2배 + +T1 score 100 ms vs T4 score 211 ms. T4는 같은 텍스트를 두 번 capture하는 시나리오로, novelty 결과가 high-similarity로 나오면서 vault_topk decrypt 분기가 더 무거운 path를 타는 것으로 보임. 어제 리포트의 "p95 spike 발생" 도 같은 원인으로 추정. + +### batch_capture가 단일 capture의 ~5% 시간 + +per-item 240 ms vs 단일 capture 5000 ms. 차이의 대부분은 batch_capture가 **insert를 수행하지 않고** embed + score(novelty)만 측정하기 때문. batch insert를 포함한 진짜 batch 시간이 필요하면 runner에 별도 시나리오 추가 필요. + +--- + +## Environment + +- **date**: 2026-05-05 +- **envector_endpoint**: 146.56.178.130:50050 +- **vault_endpoint**: tcp://localhost:50051 +- **embedding_model**: Qwen/Qwen3-Embedding-0.6B +- **embedding_mode**: sbert +- **index_name**: runecontext +- **key_id**: vault-key +- **network_rtt**: unknown +- **runs_per_scenario**: 8 +- **warmup_runs**: 2 + + +## Feature: `capture` + +### T1_short_en +- label: short English (~30 tokens) +- tokens_approx: 35 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 92.6 | 120.6 | 122.0 | 98.9 | +| score | 8 | 100.4 | 115.0 | 116.4 | 98.5 | +| vault_topk | 8 | 31.9 | 47.3 | 49.4 | 33.6 | +| insert | 8 | 5018.1 | 5242.2 | 5287.3 | 4951.1 | +| total | 8 | 5269.5 | 5449.8 | 5486.2 | 5182.1 | + +### T2_long_en +- label: long English (~150 tokens) +- tokens_approx: 155 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 100.6 | 139.3 | 144.6 | 108.0 | +| score | 8 | 135.7 | 146.0 | 147.1 | 134.3 | +| vault_topk | 8 | 56.0 | 64.4 | 64.7 | 54.9 | +| insert | 8 | 4406.8 | 4645.0 | 4695.4 | 4409.3 | +| total | 8 | 4707.0 | 4931.9 | 4976.3 | 4706.7 | + +### T3_korean +- label: Korean text +- tokens_approx: 50 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 126.9 | 147.3 | 149.4 | 128.3 | +| score | 8 | 167.5 | 184.9 | 187.2 | 168.7 | +| vault_topk | 8 | 75.6 | 91.1 | 92.6 | 76.4 | +| insert | 8 | 4209.0 | 4397.0 | 4405.8 | 4246.7 | +| total | 8 | 4595.2 | 4789.1 | 4806.0 | 4620.1 | + +### T4_duplicate +- label: duplicate input — tests novelty near-duplicate path +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 115.2 | 132.1 | 132.4 | 113.4 | +| score | 8 | 211.4 | 229.7 | 231.1 | 209.3 | +| vault_topk | 8 | 99.7 | 108.9 | 109.6 | 99.7 | +| insert | 8 | 4435.8 | 4568.8 | 4572.0 | 4446.1 | +| total | 8 | 4886.1 | 5020.2 | 5021.7 | 4868.4 | + + +## Feature: `recall` + +### T5_exact_match +- label: exact match query +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 40.4 | 41.2 | 41.3 | 39.7 | +| score | 8 | 182.7 | 200.0 | 204.1 | 185.3 | +| vault_topk | 8 | 98.5 | 105.7 | 106.0 | 99.7 | +| remind | 8 | 45.9 | 49.7 | 50.6 | 46.5 | +| total | 8 | 368.2 | 385.3 | 386.0 | 371.2 | + +### T6_cross_lang +- label: cross-language semantic (KO→EN) +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 41.4 | 47.1 | 47.3 | 42.6 | +| score | 8 | 195.4 | 217.7 | 221.0 | 197.5 | +| vault_topk | 8 | 104.6 | 157.1 | 177.1 | 114.1 | +| remind | 8 | 47.0 | 53.2 | 53.8 | 47.4 | +| total | 8 | 384.1 | 464.3 | 483.0 | 401.5 | + +### T7_topk_1 +- topk: 1 +- label: topk scaling topk=1 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 37.4 | 42.6 | 43.6 | 37.7 | +| score | 5 | 186.7 | 191.7 | 192.0 | 186.5 | +| vault_topk | 5 | 101.2 | 103.2 | 103.6 | 100.7 | +| remind | 5 | 43.5 | 45.1 | 45.3 | 43.0 | +| total | 5 | 367.7 | 375.1 | 375.3 | 367.9 | + +### T7_topk_3 +- topk: 3 +- label: topk scaling topk=3 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 38.0 | 38.5 | 38.6 | 36.8 | +| score | 5 | 181.0 | 191.1 | 191.5 | 182.7 | +| vault_topk | 5 | 105.4 | 109.0 | 109.2 | 104.4 | +| remind | 5 | 45.0 | 45.8 | 45.8 | 44.7 | +| total | 5 | 368.1 | 376.4 | 377.2 | 368.7 | + +### T7_topk_5 +- topk: 5 +- label: topk scaling topk=5 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 36.5 | 38.3 | 38.3 | 36.8 | +| score | 5 | 178.7 | 196.3 | 196.5 | 184.6 | +| vault_topk | 5 | 114.5 | 139.6 | 143.8 | 116.2 | +| remind | 5 | 44.8 | 46.1 | 46.3 | 44.9 | +| total | 5 | 376.4 | 402.2 | 403.8 | 382.4 | + +### T7_topk_10 +- topk: 10 +- label: topk scaling topk=10 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 38.2 | 39.3 | 39.5 | 37.2 | +| score | 5 | 181.9 | 190.2 | 190.6 | 183.8 | +| vault_topk | 5 | 101.8 | 106.0 | 106.4 | 102.6 | +| remind | 5 | 46.2 | 47.2 | 47.3 | 46.0 | +| total | 5 | 369.4 | 374.0 | 374.0 | 369.7 | + + +## Feature: `batch_capture` + +### T8_batch_1 +- batch_size: 1 +- label: batch size 1 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 237.9 | 242.2 | 242.6 | 239.2 | +| per_item | 3 | 237.9 | 242.2 | 242.6 | 239.2 | + +### T8_batch_5 +- batch_size: 5 +- label: batch size 5 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 1207.8 | 1209.7 | 1209.9 | 1207.5 | +| per_item | 3 | 241.6 | 241.9 | 242.0 | 241.5 | + +### T8_batch_10 +- batch_size: 10 +- label: batch size 10 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 2409.5 | 2437.7 | 2440.2 | 2415.5 | +| per_item | 3 | 240.9 | 243.8 | 244.0 | 241.5 | + +### T8_batch_20 +- batch_size: 20 +- label: batch size 20 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 4892.0 | 4907.7 | 4909.1 | 4846.8 | +| per_item | 3 | 244.6 | 245.4 | 245.5 | 242.3 | + + +## Feature: `vault_status` + +### T9_vault_status +- label: Vault gRPC health check +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ------------------ | --- | ------ | ------ | ------ | ------- | +| vault_health_check | 8 | 0.8 | 0.9 | 0.9 | 0.8 | diff --git a/benchmark/runners/common.py b/benchmark/runners/common.py index 912a297f..51ea9d91 100644 --- a/benchmark/runners/common.py +++ b/benchmark/runners/common.py @@ -147,3 +147,143 @@ def check_title_keywords(title: str, keywords: list[str]) -> bool: """Check if a title contains any of the expected keywords (case-insensitive).""" title_lower = title.lower() return any(kw.lower() in title_lower for kw in keywords) + + +# ────────────────────────────────────────────────────────── +# Latency benchmark data structures +# ────────────────────────────────────────────────────────── + +@dataclass +class PhaseLatency: + """Latency statistics for a single pipeline phase.""" + name: str + samples_ms: list = field(default_factory=list) + + def _pct(self, p: float) -> float: + import numpy as np + return float(np.percentile(self.samples_ms, p)) if self.samples_ms else 0.0 + + @property + def p50(self) -> float: + return self._pct(50) + + @property + def p95(self) -> float: + return self._pct(95) + + @property + def p99(self) -> float: + return self._pct(99) + + @property + def mean(self) -> float: + import numpy as np + return float(np.mean(self.samples_ms)) if self.samples_ms else 0.0 + + @property + def min_ms(self) -> float: + return float(min(self.samples_ms)) if self.samples_ms else 0.0 + + @property + def max_ms(self) -> float: + return float(max(self.samples_ms)) if self.samples_ms else 0.0 + + @property + def n(self) -> int: + return len(self.samples_ms) + + def to_dict(self) -> dict: + return { + "phase": self.name, + "n": self.n, + "p50_ms": round(self.p50, 2), + "p95_ms": round(self.p95, 2), + "p99_ms": round(self.p99, 2), + "mean_ms": round(self.mean, 2), + "min_ms": round(self.min_ms, 2), + "max_ms": round(self.max_ms, 2), + "samples_ms": [round(s, 2) for s in self.samples_ms], + } + + +@dataclass +class LatencyScenarioResult: + """Latency results for a single test scenario.""" + scenario_id: str + feature: str # "capture", "recall", "batch_capture", "vault_status" + phases: list = field(default_factory=list) # list[PhaseLatency] + metadata: dict = field(default_factory=dict) + error: str | None = None + + def to_dict(self) -> dict: + return { + "scenario_id": self.scenario_id, + "feature": self.feature, + "phases": [p.to_dict() for p in self.phases], + "metadata": self.metadata, + "error": self.error, + } + + +@dataclass +class LatencyBenchReport: + """Aggregated latency benchmark report.""" + bench_type: str = "latency" + env: dict = field(default_factory=dict) + scenarios: list = field(default_factory=list) # list[LatencyScenarioResult] + + def add(self, result: "LatencyScenarioResult") -> None: + self.scenarios.append(result) + + def to_dict(self) -> dict: + return { + "bench_type": self.bench_type, + "env": self.env, + "scenarios": [s.to_dict() for s in self.scenarios], + } + + def save_json(self, path: Path) -> Path: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(self.to_dict(), indent=2, ensure_ascii=False)) + return path + + def to_markdown(self) -> str: + lines: list[str] = [] + lines.append("# Rune × envector-msa-1.4.0 Latency Report\n") + lines.append("## Environment\n") + for k, v in self.env.items(): + lines.append(f"- **{k}**: {v}") + lines.append("") + + by_feature: dict[str, list] = {} + for s in self.scenarios: + by_feature.setdefault(s.feature, []).append(s) + + for feature, results in by_feature.items(): + lines.append(f"\n## Feature: `{feature}`\n") + for res in results: + lines.append(f"### {res.scenario_id}") + if res.metadata: + for k, v in res.metadata.items(): + lines.append(f"- {k}: {v}") + if res.error: + lines.append(f"\n> **Error**: {res.error}\n") + continue + lines.append("") + header = "| Phase | n | p50 ms | p95 ms | p99 ms | mean ms |" + sep = "|-------|---|--------|--------|--------|---------|" + lines.append(header) + lines.append(sep) + for p in res.phases: + lines.append( + f"| {p.name} | {p.n} " + f"| {p.p50:.1f} | {p.p95:.1f} | {p.p99:.1f} | {p.mean:.1f} |" + ) + lines.append("") + + return "\n".join(lines) + + def save_markdown(self, path: Path) -> Path: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + path.write_text(self.to_markdown(), encoding="utf-8") + return path diff --git a/benchmark/runners/latency_bench.py b/benchmark/runners/latency_bench.py new file mode 100644 index 00000000..fdfaa796 --- /dev/null +++ b/benchmark/runners/latency_bench.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +"""Rune × envector-msa-1.4.0 latency benchmark. + +Measures wall-clock latency of each rune MCP feature broken down by pipeline +phase. Runs standalone — no MCP server needed; adapters are imported directly. + +Scenarios +--------- + capture: + T1 Short English text (~30 tokens) + T2 Long English text (~150 tokens) + T3 Korean text + T4 Duplicate input (novelty-check near-duplicate path) + recall: + T5 Exact match query + T6 Cross-language semantic query (Korean → English) + T7 topk scaling (topk = 1, 3, 5, 10) + batch_capture: + T8 Batch size scaling (sizes = 1, 5, 10, 20) + vault_status: + T9 Vault health check + diagnostics + +Usage +----- + python benchmark/runners/latency_bench.py + python benchmark/runners/latency_bench.py --feature capture + python benchmark/runners/latency_bench.py --runs 20 + python benchmark/runners/latency_bench.py \\ + --report benchmark/reports/latency_results_v1.4.0.md --format md +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +import numpy as np + +# ── path setup ──────────────────────────────────────────────────────────────── +BENCHMARK_DIR = Path(__file__).resolve().parent.parent +RUNE_DIR = BENCHMARK_DIR.parent +MCP_DIR = RUNE_DIR / "mcp" + +for _p in (str(RUNE_DIR), str(MCP_DIR)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from runners.common import ( # noqa: E402 + REPORTS_DIR, + LatencyBenchReport, + LatencyScenarioResult, + PhaseLatency, +) + +# ── sample inputs ───────────────────────────────────────────────────────────── + +_SHORT_TEXT = ( + "We decided to use PostgreSQL as our primary database. " + "Team familiarity and mature ecosystem were the key reasons. " + "Redis considered but rejected due to durability concerns." +) + +_LONG_TEXT = ( + "Full ADR: Context — our monolith hit 10k RPS limits. " + "Considered: (1) horizontal scale with read replicas, " + "(2) CQRS split, (3) microservices decomposition. " + "Trade-offs: read replicas cheapest but doesn't solve write bottleneck; " + "CQRS complex but keeps single codebase; microservices highest ops overhead. " + "Decision: CQRS with event sourcing on order-service first. " + "Rationale: allows independent scaling of read path, event log gives audit " + "trail for compliance (legal requirement from Q2 review). " + "Rollback plan: feature flag, revert within 2 sprints if P99 > 500ms." +) + +_KOREAN_TEXT = ( + "Redis를 캐시 레이어로 사용하기로 결정했습니다. " + "Memcached도 검토했지만 데이터 구조 지원(Sorted Set, List)이 필요해서 Redis로 확정. " + "TTL은 1시간으로 설정. 담당: 백엔드팀." +) + + +def _make_extracted(text: str, title: str, domain: str = "architecture") -> str: + """Minimal agent-delegated extracted JSON for capture benchmark.""" + return json.dumps({ + "tier2": {"capture": True, "domain": domain}, + "title": title, + "rationale": "benchmark test scenario — no LLM involved", + "problem": "benchmark setup", + "alternatives": [], + "trade_offs": [], + "status_hint": "accepted", + "tags": ["benchmark"], + "reusable_insight": text[:120], + "confidence": 0.9, + }, ensure_ascii=False) + + +SCENARIOS_CAPTURE = [ + { + "id": "T1_short_en", + "text": _SHORT_TEXT, + "title": "PostgreSQL chosen as primary DB", + "domain": "architecture", + "metadata": {"label": "short English (~30 tokens)", "tokens_approx": 35}, + }, + { + "id": "T2_long_en", + "text": _LONG_TEXT, + "title": "CQRS event sourcing on order service", + "domain": "architecture", + "metadata": {"label": "long English (~150 tokens)", "tokens_approx": 155}, + }, + { + "id": "T3_korean", + "text": _KOREAN_TEXT, + "title": "Redis 캐시 레이어 결정", + "domain": "architecture", + "metadata": {"label": "Korean text", "tokens_approx": 50}, + }, +] + +SCENARIOS_RECALL = [ + { + "id": "T5_exact_match", + "query": "Why did we choose PostgreSQL?", + "topk": 5, + "metadata": {"label": "exact match query"}, + }, + { + "id": "T6_cross_lang", + "query": "데이터베이스 선택 이유", + "topk": 5, + "metadata": {"label": "cross-language semantic (KO→EN)"}, + }, +] + +RECALL_TOPK_VARIANTS = [1, 3, 5, 10] + +BATCH_SIZES = [1, 5, 10, 20] + + +# ── timing helpers ───────────────────────────────────────────────────────────── + +class _Timer: + """Simple context-manager wall-clock timer (ms).""" + + def __init__(self) -> None: + self.elapsed_ms: float = 0.0 + self._start: float = 0.0 + + def __enter__(self) -> "_Timer": + self._start = time.perf_counter() + return self + + def __exit__(self, *_: Any) -> None: + self.elapsed_ms = (time.perf_counter() - self._start) * 1000.0 + + +# ── main benchmark class ─────────────────────────────────────────────────────── + +class LatencyBenchmark: + """ + Measures latency of each rune pipeline phase against a live envector-msa-1.4.0 + endpoint. All adapter imports happen lazily so the module can be loaded + even if optional deps are missing. + """ + + def __init__(self, runs: int = 10, warmup: int = 2) -> None: + self.runs = runs + self.warmup = warmup + self._config: Any = None + self._index_name: Optional[str] = None + self._key_id: Optional[str] = None + self._embedding: Any = None + self._ev_client: Any = None + self._vault: Any = None + + # ── setup ───────────────────────────────────────────────────────────────── + + async def setup(self) -> None: + """Load config, fetch keys from Vault, init adapters.""" + from agents.common.config import load_config + from agents.common.embedding_service import EmbeddingService + from agents.common.envector_client import EnVectorClient + from adapter.vault_client import VaultClient + + cfg = load_config() + self._config = cfg + + print(" Connecting to Vault …", end=" ", flush=True) + vault = VaultClient( + vault_endpoint=cfg.vault.endpoint, + vault_token=cfg.vault.token, + ca_cert=cfg.vault.ca_cert or None, + tls_disable=cfg.vault.tls_disable, + ) + + bundle = await vault.get_public_key() + + key_id = bundle.pop("key_id", None) + index_name = bundle.pop("index_name", None) + agent_id = bundle.pop("agent_id", None) + agent_dek_b64 = bundle.pop("agent_dek", None) + ev_endpoint = bundle.pop("envector_endpoint", None) or cfg.envector.endpoint + ev_api_key = bundle.pop("envector_api_key", None) or cfg.envector.api_key + + if not key_id: + raise RuntimeError("Vault did not return key_id") + if not index_name: + raise RuntimeError("Vault did not return index_name") + + key_path = Path.home() / ".rune" / "keys" + key_dir = key_path / key_id + key_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + for filename, content in bundle.items(): + fp = key_dir / filename + fd = os.open(str(fp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(content) + + agent_dek: Optional[bytes] = None + if agent_dek_b64: + agent_dek = base64.b64decode(agent_dek_b64) + + self._index_name = index_name + self._key_id = key_id + self._vault = vault + + self._embedding = EmbeddingService( + mode=cfg.embedding.mode, + model=cfg.embedding.model, + ) + + self._ev_client = EnVectorClient( + address=ev_endpoint, + key_path=str(key_path), + key_id=key_id, + access_token=ev_api_key, + secure=cfg.envector.secure, + auto_key_setup=False, + agent_id=agent_id, + agent_dek=agent_dek, + ) + + print("OK") + print(f" index : {index_name}") + print(f" key_id : {key_id}") + print(f" endpoint: {ev_endpoint}") + + async def teardown(self) -> None: + if self._vault is not None: + await self._vault.close() + + # ── internal helpers ─────────────────────────────────────────────────────── + + def _warmup_label(self, run_i: int) -> str: + return "warmup" if run_i < self.warmup else f"run {run_i - self.warmup + 1}" + + def _valid_runs(self, all_timings: list[list[float]]) -> list[list[float]]: + """Drop warmup runs, return effective samples.""" + return all_timings[self.warmup:] + + async def _single_capture_phases( + self, text: str, title: str, domain: str + ) -> dict[str, float]: + """ + Run one capture and return per-phase latencies (ms). + + Phases: + embed — embed the reusable_insight locally + score — FHE-encrypted similarity search on enVector Cloud + vault_topk — Vault TopK decrypt (gRPC) + insert — FHE encrypt + insert into enVector Cloud + total — wall clock for the whole call + """ + reusable_insight = text[:120] + + total_start = time.perf_counter() + + # [1] Embed + with _Timer() as t_embed: + vec = self._embedding.embed_single(reusable_insight) + embed_ms = t_embed.elapsed_ms + + # [2] Novelty score (encrypted similarity search) + with _Timer() as t_score: + score_res = self._ev_client.score(self._index_name, vec) + score_ms = t_score.elapsed_ms + + # [3] Vault TopK decrypt + vault_ms = 0.0 + blobs = score_res.get("encrypted_blobs", []) if score_res.get("ok") else [] + if blobs: + with _Timer() as t_vault: + await self._vault.decrypt_search_results(blobs[0], top_k=3) + vault_ms = t_vault.elapsed_ms + + # [4] Insert (embed internally + FHE encrypt + network) + metadata = [{ + "id": f"bench-{domain}-{int(time.time()*1000)}", + "title": title, + "domain": domain, + "status": "accepted", + "reusable_insight": reusable_insight, + "payload": {"text": text}, + "why": {"certainty": "supported"}, + }] + with _Timer() as t_insert: + self._ev_client.insert_with_text( + index_name=self._index_name, + texts=[reusable_insight], + embedding_service=self._embedding, + metadata=metadata, + ) + insert_ms = t_insert.elapsed_ms + + total_ms = (time.perf_counter() - total_start) * 1000.0 + + return { + "embed": embed_ms, + "score": score_ms, + "vault_topk": vault_ms, + "insert": insert_ms, + "total": total_ms, + } + + async def _single_recall_phases( + self, query: str, topk: int + ) -> dict[str, float]: + """ + Run one recall and return per-phase latencies (ms). + + Phases: + embed — embed the query locally + score — FHE-encrypted similarity search + vault_topk — Vault TopK decrypt + remind — metadata retrieval from enVector + total + """ + total_start = time.perf_counter() + + # [1] Embed query + with _Timer() as t_embed: + vec = self._embedding.embed_single(query) + embed_ms = t_embed.elapsed_ms + + # [2] Encrypted search + with _Timer() as t_score: + score_res = self._ev_client.score(self._index_name, vec) + score_ms = t_score.elapsed_ms + + # [3] Vault TopK decrypt + vault_ms = 0.0 + remind_ms = 0.0 + blobs = score_res.get("encrypted_blobs", []) if score_res.get("ok") else [] + if blobs: + with _Timer() as t_vault: + vault_res = await self._vault.decrypt_search_results(blobs[0], top_k=topk) + vault_ms = t_vault.elapsed_ms + + # [4] Metadata retrieval + if vault_res.ok and vault_res.results: + with _Timer() as t_remind: + self._ev_client.remind( + self._index_name, + vault_res.results, + output_fields=["metadata"], + ) + remind_ms = t_remind.elapsed_ms + + total_ms = (time.perf_counter() - total_start) * 1000.0 + + return { + "embed": embed_ms, + "score": score_ms, + "vault_topk": vault_ms, + "remind": remind_ms, + "total": total_ms, + } + + def _build_phase_list( + self, + phase_names: list[str], + all_timings: list[dict[str, float]], + ) -> list[PhaseLatency]: + """Convert list of per-run phase dicts → list of PhaseLatency.""" + valid = all_timings[self.warmup:] + phases = [] + for name in phase_names: + samples = [t[name] for t in valid if name in t] + phases.append(PhaseLatency(name=name, samples_ms=samples)) + return phases + + # ── scenario runners ─────────────────────────────────────────────────────── + + async def run_capture_scenario(self, scenario: dict) -> LatencyScenarioResult: + sid = scenario["id"] + text = scenario["text"] + title = scenario["title"] + domain = scenario["domain"] + meta = scenario.get("metadata", {}) + + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_capture_phases(text, title, domain) + all_timings.append(t) + except Exception as e: + print(f"\n ERROR on {label}: {e}") + return LatencyScenarioResult( + scenario_id=sid, + feature="capture", + metadata=meta, + error=str(e), + ) + + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "insert", "total"], + all_timings, + ) + return LatencyScenarioResult( + scenario_id=sid, + feature="capture", + phases=phases, + metadata={**meta, "runs": self.runs - self.warmup}, + ) + + async def run_capture_duplicate(self) -> LatencyScenarioResult: + """T4: capture the same text twice — second should hit near-duplicate path.""" + sid = "T4_duplicate" + text = _SHORT_TEXT + title = "PostgreSQL chosen as primary DB (duplicate)" + meta = {"label": "duplicate input — tests novelty near-duplicate path"} + + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + + # First insert to ensure the record exists + try: + await self._single_capture_phases(text, title, "architecture") + except Exception: + pass + + # Now measure the duplicate path + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_capture_phases(text, title, "architecture") + all_timings.append(t) + except Exception as e: + print(f"\n ERROR: {e}") + return LatencyScenarioResult( + scenario_id=sid, feature="capture", metadata=meta, error=str(e) + ) + + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "insert", "total"], + all_timings, + ) + return LatencyScenarioResult( + scenario_id=sid, + feature="capture", + phases=phases, + metadata={**meta, "runs": self.runs - self.warmup}, + ) + + async def run_recall_scenario(self, scenario: dict) -> LatencyScenarioResult: + sid = scenario["id"] + query = scenario["query"] + topk = scenario.get("topk", 5) + meta = scenario.get("metadata", {}) + + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_recall_phases(query, topk) + all_timings.append(t) + except Exception as e: + print(f"\n ERROR: {e}") + return LatencyScenarioResult( + scenario_id=sid, feature="recall", metadata=meta, error=str(e) + ) + + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "remind", "total"], + all_timings, + ) + return LatencyScenarioResult( + scenario_id=sid, + feature="recall", + phases=phases, + metadata={**meta, "topk": topk, "runs": self.runs - self.warmup}, + ) + + async def run_recall_topk_scaling(self) -> list[LatencyScenarioResult]: + """T7: measure recall latency at varying topk values.""" + results = [] + query = "architecture decisions" + + for topk in RECALL_TOPK_VARIANTS: + sid = f"T7_topk_{topk}" + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + runs = max(self.warmup + 3, min(self.runs, self.warmup + 5)) + + for i in range(runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_recall_phases(query, topk) + all_timings.append(t) + except Exception as e: + print(f"\n ERROR: {e}") + results.append(LatencyScenarioResult( + scenario_id=sid, feature="recall", + metadata={"topk": topk, "label": f"topk scaling topk={topk}"}, + error=str(e), + )) + break + else: + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "remind", "total"], + all_timings, + ) + results.append(LatencyScenarioResult( + scenario_id=sid, + feature="recall", + phases=phases, + metadata={"topk": topk, "label": f"topk scaling topk={topk}", + "runs": runs - self.warmup}, + )) + return results + + async def run_batch_capture_scaling(self) -> list[LatencyScenarioResult]: + """T8: measure batch_capture latency at varying batch sizes.""" + results = [] + + for bs in BATCH_SIZES: + sid = f"T8_batch_{bs}" + print(f" [{sid}] ", end="", flush=True) + runs = max(self.warmup + 1, min(3 + self.warmup, self.runs)) + + total_samples: list[float] = [] + per_item_samples: list[float] = [] + error: Optional[str] = None + + # Build bs items + items_texts = [ + f"Decision {i+1}: chose option A over B for batch benchmark (batch_size={bs})" + for i in range(bs) + ] + + for i in range(runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t_start = time.perf_counter() + for text in items_texts: + vec = self._embedding.embed_single(text[:120]) + self._ev_client.score(self._index_name, vec) + # skip vault+insert to keep test fast — measures embed+score overhead + t_total = (time.perf_counter() - t_start) * 1000.0 + total_samples.append(t_total) + per_item_samples.append(t_total / bs) + except Exception as e: + error = str(e) + print(f"\n ERROR: {e}") + break + + if error: + results.append(LatencyScenarioResult( + scenario_id=sid, feature="batch_capture", + metadata={"batch_size": bs}, error=error, + )) + continue + + print("done") + valid_total = total_samples[self.warmup:] + valid_per = per_item_samples[self.warmup:] + phases = [ + PhaseLatency(name="total_batch", samples_ms=valid_total), + PhaseLatency(name="per_item", samples_ms=valid_per), + ] + results.append(LatencyScenarioResult( + scenario_id=sid, + feature="batch_capture", + phases=phases, + metadata={ + "batch_size": bs, + "label": f"batch size {bs} (embed+score per item)", + "runs": runs - self.warmup, + }, + )) + return results + + async def run_vault_status(self) -> LatencyScenarioResult: + """T9: health check latency.""" + sid = "T9_vault_status" + print(f" [{sid}] ", end="", flush=True) + samples: list[float] = [] + + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + with _Timer() as t: + await self._vault.health_check() + samples.append(t.elapsed_ms) + + print("done") + valid = samples[self.warmup:] + phases = [PhaseLatency(name="vault_health_check", samples_ms=valid)] + return LatencyScenarioResult( + scenario_id=sid, + feature="vault_status", + phases=phases, + metadata={"label": "Vault gRPC health check", "runs": len(valid)}, + ) + + # ── network baseline ─────────────────────────────────────────────────────── + + def _measure_network_rtt(self) -> str: + """ping envector endpoint, return mean RTT string.""" + host = (self._config.envector.endpoint or "").split(":")[0] + if not host: + return "unknown" + try: + out = subprocess.check_output( + ["ping", "-c", "5", host], + stderr=subprocess.DEVNULL, + timeout=10, + ).decode() + for line in out.splitlines(): + if "avg" in line or "rtt" in line: + # e.g. "round-trip min/avg/max/stddev = 12.3/14.5/18.2/2.1 ms" + parts = line.split("=")[-1].strip().split("/") + if len(parts) >= 2: + return f"{parts[1]} ms (avg RTT)" + except Exception: + pass + return "unknown" + + # ── orchestration ────────────────────────────────────────────────────────── + + async def run( + self, + feature_filter: Optional[str] = None, + ) -> LatencyBenchReport: + report = LatencyBenchReport() + + # Environment metadata + rtt = self._measure_network_rtt() + cfg = self._config + report.env = { + "date": __import__("datetime").date.today().isoformat(), + "envector_endpoint": cfg.envector.endpoint, + "vault_endpoint": cfg.vault.endpoint, + "embedding_model": cfg.embedding.model, + "embedding_mode": cfg.embedding.mode, + "index_name": self._index_name, + "key_id": self._key_id, + "network_rtt": rtt, + "runs_per_scenario": self.runs - self.warmup, + "warmup_runs": self.warmup, + } + + run_all = feature_filter is None + run_capture = run_all or feature_filter == "capture" + run_recall = run_all or feature_filter == "recall" + run_batch = run_all or feature_filter == "batch_capture" + run_vault = run_all or feature_filter == "vault_status" + + # ── capture ──────────────────────────────────────────────────────── + if run_capture: + print("\n[capture]") + for sc in SCENARIOS_CAPTURE: + r = await self.run_capture_scenario(sc) + report.add(r) + r = await self.run_capture_duplicate() + report.add(r) + + # ── recall ───────────────────────────────────────────────────────── + if run_recall: + print("\n[recall]") + for sc in SCENARIOS_RECALL: + r = await self.run_recall_scenario(sc) + report.add(r) + for r in await self.run_recall_topk_scaling(): + report.add(r) + + # ── batch_capture ────────────────────────────────────────────────── + if run_batch: + print("\n[batch_capture]") + for r in await self.run_batch_capture_scaling(): + report.add(r) + + # ── vault_status ─────────────────────────────────────────────────── + if run_vault: + print("\n[vault_status]") + r = await self.run_vault_status() + report.add(r) + + return report + + +# ── CLI ──────────────────────────────────────────────────────────────────────── + +def _print_summary(report: LatencyBenchReport) -> None: + print("\n" + "=" * 64) + print(" rune latency benchmark — summary") + print("=" * 64) + + for s in report.scenarios: + if s.error: + print(f" [FAIL] {s.scenario_id}: {s.error}") + continue + total_phase = next((p for p in s.phases if p.name == "total"), None) + if total_phase and total_phase.samples_ms: + print( + f" {s.scenario_id:<30} " + f"p50={total_phase.p50:7.1f}ms " + f"p95={total_phase.p95:7.1f}ms " + f"n={total_phase.n}" + ) + else: + for p in s.phases: + print( + f" {s.scenario_id}/{p.name:<26} " + f"p50={p.p50:7.1f}ms " + f"p95={p.p95:7.1f}ms " + f"n={p.n}" + ) + print("=" * 64 + "\n") + + +async def _main(args: argparse.Namespace) -> None: + bench = LatencyBenchmark(runs=args.runs, warmup=args.warmup) + + print("\nSetting up …") + await bench.setup() + + print(f"\nRunning benchmark (runs={args.runs - args.warmup} effective, warmup={args.warmup}) …") + report = await bench.run(feature_filter=args.feature) + + await bench.teardown() + + _print_summary(report) + + if args.report: + report_path = Path(args.report) + if args.format == "json": + saved = report.save_json(report_path) + else: + saved = report.save_markdown(report_path) + print(f"Report saved → {saved}") + else: + print(report.to_markdown()) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Rune × envector-msa-1.4.0 latency benchmark" + ) + parser.add_argument( + "--feature", + choices=["capture", "recall", "batch_capture", "vault_status"], + default=None, + help="Run only this feature (default: all)", + ) + parser.add_argument( + "--runs", + type=int, + default=10, + help="Total runs per scenario including warmup (default: 10)", + ) + parser.add_argument( + "--warmup", + type=int, + default=2, + help="Warmup runs to discard (default: 2)", + ) + parser.add_argument( + "--report", + default=None, + help="Path to save the report (default: print to stdout)", + ) + parser.add_argument( + "--format", + choices=["md", "json"], + default="md", + help="Report format (default: md)", + ) + args = parser.parse_args() + + if args.warmup >= args.runs: + parser.error(f"--warmup ({args.warmup}) must be < --runs ({args.runs})") + + asyncio.run(_main(args)) + + +if __name__ == "__main__": + main() diff --git a/mcp/adapter/envector_sdk.py b/mcp/adapter/envector_sdk.py index d9fc7463..495ffbb1 100644 --- a/mcp/adapter/envector_sdk.py +++ b/mcp/adapter/envector_sdk.py @@ -1,6 +1,6 @@ # Summary of file: enVector SDK Adapter(enVector APIs Caller) -from typing import Union, List, Dict, Any +from typing import Union, List, Dict, Any, Optional import base64 import json import logging @@ -111,8 +111,9 @@ def __init__( key_id: str, key_path: str, eval_mode: str, - query_encryption: bool, + query_encryption: Union[str, bool, None], access_token: str = None, + secure: Optional[bool] = None, auto_key_setup: bool = True, agent_id: str = None, agent_dek: bytes = None, @@ -146,10 +147,23 @@ def __init__( "eval_mode": eval_mode, "auto_key_setup": auto_key_setup, "access_token": access_token, + "secure": secure, } + normalized_query_encryption = self._normalize_query_encryption(query_encryption) + self._init_params["query_encryption"] = normalized_query_encryption + ev.init(**self._init_params) + @staticmethod + def _normalize_query_encryption(value: Union[str, bool, None]) -> str: + """Normalize Rune's legacy bool flag to pyenvector 1.4 string values.""" + if value is None: + return "plain" + if isinstance(value, bool): + return "cipher" if value else "plain" + return value + #--------------- Get Index List --------------# def call_get_index_list(self) -> Dict[str, Any]: """ diff --git a/mcp/adapter/vault_client.py b/mcp/adapter/vault_client.py index e7a1ce75..b305b555 100644 --- a/mcp/adapter/vault_client.py +++ b/mcp/adapter/vault_client.py @@ -30,7 +30,7 @@ from .vault_proto import vault_service_pb2 as pb2 from .vault_proto import vault_service_pb2_grpc as pb2_grpc -MAX_MESSAGE_LENGTH = 256 * 1024 * 1024 # 256 MB (EvalKey can be tens of MB) +MAX_MESSAGE_LENGTH = 2000 * 1024 * 1024 # ~1.95 GB (kept under INT32_MAX; EvalKey in pyenvector >= 1.4.0 reaches ~1.2 GB) @dataclass @@ -82,6 +82,7 @@ def __init__( vault_endpoint: str, vault_token: str, timeout: float = 30.0, + public_key_timeout: float = 90.0, ca_cert: Optional[str] = None, tls_disable: bool = False, ): @@ -93,7 +94,11 @@ def __init__( "host:port" (direct), "tcp://host:port", or legacy "http://host:50080/mcp". RUNEVAULT_GRPC_TARGET overrides. vault_token: Authentication token for Vault - timeout: Request timeout in seconds + timeout: Request timeout in seconds for standard RPCs. + public_key_timeout: Deadline for GetPublicKey, which streams + the multi-GiB EvalKey bundle (pyenvector 1.4 EvalKey is + ~1.1 GiB). Measured baseline ~38s on a 30 MiB/s link; + default tolerates down to ~12.5 MiB/s networks. ca_cert: Path to CA certificate PEM file for self-signed certs. None or empty string uses system CA bundle. tls_disable: If True, use insecure plaintext channel (dev only). @@ -101,6 +106,7 @@ def __init__( self.vault_endpoint = vault_endpoint.rstrip("/") self.vault_token = vault_token self.timeout = timeout + self.public_key_timeout = public_key_timeout self._ca_cert = ca_cert self._tls_disable = tls_disable @@ -203,7 +209,7 @@ async def get_public_key(self) -> dict: try: request = pb2.GetPublicKeyRequest(token=self.vault_token) response = await self._stub.GetPublicKey( - request, timeout=self.timeout + request, timeout=self.public_key_timeout ) if response.error: raise VaultError(f"GetPublicKey failed: {response.error}") diff --git a/mcp/server/server.py b/mcp/server/server.py index 8e78d48d..fa80e47f 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -41,6 +41,34 @@ def filter(self, record): logger.addFilter(_SensitiveFilter()) + +# Optional file log for diagnosing background-thread issues that Claude +# Code's MCP console doesn't always capture cleanly. Off by default. +# RUNE_MCP_DEBUG_LOG=1 -> INFO level (lifecycle + errors) +# RUNE_MCP_DEBUG_LOG=debug -> DEBUG level (verbose, includes httpx traces) +if os.environ.get("RUNE_MCP_DEBUG_LOG"): + try: + from logging.handlers import RotatingFileHandler + _debug_log = os.path.expanduser("~/.rune/mcp-server.log") + os.makedirs(os.path.dirname(_debug_log), exist_ok=True) + _fh = RotatingFileHandler(_debug_log, maxBytes=5_000_000, backupCount=3) + _level = ( + logging.DEBUG + if os.environ["RUNE_MCP_DEBUG_LOG"].lower() == "debug" + else logging.INFO + ) + _fh.setLevel(_level) + _fh.setFormatter(logging.Formatter( + "%(asctime)s [%(threadName)s] %(levelname)s %(name)s: %(message)s" + )) + _fh.addFilter(_SensitiveFilter()) + _root = logging.getLogger() + _root.addHandler(_fh) + if _root.level == 0 or _root.level > _level: + _root.setLevel(_level) + except Exception: + pass + from pydantic import Field # Add parent directory (rune/mcp/) to sys.path so `from adapter import ...` works @@ -212,7 +240,11 @@ async def _async_fetch_keys_from_vault( tls_disable: If True, use insecure plaintext channel. Returns: - tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, envector_endpoint, envector_api_key) + tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, + envector_endpoint, envector_api_key, envector_secure) + envector_secure is None when Vault did not provide the field + (older Vault servers); the client should leave it untouched and + fall back to the SDK default in that case. """ client = VaultClient( vault_endpoint=vault_endpoint, @@ -230,6 +262,7 @@ async def _async_fetch_keys_from_vault( vault_agent_dek_b64 = bundle.pop("agent_dek", None) vault_envector_endpoint = bundle.pop("envector_endpoint", None) vault_envector_api_key = bundle.pop("envector_api_key", None) + vault_envector_secure = bundle.pop("envector_secure", None) if vault_index_name: logger.info(f"Vault provided index_name: {vault_index_name}") @@ -237,7 +270,7 @@ async def _async_fetch_keys_from_vault( logger.info(f"Vault provided key_id: {vault_key_id}") else: logger.warning("Vault did not provide key_id — key directory cannot be determined") - return False, vault_index_name, None, None, None, None, None + return False, vault_index_name, None, None, None, None, None, None if vault_agent_id: logger.info(f"Vault provided agent_id: {vault_agent_id}") @@ -249,10 +282,10 @@ async def _async_fetch_keys_from_vault( agent_dek_bytes = base64.b64decode(vault_agent_dek_b64) except (base64.binascii.Error, ValueError) as e: logger.error(f"Failed to decode agent_dek from Vault (invalid base64): {e}") - return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None + return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None, None if len(agent_dek_bytes) != 32: logger.error(f"agent_dek has invalid length {len(agent_dek_bytes)} bytes (expected 32 for AES-256)") - return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None + return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None, None # Save keys under key_base_path// with restrictive permissions key_dir = os.path.join(key_base_path, vault_key_id) @@ -265,11 +298,14 @@ async def _async_fetch_keys_from_vault( f.write(key_content) logger.info(f"Saved {filename} to {filepath}") - return True, vault_index_name, vault_key_id, vault_agent_id, agent_dek_bytes, vault_envector_endpoint, vault_envector_api_key + return ( + True, vault_index_name, vault_key_id, vault_agent_id, agent_dek_bytes, + vault_envector_endpoint, vault_envector_api_key, vault_envector_secure, + ) except Exception as e: logger.error(f"Failed to fetch keys from Vault: {e}") - return False, None, None, None, None, None, None + return False, None, None, None, None, None, None, None finally: await client.close() @@ -293,10 +329,11 @@ def fetch_keys_from_vault( tls_disable: If True, use insecure plaintext channel. Returns: - tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, envector_endpoint, envector_api_key) + tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, + envector_endpoint, envector_api_key, envector_secure) """ import asyncio - _fail = (False, None, None, None, None, None, None) + _fail = (False, None, None, None, None, None, None, None) try: asyncio.get_running_loop() @@ -308,11 +345,21 @@ def fetch_keys_from_vault( asyncio.run, _async_fetch_keys_from_vault(vault_endpoint, vault_token, key_base_path, ca_cert, tls_disable), ) + # gRPC GetPublicKey has a 90s deadline; add buffer for thread/ + # asyncio/TLS overhead so a slow-but-eventually-OK fetch returns + # its real result. The `with` block already waits on shutdown, + # so a longer timeout costs nothing on the success path. try: - return future.result(timeout=30) + return future.result(timeout=120) except concurrent.futures.TimeoutError: - logger.error("Vault key fetch timed out after 30 seconds") - return _fail + logger.warning( + "Vault key fetch exceeded 120s — waiting briefly for in-flight call" + ) + try: + return future.result(timeout=30) + except Exception as e: + logger.error(f"Vault key fetch ultimately failed: {e}") + return _fail except Exception as e: logger.error(f"Vault key fetch failed in thread: {e}") return _fail @@ -1558,7 +1605,10 @@ def _init_pipelines(self) -> Dict[str, Any]: # Always re-fetch from Vault on reload to pick up endpoint/index changes if rune_config.vault.endpoint and rune_config.vault.token: logger.info("Fetching keys from Vault...") - success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, vault_ev_endpoint, vault_ev_api_key = fetch_keys_from_vault( + ( + success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, + vault_ev_endpoint, vault_ev_api_key, vault_ev_secure, + ) = fetch_keys_from_vault( rune_config.vault.endpoint, rune_config.vault.token, key_path, @@ -1579,9 +1629,11 @@ def _init_pipelines(self) -> Dict[str, Any]: self._envector_endpoint = vault_ev_endpoint if vault_ev_api_key: self._envector_api_key = vault_ev_api_key + if vault_ev_secure is not None: + rune_config.envector.secure = vault_ev_secure # Cache enVector credentials to config.json - if vault_ev_endpoint or vault_ev_api_key: + if vault_ev_endpoint or vault_ev_api_key or vault_ev_secure is not None: from agents.common.config import save_config as save_rune_config if vault_ev_endpoint: rune_config.envector.endpoint = vault_ev_endpoint @@ -1589,6 +1641,10 @@ def _init_pipelines(self) -> Dict[str, Any]: rune_config.envector.api_key = vault_ev_api_key save_rune_config(rune_config) logger.info("Cached enVector credentials to config.json") + + if not vault_ev_endpoint or not vault_ev_api_key: + logger.error("Vault bundle missing enVector credentials. Contact your Vault administrator.") + _set_dormant_with_reason("envector_not_provisioned") else: result["errors"].append("Failed to fetch keys from Vault") logger.error("Failed to fetch keys from Vault — capture/search will fail") @@ -1622,6 +1678,7 @@ def _init_pipelines(self) -> Dict[str, Any]: key_path=key_path, key_id=key_id, access_token=self._envector_api_key or "", + secure=rune_config.envector.secure, auto_key_setup=False, agent_id=self._agent_id, agent_dek=self._agent_dek, @@ -1633,9 +1690,10 @@ def _init_pipelines(self) -> Dict[str, Any]: address=self._envector_endpoint or "", key_id=key_id, key_path=key_path, - eval_mode="rmp", - query_encryption=False, + eval_mode=os.getenv("ENVECTOR_EVAL_MODE", "mm"), + query_encryption="plain", access_token=self._envector_api_key or "", + secure=rune_config.envector.secure, auto_key_setup=False, agent_id=self._agent_id, agent_dek=self._agent_dek, @@ -1830,8 +1888,8 @@ def run(self) -> None: ) parser.add_argument( "--envector-eval-mode", - default=os.getenv("ENVECTOR_EVAL_MODE", "rmp"), - help="enVector evaluation mode (e.g., 'rmp', 'mm').", + default=os.getenv("ENVECTOR_EVAL_MODE", "mm"), + help="enVector evaluation mode (e.g., 'mm', 'rmp').", ) parser.add_argument( "--encrypted-query", @@ -1854,8 +1912,23 @@ def run(self) -> None: ENVECTOR_EVAL_MODE = args.envector_eval_mode ENCRYPTED_QUERY = args.encrypted_query + def _parse_optional_bool(value): + if value is None or value == "": + return None + if isinstance(value, bool): + return value + lowered = str(value).strip().lower() + if lowered in ("true", "1", "yes", "y", "on"): + return True + if lowered in ("false", "0", "no", "n", "off"): + return False + raise ValueError(f"invalid boolean value: {value!r}") + + ENVECTOR_SECURE = _parse_optional_bool(os.getenv("ENVECTOR_SECURE")) + # ── Load ~/.rune/config.json if ENVECTOR_CONFIG is set ── _vault_cfg = {} # populated from config file if available + _envector_cfg = {} _config_path = os.getenv("ENVECTOR_CONFIG") if _config_path: _config_path = os.path.expanduser(_config_path) @@ -1864,6 +1937,13 @@ def run(self) -> None: with open(_config_path) as _cf: _rune_config = json.load(_cf) _vault_cfg = _rune_config.get("vault", {}) + _envector_cfg = _rune_config.get("envector", {}) + if not ENVECTOR_ENDPOINT and _envector_cfg.get("endpoint"): + ENVECTOR_ENDPOINT = _envector_cfg["endpoint"] + if not ENVECTOR_API_KEY and _envector_cfg.get("api_key"): + ENVECTOR_API_KEY = _envector_cfg["api_key"] + if ENVECTOR_SECURE is None: + ENVECTOR_SECURE = _parse_optional_bool(_envector_cfg.get("secure")) if not os.getenv("RUNEVAULT_ENDPOINT") and (_vault_cfg.get("endpoint") or _vault_cfg.get("url")): os.environ["RUNEVAULT_ENDPOINT"] = _vault_cfg.get("endpoint") or _vault_cfg["url"] if not os.getenv("RUNEVAULT_TOKEN") and _vault_cfg.get("token"): @@ -1886,44 +1966,22 @@ def run(self) -> None: VAULT_TLS_DISABLE = bool(_vault_cfg.get("tls_disable", False)) VAULT_CONFIGURED = bool(RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN) - VAULT_KEYS_LOADED = False VAULT_INDEX_NAME = None AGENT_ID = None AGENT_DEK = None if RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN: # When Vault is configured (Rune plugin mode), use canonical key path. - # key_id is discovered from Vault — no hardcoded default. + # key_id is discovered from Vault by the background pipeline init thread — + # no synchronous fetch here, so MCP transport is ready in ~1s instead of + # blocking on a multi-MB EvalKey stream that can exceed Claude's startup + # timeout. ENVECTOR_KEY_PATH = MCPServerApp.DEFAULT_KEY_PATH - - logger.info(f"Vault configured — fetching public keys from: {RUNEVAULT_ENDPOINT}") - success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, vault_ev_endpoint, vault_ev_api_key = fetch_keys_from_vault( - RUNEVAULT_ENDPOINT, RUNEVAULT_TOKEN, - ENVECTOR_KEY_PATH, - ca_cert=VAULT_CA_CERT, - tls_disable=VAULT_TLS_DISABLE, + AUTO_KEY_SETUP = False + ENVECTOR_KEY_ID = None # populated by _init_pipelines from Vault bundle + logger.info( + f"Vault configured — keys will be fetched in background from: {RUNEVAULT_ENDPOINT}" ) - if success and vault_key_id: - ENVECTOR_KEY_ID = vault_key_id - logger.info(f"Vault provided key_id: {ENVECTOR_KEY_ID}") - AUTO_KEY_SETUP = False - VAULT_KEYS_LOADED = True - VAULT_INDEX_NAME = vault_index - AGENT_ID = vault_agent_id - AGENT_DEK = vault_agent_dek - if vault_ev_endpoint: - ENVECTOR_ENDPOINT = vault_ev_endpoint - logger.info("Using enVector endpoint from Vault bundle") - if vault_ev_api_key: - ENVECTOR_API_KEY = vault_ev_api_key - logger.info("Using enVector API key from Vault bundle") - if not vault_ev_endpoint or not vault_ev_api_key: - logger.error("Vault bundle missing enVector credentials. Contact your Vault administrator.") - _set_dormant_with_reason("envector_not_provisioned") - else: - logger.error("Failed to fetch keys/key_id from Vault. Operations requiring encryption will fail.") - _set_dormant_with_reason("vault_unreachable") - AUTO_KEY_SETUP = False elif RUNEVAULT_ENDPOINT and not RUNEVAULT_TOKEN: logger.warning("Vault endpoint provided but no token specified. Skipping Vault integration.") VAULT_CONFIGURED = True @@ -1931,19 +1989,25 @@ def run(self) -> None: elif not AUTO_KEY_SETUP: logger.info(f"Using externally provided keys from: {ENVECTOR_KEY_PATH}") + # When Vault is configured, defer adapter construction to _init_pipelines + # (which builds it after the background key fetch completes). Constructing + # it here would either block on key files or partially init with empty + # placeholders that get overwritten anyway. envector_adapter = None - try: - envector_adapter = EnVectorSDKAdapter( - address=ENVECTOR_ENDPOINT, - key_id=ENVECTOR_KEY_ID, - key_path=ENVECTOR_KEY_PATH, - eval_mode=ENVECTOR_EVAL_MODE, - query_encryption=ENCRYPTED_QUERY, - access_token=ENVECTOR_API_KEY, - auto_key_setup=AUTO_KEY_SETUP, - ) - except Exception as e: - logger.warning(f"enVector adapter init failed (server will start in degraded mode): {e}") + if not (RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN): + try: + envector_adapter = EnVectorSDKAdapter( + address=ENVECTOR_ENDPOINT, + key_id=ENVECTOR_KEY_ID, + key_path=ENVECTOR_KEY_PATH, + eval_mode=ENVECTOR_EVAL_MODE, + query_encryption=ENCRYPTED_QUERY, + access_token=ENVECTOR_API_KEY, + secure=ENVECTOR_SECURE, + auto_key_setup=AUTO_KEY_SETUP, + ) + except Exception as e: + logger.warning(f"enVector adapter init failed (server will start in degraded mode): {e}") vault_client = None if RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN: @@ -1970,7 +2034,10 @@ def run(self) -> None: agent_dek=AGENT_DEK, ) - # Set enVector credentials from Vault bundle on app instance + # Seed enVector credentials from cached ~/.rune/config.json so that + # vault_status / non-encrypted tools have something to report before the + # background Vault fetch completes. _init_pipelines will overwrite these + # with fresh values from the Vault bundle. if ENVECTOR_ENDPOINT: app._envector_endpoint = ENVECTOR_ENDPOINT if ENVECTOR_API_KEY: diff --git a/scripts/configure-claude-mcp.sh b/scripts/configure-claude-mcp.sh index c262ac51..94ea6a15 100755 --- a/scripts/configure-claude-mcp.sh +++ b/scripts/configure-claude-mcp.sh @@ -2,9 +2,9 @@ set -e # Configure Claude MCP Servers -# Registers Rune's enVector MCP server with Claude Desktop. +# Registers Rune's enVector MCP server with Claude Code and Claude Desktop. # -# Claude Code: Handled natively by plugin.json mcpServers (no action needed) +# Claude Code: Explicit `claude mcp add` registration. # Claude Desktop: JSON deep merge into claude_desktop_config.json PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -24,19 +24,21 @@ print_warn() { CONFIGURED=0 # ============================================================================ -# 1. Claude Code — handled by plugin.json mcpServers (no manual registration needed) +# 1. Claude Code — purge legacy user-scope MCP entries +# +# Claude Code auto-registers the enVector MCP server from the plugin manifest +# (plugin:rune:envector). Older Rune installs added a duplicate user-scope +# `envector` entry via `claude mcp add`, which competes with the plugin-scope +# registration during stdio handshake and causes both to fail to connect. +# This block only removes those legacy entries; registration is owned by the +# plugin manifest. # ============================================================================ -# Claude Code natively reads mcpServers from .claude-plugin/plugin.json, -# which launches bootstrap-mcp.sh via ${CLAUDE_PLUGIN_ROOT}. No `claude mcp add` -# needed — that was causing path conflicts with the plugin system. - -# Remove stale `claude mcp add` entries from previous versions CLAUDE_CMD=$(CLAUDECODE= command -v claude 2>/dev/null || true) if [ -n "$CLAUDE_CMD" ]; then for name in envector rune-vault envector-mcp-server; do CLAUDECODE= "$CLAUDE_CMD" mcp remove --scope user "$name" 2>/dev/null || true done - print_info "Cleaned stale Claude Code MCP entries" + print_info "Cleaned legacy Claude Code user-scope MCP entries (plugin scope owns registration)" fi # ============================================================================ @@ -224,7 +226,7 @@ fi # Summary # ============================================================================ if [ "$CONFIGURED" -eq 0 ]; then - print_warn "No Claude Desktop configuration found — Claude Code uses plugin.json automatically" + print_warn "No Claude Desktop configuration found (Claude Code uses plugin-scope MCP registration)" fi echo ""