From f46cfb886fde68f336d217d663aec5faad0c079a Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Mon, 4 May 2026 16:42:30 +0900 Subject: [PATCH 01/16] feat: add secure (TLS) param and normalize query_encryption for pyenvector 1.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add optional `secure` field to EnVectorConfig, EnVectorClient, SDK adapter, and MCP server; reads from config.json and ENVECTOR_SECURE env var - Normalize query_encryption bool → string ("plain"/"cipher") to match pyenvector 1.4 API Co-Authored-By: Claude Sonnet 4.6 --- agents/common/config.py | 24 ++++++++++++++++++++++++ agents/common/envector_client.py | 6 ++++-- mcp/adapter/envector_sdk.py | 18 ++++++++++++++++-- mcp/server/server.py | 23 ++++++++++++++++++++++- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/agents/common/config.py b/agents/common/config.py index 2be3a1ff..9cb48d7e 100644 --- a/agents/common/config.py +++ b/agents/common/config.py @@ -8,6 +8,7 @@ import json from pathlib import Path from dataclasses import dataclass, field +from typing import Optional # Default config paths CONFIG_DIR = Path.home() / ".rune" @@ -59,6 +60,7 @@ class EnVectorConfig: """enVector Cloud credentials (cached from Vault bundle)""" endpoint: str = "" api_key: str = "" + secure: Optional[bool] = None # None = pyenvector default; API key defaults to TLS @dataclass @@ -146,9 +148,25 @@ def _parse_envector_config(data: dict) -> EnVectorConfig: return EnVectorConfig( endpoint=ev_data.get("endpoint", ""), api_key=ev_data.get("api_key", ""), + secure=_parse_optional_bool(ev_data.get("secure")), ) +def _parse_optional_bool(value) -> Optional[bool]: + """Parse bool-like config values while preserving None as 'use SDK default'.""" + if value is None or value == "": + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ("true", "1", "yes", "y", "on"): + return True + if lowered in ("false", "0", "no", "n", "off"): + return False + raise ValueError(f"invalid boolean value: {value!r}") + + def _parse_llm_config(data: dict) -> LLMConfig: """Parse LLM configuration with backward-compatible migration. @@ -252,6 +270,11 @@ def load_config() -> RuneConfig: config.scribe.slack_signing_secret = os.getenv("SLACK_SIGNING_SECRET") if os.getenv("NOTION_SIGNING_SECRET"): config.scribe.notion_signing_secret = os.getenv("NOTION_SIGNING_SECRET") + if os.getenv("ENVECTOR_SECURE"): + try: + config.envector.secure = _parse_optional_bool(os.getenv("ENVECTOR_SECURE")) + except ValueError: + print(f"[Config] Warning: invalid ENVECTOR_SECURE value: {os.getenv('ENVECTOR_SECURE')}") # LLM env var overrides (target config.llm, track env-sourced keys) _env_llm_map = { @@ -318,6 +341,7 @@ def save_config(config: RuneConfig) -> None: "envector": { "endpoint": config.envector.endpoint, "api_key": config.envector.api_key, + "secure": config.envector.secure, }, "embedding": { "mode": config.embedding.mode, diff --git a/agents/common/envector_client.py b/agents/common/envector_client.py index 9afda072..6a070b4a 100644 --- a/agents/common/envector_client.py +++ b/agents/common/envector_client.py @@ -33,6 +33,7 @@ def __init__( key_path: str = "~/.rune/keys", key_id: str = None, access_token: Optional[str] = None, + secure: Optional[bool] = None, auto_key_setup: bool = True, agent_id: Optional[str] = None, agent_dek: Optional[bytes] = None, @@ -53,6 +54,7 @@ def __init__( self._key_path = Path(key_path).expanduser() self._key_id = key_id self._access_token = access_token + self._secure = secure self._auto_key_setup = auto_key_setup self._agent_id = agent_id self._agent_dek = agent_dek @@ -75,8 +77,9 @@ def _ensure_initialized(self) -> None: key_id=self._key_id, key_path=str(self._key_path), eval_mode="rmp", - query_encryption=False, # Plain queries for simplicity + query_encryption="plain", # Plain queries for simplicity access_token=self._access_token, + secure=self._secure, auto_key_setup=self._auto_key_setup, agent_id=self._agent_id, agent_dek=self._agent_dek, @@ -217,4 +220,3 @@ def remind( indices=indices, output_fields=output_fields, ) - diff --git a/mcp/adapter/envector_sdk.py b/mcp/adapter/envector_sdk.py index d9fc7463..495ffbb1 100644 --- a/mcp/adapter/envector_sdk.py +++ b/mcp/adapter/envector_sdk.py @@ -1,6 +1,6 @@ # Summary of file: enVector SDK Adapter(enVector APIs Caller) -from typing import Union, List, Dict, Any +from typing import Union, List, Dict, Any, Optional import base64 import json import logging @@ -111,8 +111,9 @@ def __init__( key_id: str, key_path: str, eval_mode: str, - query_encryption: bool, + query_encryption: Union[str, bool, None], access_token: str = None, + secure: Optional[bool] = None, auto_key_setup: bool = True, agent_id: str = None, agent_dek: bytes = None, @@ -146,10 +147,23 @@ def __init__( "eval_mode": eval_mode, "auto_key_setup": auto_key_setup, "access_token": access_token, + "secure": secure, } + normalized_query_encryption = self._normalize_query_encryption(query_encryption) + self._init_params["query_encryption"] = normalized_query_encryption + ev.init(**self._init_params) + @staticmethod + def _normalize_query_encryption(value: Union[str, bool, None]) -> str: + """Normalize Rune's legacy bool flag to pyenvector 1.4 string values.""" + if value is None: + return "plain" + if isinstance(value, bool): + return "cipher" if value else "plain" + return value + #--------------- Get Index List --------------# def call_get_index_list(self) -> Dict[str, Any]: """ diff --git a/mcp/server/server.py b/mcp/server/server.py index 8e78d48d..a1eb9253 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -1622,6 +1622,7 @@ def _init_pipelines(self) -> Dict[str, Any]: key_path=key_path, key_id=key_id, access_token=self._envector_api_key or "", + secure=rune_config.envector.secure, auto_key_setup=False, agent_id=self._agent_id, agent_dek=self._agent_dek, @@ -1634,8 +1635,9 @@ def _init_pipelines(self) -> Dict[str, Any]: key_id=key_id, key_path=key_path, eval_mode="rmp", - query_encryption=False, + query_encryption="plain", access_token=self._envector_api_key or "", + secure=rune_config.envector.secure, auto_key_setup=False, agent_id=self._agent_id, agent_dek=self._agent_dek, @@ -1854,8 +1856,23 @@ def run(self) -> None: ENVECTOR_EVAL_MODE = args.envector_eval_mode ENCRYPTED_QUERY = args.encrypted_query + def _parse_optional_bool(value): + if value is None or value == "": + return None + if isinstance(value, bool): + return value + lowered = str(value).strip().lower() + if lowered in ("true", "1", "yes", "y", "on"): + return True + if lowered in ("false", "0", "no", "n", "off"): + return False + raise ValueError(f"invalid boolean value: {value!r}") + + ENVECTOR_SECURE = _parse_optional_bool(os.getenv("ENVECTOR_SECURE")) + # ── Load ~/.rune/config.json if ENVECTOR_CONFIG is set ── _vault_cfg = {} # populated from config file if available + _envector_cfg = {} _config_path = os.getenv("ENVECTOR_CONFIG") if _config_path: _config_path = os.path.expanduser(_config_path) @@ -1864,6 +1881,9 @@ def run(self) -> None: with open(_config_path) as _cf: _rune_config = json.load(_cf) _vault_cfg = _rune_config.get("vault", {}) + _envector_cfg = _rune_config.get("envector", {}) + if ENVECTOR_SECURE is None: + ENVECTOR_SECURE = _parse_optional_bool(_envector_cfg.get("secure")) if not os.getenv("RUNEVAULT_ENDPOINT") and (_vault_cfg.get("endpoint") or _vault_cfg.get("url")): os.environ["RUNEVAULT_ENDPOINT"] = _vault_cfg.get("endpoint") or _vault_cfg["url"] if not os.getenv("RUNEVAULT_TOKEN") and _vault_cfg.get("token"): @@ -1940,6 +1960,7 @@ def run(self) -> None: eval_mode=ENVECTOR_EVAL_MODE, query_encryption=ENCRYPTED_QUERY, access_token=ENVECTOR_API_KEY, + secure=ENVECTOR_SECURE, auto_key_setup=AUTO_KEY_SETUP, ) except Exception as e: From 622728926be58ef47617ae1a1157ba4d0887ed86 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Mon, 4 May 2026 17:43:27 +0900 Subject: [PATCH 02/16] fix: register envector MCP via explicit `claude mcp add` in Claude Code Switches Claude Code MCP registration from relying on plugin.json mcpServers auto-discovery to explicit `claude mcp add --scope user` invocation, which is more reliable and avoids path-conflict issues with the plugin system. Co-Authored-By: Claude Sonnet 4.6 --- scripts/configure-claude-mcp.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/configure-claude-mcp.sh b/scripts/configure-claude-mcp.sh index c262ac51..1a087c8a 100755 --- a/scripts/configure-claude-mcp.sh +++ b/scripts/configure-claude-mcp.sh @@ -2,9 +2,9 @@ set -e # Configure Claude MCP Servers -# Registers Rune's enVector MCP server with Claude Desktop. +# Registers Rune's enVector MCP server with Claude Code and Claude Desktop. # -# Claude Code: Handled natively by plugin.json mcpServers (no action needed) +# Claude Code: Explicit `claude mcp add` registration. # Claude Desktop: JSON deep merge into claude_desktop_config.json PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -24,19 +24,25 @@ print_warn() { CONFIGURED=0 # ============================================================================ -# 1. Claude Code — handled by plugin.json mcpServers (no manual registration needed) +# 1. Claude Code — explicit `claude mcp add` registration # ============================================================================ -# Claude Code natively reads mcpServers from .claude-plugin/plugin.json, -# which launches bootstrap-mcp.sh via ${CLAUDE_PLUGIN_ROOT}. No `claude mcp add` -# needed — that was causing path conflicts with the plugin system. - -# Remove stale `claude mcp add` entries from previous versions CLAUDE_CMD=$(CLAUDECODE= command -v claude 2>/dev/null || true) if [ -n "$CLAUDE_CMD" ]; then for name in envector rune-vault envector-mcp-server; do CLAUDECODE= "$CLAUDE_CMD" mcp remove --scope user "$name" 2>/dev/null || true done print_info "Cleaned stale Claude Code MCP entries" + + CLAUDECODE= "$CLAUDE_CMD" mcp add --scope user envector -- \ + env \ + ENVECTOR_CONFIG="$HOME/.rune/config.json" \ + ENVECTOR_AUTO_KEY_SETUP=false \ + PYTHONPATH="$PLUGIN_DIR/mcp" \ + "$PLUGIN_DIR/scripts/bootstrap-mcp.sh" >/dev/null + print_info "MCP server registered in Claude Code user scope" + CONFIGURED=$((CONFIGURED + 1)) +else + print_warn "Claude Code CLI not found; skipping Claude Code MCP registration" fi # ============================================================================ @@ -224,7 +230,7 @@ fi # Summary # ============================================================================ if [ "$CONFIGURED" -eq 0 ]; then - print_warn "No Claude Desktop configuration found — Claude Code uses plugin.json automatically" + print_warn "No Claude Code CLI or Claude Desktop configuration found" fi echo "" From 510cc54449500238609c811fd6d5e0ef84844d11 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Mon, 4 May 2026 17:44:00 +0900 Subject: [PATCH 03/16] fix: fall back to config.json for envector endpoint and api_key When ENVECTOR_ENDPOINT or ENVECTOR_API_KEY are not set via environment variables, read them from the envector section of ~/.rune/config.json, consistent with how other config values (secure, vault) are loaded. Co-Authored-By: Claude Sonnet 4.6 --- mcp/server/server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mcp/server/server.py b/mcp/server/server.py index a1eb9253..bb8f54aa 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -1882,6 +1882,10 @@ def _parse_optional_bool(value): _rune_config = json.load(_cf) _vault_cfg = _rune_config.get("vault", {}) _envector_cfg = _rune_config.get("envector", {}) + if not ENVECTOR_ENDPOINT and _envector_cfg.get("endpoint"): + ENVECTOR_ENDPOINT = _envector_cfg["endpoint"] + if not ENVECTOR_API_KEY and _envector_cfg.get("api_key"): + ENVECTOR_API_KEY = _envector_cfg["api_key"] if ENVECTOR_SECURE is None: ENVECTOR_SECURE = _parse_optional_bool(_envector_cfg.get("secure")) if not os.getenv("RUNEVAULT_ENDPOINT") and (_vault_cfg.get("endpoint") or _vault_cfg.get("url")): From be436e84d7ada8a2f1a6b10d3833d3029cf8d478 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Mon, 4 May 2026 18:42:28 +0900 Subject: [PATCH 04/16] feat(benchmark): add latency bench runner for envector-msa-1.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures per-phase latency (embed → FHE score → vault_topk → insert/recall) for capture, recall, batch_capture, and vault_status against live envector-msa-1.4.0. Includes plan doc, v1.4.0 results report, and README updates. Co-Authored-By: Claude Sonnet 4.6 --- benchmark/README.md | 40 + .../latency_bench_plan_envector_v1.4.0.md | 279 ++++++ benchmark/reports/latency_results_v1.4.0.md | 260 ++++++ benchmark/runners/common.py | 140 +++ benchmark/runners/latency_bench.py | 822 ++++++++++++++++++ 5 files changed, 1541 insertions(+) create mode 100644 benchmark/plans/latency_bench_plan_envector_v1.4.0.md create mode 100644 benchmark/reports/latency_results_v1.4.0.md create mode 100644 benchmark/runners/latency_bench.py diff --git a/benchmark/README.md b/benchmark/README.md index db4c1a2b..510cbbf1 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -13,9 +13,12 @@ This means **capture quality is determined by the scribe prompt**, not by Rune's | **scribe_bench** (capture) | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON | | **scribe_bench** (extraction) | Is the extracted JSON well-structured? | Same LLM call, score field coverage and extraction type | | **retriever_bench** | Can stored decisions be retrieved with various queries? | Offline embedding similarity (FHE is transparent to scores) | +| **latency_bench** | How long does each pipeline phase take? | Direct adapter calls against live envector-msa-1.4.0, per-phase timing | ## Running +### Quality benchmarks (scribe / retriever) + ```bash # Default: uses claude CLI (no API key needed) python benchmark/runners/scribe_bench.py @@ -43,6 +46,43 @@ python benchmark/runners/scribe_bench.py --api-key $ANTHROPIC_API_KEY --provider python benchmark/runners/scribe_bench.py --api-key $OPENAI_API_KEY --provider openai --model gpt-4o ``` +### Latency benchmark (envector-msa-1.4.0) + +Requires: Vault running (`tcp://localhost:50051`) and envector-msa-1.4.0 endpoint reachable. + +```bash +# All scenarios (10 runs, 2 warmup) +python benchmark/runners/latency_bench.py + +# Single feature +python benchmark/runners/latency_bench.py --feature capture +python benchmark/runners/latency_bench.py --feature recall +python benchmark/runners/latency_bench.py --feature batch_capture +python benchmark/runners/latency_bench.py --feature vault_status + +# Custom run count (20 total, 3 warmup → 17 effective) +python benchmark/runners/latency_bench.py --runs 20 --warmup 3 + +# Save Markdown report +python benchmark/runners/latency_bench.py \ + --report benchmark/reports/latency_results_v1.4.0.md \ + --format md + +# Save JSON report +python benchmark/runners/latency_bench.py \ + --report benchmark/reports/latency_results_v1.4.0.json \ + --format json +``` + +Pipeline phases measured per feature: + +| Feature | Phases | +|---------|--------| +| `capture` | embed → score (FHE search) → vault_topk → insert → total | +| `recall` | embed → score → vault_topk → remind → total | +| `batch_capture` | total_batch, per_item (embed+score per item) | +| `vault_status` | vault_health_check | + ## Scenarios (104 total) ``` diff --git a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md new file mode 100644 index 00000000..0a3a268f --- /dev/null +++ b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md @@ -0,0 +1,279 @@ +# Rune × envector-msa-1.4.0 Latency Benchmark Plan + +> **[측정 환경]** envector 연결은 `secure=false`(plaintext gRPC)로 측정. Vault(`localhost:50051`)는 TLS 활성 상태. +> plaintext로 진행해도 무방하다고 판단한 이유: gRPC는 HTTP/2 커넥션을 재사용하므로 TLS 핸드셰이크 비용이 warmup 이후 측정값에 포함되지 않으며, 주요 병목이 FHE 연산에 있어 TLS 유무가 결과에 실질적 영향이 낮을 수 있다고 판단. +> 이후 envector TLS 환경(`secure=true`)에서도 동일 시나리오로 벤치마크를 진행할 예정. + +## Context + +기존 benchmark suite는 **capture 품질(정확도)과 recall 품질(hit rate)**을 측정하는 데 집중되어 있다. +이번 작업은 rune의 각 MCP 기능을 실행했을 때 **실제 응답 시간(latency)**을 측정하는 것이 목표다. + +envector-msa-1.4.0은 이전 버전(rmp)에서 크게 변경되었다: +- **KMS**: 독립 서비스로 분리 (Port 50060) +- **Audit**: 독립 서비스로 분리 (Port 50052) +- **pyenvector 1.4.0a5**: `secure` 파라미터 추가, 암호화 최적화 (PPMM cache 1.31–1.40x) +- **gRPC-Gateway**: REST 전사(transcoding) 지원 + +rune의 현재 config는 `envector.endpoint = 146.56.178.130:50050` (원격 클라우드)를 사용하고 있으며, +vault는 `tcp://localhost:50051`에서 로컬 gRPC로 실행된다. + +--- + +## 측정 대상 기능 및 파이프라인 분해 + +### Feature 1: `capture` (MCP tool) + +전체 파이프라인: +``` +[1] 텍스트 → Embedding (로컬 fastembed, Qwen/Qwen3-Embedding-0.6B) +[2] Novelty Check → envector inner_product (FHE 암호화 쿼리 전송) +[3] Vault TopK Decrypt (gRPC localhost:50051) +[4] FHE Encrypt → pyenvector insert_data (envector 저장) +[5] capture_log 기록 (로컬 파일) +──── +Total end-to-end +``` + +### Feature 2: `recall` (MCP tool) + +전체 파이프라인: +``` +[1] 쿼리 → Embedding (로컬) +[2] Encrypted Search → envector inner_product +[3] Vault TopK Decrypt (gRPC) +[4] 메타데이터 조회 (로컬 JSON 파싱) +──── +Total end-to-end +``` + +### Feature 3: `batch_capture` (MCP tool) + +- batch size: 1, 5, 10, 20 +- 총 시간 + item당 평균 latency + +### Feature 4: `vault_status` / `diagnostics` (MCP tool) + +- Vault gRPC 연결 latency +- envector 헬스체크 latency + +--- + +## 구체적 테스트 시나리오 (각 기능별) + +### [T1] Capture — 짧은 영문 텍스트 (baseline) +```json +{ + "input": "We decided to use PostgreSQL as our primary database. Team familiarity and mature ecosystem were the key reasons. Redis considered but rejected due to durability concerns.", + "domain": "architecture", + "expected_insight": "PostgreSQL chosen over Redis for primary DB due to team familiarity and durability" +} +``` +반복: 10회 (첫 2회 warmup 제외) + +### [T2] Capture — 긴 영문 텍스트 (embedding 부하 테스트) +```json +{ + "input": "Full ADR: Context — our monolith hit 10k RPS limits. Considered: (1) horizontal scale with read replicas, (2) CQRS split, (3) microservices decomposition. Trade-offs: read replicas cheapest but doesn't solve write bottleneck; CQRS complex but keeps single codebase; microservices highest ops overhead. Decision: CQRS with event sourcing on order-service first. Rationale: allows independent scaling of read path, event log gives audit trail for compliance (legal requirement from Q2 review). Rollback plan: feature flag, revert within 2 sprints if P99 > 500ms.", + "domain": "architecture", + "label": "long_text_~150_tokens" +} +``` +반복: 10회 + +### [T3] Capture — 한국어 텍스트 (multi-language) +```json +{ + "input": "Redis를 캐시 레이어로 사용하기로 결정했습니다. Memcached도 검토했지만 데이터 구조 지원(Sorted Set, List)이 필요해서 Redis로 확정. TTL은 1시간으로 설정. 담당: 백엔드팀.", + "domain": "architecture", + "label": "korean_text" +} +``` +반복: 10회 + +### [T4] Capture — 중복 입력 (novelty check 동작 확인) +- T1의 입력을 두 번 연속 capture +- 두 번째 호출의 novelty check 결과(near-duplicate) 및 응답 시간 비교 +- 중복 스킵 여부 확인 + +### [T5] Recall — Exact match 쿼리 +```json +{ + "query": "Why did we choose PostgreSQL?", + "topk": 5, + "label": "exact_match" +} +``` +반복: 10회 + +### [T6] Recall — Semantic match 쿼리 (의미 기반) +```json +{ + "query": "데이터베이스 선택 이유", + "topk": 5, + "label": "cross_language_semantic" +} +``` +반복: 10회 + +### [T7] Recall — topk 변화에 따른 latency +```json +{ + "query": "architecture decisions", + "topk_variants": [1, 3, 5, 10], + "label": "topk_scaling" +} +``` +각 topk 당 5회 반복 + +### [T8] Batch Capture — batch size 스케일링 +```json +{ + "batch_sizes": [1, 5, 10, 20], + "inputs": ["decision_{i}: chose option A over B because ..."], + "label": "batch_scaling" +} +``` +각 batch size 당 3회 반복 + +### [T9] Vault Status / Diagnostics 헬스체크 +- `vault_status` 10회 반복 +- `diagnostics` 10회 반복 +- Vault gRPC cold/warm 연결 구분 + +### [T10] Memory 크기별 Recall latency (회귀 테스트) +- 빈 index에서 recall +- 10개 capture 후 recall +- 50개 capture 후 recall +- 각각 5회 반복하여 index 크기가 latency에 미치는 영향 확인 + +--- + +## 측정 방법론 + +### 통계 기준 +- **반복**: 최소 10회 (warmup 2회 제외하여 8회 유효) +- **보고 지표**: p50, p95, p99, mean, min, max (ms 단위) +- **타이머**: `time.perf_counter()` (Python 고정밀 타이머) +- **단계별 측정**: 각 파이프라인 단계를 개별 타이머로 계측 + +### 환경 조건 +- envector-msa-1.4.0: 원격 클라우드 (`146.56.178.130:50050`, `secure=false`) +- Vault: 로컬 (`tcp://localhost:50051`) +- Embedding model: `Qwen/Qwen3-Embedding-0.6B` (로컬) +- 네트워크 조건: 고정 IP, 테스트 전 `ping 146.56.178.130` 기준 RTT 기록 + +--- + +## 구현할 파일 + +### 1. `benchmark/runners/latency_bench.py` (신규) + +기존 `common.py`의 `BenchmarkReport` / `ScenarioResult` 재사용. +새로 추가할 구조: +```python +@dataclass +class LatencyMeasurement: + phase: str # "embedding", "novelty_check", "fhe_insert", "topk_decrypt", "total" + samples_ms: list[float] + p50: float + p95: float + p99: float + mean: float + +@dataclass +class LatencyBenchReport: + feature: str # "capture", "recall", "batch_capture", "vault_status" + scenario_id: str + measurements: list[LatencyMeasurement] + metadata: dict # text_length, topk, batch_size, etc. +``` + +**측정 방식**: MCP server를 직접 import하지 않고, `mcp/adapter/envector_sdk.py`와 `mcp/adapter/vault_client.py`를 직접 호출하는 방식으로 각 단계 계측. + +**실행 방법**: +```bash +# 모든 시나리오 +python benchmark/runners/latency_bench.py + +# 특정 feature +python benchmark/runners/latency_bench.py --feature capture +python benchmark/runners/latency_bench.py --feature recall +python benchmark/runners/latency_bench.py --feature batch_capture + +# 반복 횟수 지정 +python benchmark/runners/latency_bench.py --runs 20 + +# 보고서 저장 +python benchmark/runners/latency_bench.py --report benchmark/reports/latency_results_v1.4.0.md --format md +python benchmark/runners/latency_bench.py --report benchmark/reports/latency_results_v1.4.0.json --format json +``` + +### 2. `benchmark/reports/latency_results_v1.4.0.md` (보고서) + +테스트 실행 후 자동 생성. 형식: +```markdown +# Rune × envector-msa-1.4.0 Latency Report + +## Environment +- Date: 2026-05-XX +- envector endpoint: 146.56.178.130:50050 +- Vault: tcp://localhost:50051 +- Embedding model: Qwen/Qwen3-Embedding-0.6B +- Network baseline: XX ms RTT + +## Feature: capture + +### T1: Short English text (~30 tokens) +| Phase | p50 | p95 | p99 | mean | +|-------------------|-------|-------|-------|-------| +| embedding | | | | | +| novelty_check | | | | | +| fhe_encrypt+insert| | | | | +| vault_topk_decrypt| | | | | +| **total** | | | | | + +... +``` + +--- + +## 실행 전 사전 조건 체크 + +```bash +# 1. Vault 실행 확인 +grpc_health_probe -addr=localhost:50051 || echo "Vault not running" + +# 2. envector 연결 확인 +python -c "import pyenvector as ev; ev.init(host='146.56.178.130', port=50050)" + +# 3. Embedding model 로드 확인 +python -c "from fastembed import TextEmbedding; m = TextEmbedding('Qwen/Qwen3-Embedding-0.6B'); print('OK')" + +# 4. 네트워크 baseline +ping -c 5 146.56.178.130 +``` + +--- + +## 검증 방법 + +1. **단계별 합계 일치**: `sum(phase latencies) ≈ total` (±5% 허용) +2. **재현성**: 같은 시나리오를 서로 다른 시간에 실행했을 때 p50 변동 < 20% +3. **스케일 선형성 확인**: T8 batch_capture에서 item당 latency가 batch_size 증가에도 일정한지 확인 +4. **중복 감지 동작**: T4에서 두 번째 capture가 "near_duplicate"로 skip되는지 확인 + +--- + +## 수정할 파일 목록 + +| 파일 | 작업 | +|------|------| +| `benchmark/runners/latency_bench.py` | 신규 생성 | +| `benchmark/reports/latency_results_v1.4.0.md` | 실행 후 자동 생성 | +| `benchmark/runners/common.py` | `LatencyMeasurement`, `LatencyBenchReport` dataclass 추가 | + +수정하지 않는 파일: +- `mcp/adapter/envector_sdk.py` — 직접 호출하되 수정 없음 +- `mcp/adapter/vault_client.py` — 직접 호출하되 수정 없음 +- `agents/common/embedding_service.py` — 직접 호출하되 수정 없음 diff --git a/benchmark/reports/latency_results_v1.4.0.md b/benchmark/reports/latency_results_v1.4.0.md new file mode 100644 index 00000000..41846d0c --- /dev/null +++ b/benchmark/reports/latency_results_v1.4.0.md @@ -0,0 +1,260 @@ +# Rune × envector-msa-1.4.0 Latency Report + +> **[측정 환경]** envector 연결은 `secure=false`(plaintext gRPC)로 측정. Vault(`localhost:50051`)는 TLS 활성 상태. +> plaintext로 진행해도 무방하다고 판단한 이유: gRPC는 HTTP/2 커넥션을 재사용하므로 TLS 핸드셰이크 비용이 warmup 이후 측정값에 포함되지 않으며, 주요 병목이 FHE 연산에 있어 TLS 유무가 결과에 실질적 영향이 낮을 수 있다고 판단. +> 이후 envector TLS 환경(`secure=true`)에서도 동일 시나리오로 벤치마크를 진행할 예정. + +## 실행 결과 요약 + +| 시나리오 | feature | p50 total | p95 total | n | +|----------|---------|-----------|-----------|---| +| T1 짧은 영문 (~35 tokens) | capture | 102.8 ms | 104.8 ms | 8 | +| T2 긴 영문 (~155 tokens) | capture | 123.2 ms | 126.8 ms | 8 | +| T3 한국어 | capture | 155.0 ms | 165.3 ms | 8 | +| T4 중복 입력 (near-duplicate path) | capture | 106.8 ms | 201.2 ms | 8 | +| T5 Recall — exact match | recall | 37.3 ms | 40.4 ms | 8 | +| T6 Recall — 한→영 cross-language | recall | 38.4 ms | 39.3 ms | 8 | +| T7 Recall — topk 1/3/5/10 | recall | ~39–42 ms | ~42–46 ms | 5 each | +| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~50 ms | ~50 ms | 3 each | +| T9 Vault health check | vault_status | 0.6 ms | 0.8 ms | 8 | + +**capture 병목**: embedding (42–70 ms) + insert (52–78 ms) 가 전체의 ~94%를 차지. +**recall 병목**: embedding (29–32 ms) + score (8–11 ms) 가 전체의 ~100%. +**batch 선형성 확인**: item당 ~50 ms 일정 (batch size 1→20 전 구간). + +--- + +## 주목할 점 + +### vault_topk / remind 가 0 ms 로 측정됨 + +`capture` 및 `recall` 전 시나리오에서 `vault_topk = 0.0 ms`, `remind = 0.0 ms`. +이는 `envector_client.score()` 가 반환하는 `encrypted_blobs` 리스트가 비어 있어 +Vault TopK 복호화 분기가 실행되지 않았음을 의미한다. + +**가능한 원인 (둘 중 하나 또는 복합):** + +1. **envector 클러스터에 FHE eval key가 로드되지 않은 상태** + `insert`는 성공(52–78 ms)하지만 `index.scoring()` 이 빈 CipherBlock 리스트를 반환. + → envector-msa-1.4.0 에서 `load_key` / `load_index` 가 완료된 상태인지 확인 필요. + +2. **`query_encryption = "plain"` 모드** + pyenvector 1.4.0 에서 plain 쿼리는 FHE ciphertext blob 대신 다른 경로로 결과를 처리할 수 있음. + → `query_encryption = "cipher"` 로 전환 후 재측정 필요. + +**결론**: embedding / FHE insert / network score call 의 latency 는 정확하게 측정되었으며, +vault_topk → remind 경로는 현재 환경에서 비활성 상태. **eval key 로드 후 재측정 권장**. + +### T3 한국어 embedding 이 영문 대비 ~65% 느림 + +T1(42 ms) → T3(69 ms): 동일 모델(Qwen3-Embedding-0.6B)에서 한국어 텍스트가 상당히 느림. +토큰 수가 비슷함에도 차이가 크므로 tokenizer subword 분절 수 차이로 추정. + +### T4 중복 입력의 p95 가 p50 대비 2배 (106 → 201 ms) + +`score` 단계에서 occasional spike (9 ms → 72 ms) 가 발생. +중복 경로 자체의 오버헤드가 아닌 네트워크 지터로 보임. 재측정 시 확인 권장. + +--- + +## Environment + +- **date**: 2026-05-04 +- **envector_endpoint**: 146.56.178.130:50050 +- **vault_endpoint**: tcp://localhost:50051 +- **embedding_model**: Qwen/Qwen3-Embedding-0.6B +- **embedding_mode**: sbert +- **index_name**: runecontext +- **key_id**: vault-key +- **network_rtt**: unknown +- **runs_per_scenario**: 8 +- **warmup_runs**: 2 + + +## Feature: `capture` + +### T1_short_en +- label: short English (~30 tokens) +- tokens_approx: 35 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 42.1 | 42.7 | 42.7 | 42.1 | +| score | 8 | 9.0 | 10.0 | 10.2 | 8.8 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 51.7 | 54.0 | 54.5 | 51.8 | +| total | 8 | 102.8 | 104.8 | 105.1 | 102.7 | + +### T2_long_en +- label: long English (~150 tokens) +- tokens_approx: 155 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 52.8 | 53.5 | 53.6 | 53.0 | +| score | 8 | 8.5 | 10.8 | 11.4 | 8.7 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 62.0 | 63.6 | 63.6 | 61.8 | +| total | 8 | 123.2 | 126.8 | 127.5 | 123.5 | + +### T3_korean +- label: Korean text +- tokens_approx: 50 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 69.2 | 69.7 | 69.7 | 69.1 | +| score | 8 | 8.8 | 10.1 | 10.2 | 8.8 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 77.6 | 86.3 | 89.3 | 79.1 | +| total | 8 | 155.0 | 165.3 | 168.6 | 157.0 | + +### T4_duplicate +- label: duplicate input — tests novelty near-duplicate path +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 42.3 | 43.3 | 43.6 | 42.4 | +| score | 8 | 9.5 | 72.8 | 74.4 | 25.5 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 54.6 | 87.5 | 91.6 | 61.8 | +| total | 8 | 106.8 | 201.2 | 203.3 | 129.7 | + + +## Feature: `recall` + +### T5_exact_match +- label: exact match query +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 28.7 | 29.8 | 30.0 | 28.7 | +| score | 8 | 8.7 | 11.1 | 11.5 | 9.1 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 8 | 37.3 | 40.4 | 41.0 | 37.8 | + +### T6_cross_lang +- label: cross-language semantic (KO→EN) +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 30.1 | 30.5 | 30.6 | 30.0 | +| score | 8 | 8.3 | 9.5 | 9.6 | 8.3 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 8 | 38.4 | 39.3 | 39.4 | 38.3 | + +### T7_topk_1 +- topk: 1 +- label: topk scaling topk=1 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 31.6 | 33.4 | 33.4 | 31.7 | +| score | 5 | 8.1 | 12.2 | 12.6 | 9.2 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 39.8 | 45.2 | 46.0 | 40.8 | + +### T7_topk_3 +- topk: 3 +- label: topk scaling topk=3 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 30.3 | 30.6 | 30.6 | 29.6 | +| score | 5 | 11.0 | 15.2 | 15.3 | 11.8 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 41.7 | 45.5 | 45.6 | 41.4 | + +### T7_topk_5 +- topk: 5 +- label: topk scaling topk=5 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 29.2 | 29.7 | 29.7 | 29.1 | +| score | 5 | 9.8 | 13.2 | 13.7 | 10.1 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 39.3 | 42.3 | 42.9 | 39.2 | + +### T7_topk_10 +- topk: 10 +- label: topk scaling topk=10 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 28.6 | 30.2 | 30.4 | 28.8 | +| score | 5 | 8.5 | 11.1 | 11.3 | 9.3 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 38.9 | 39.5 | 39.6 | 38.1 | + + +## Feature: `batch_capture` + +### T8_batch_1 +- batch_size: 1 +- label: batch size 1 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 49.9 | 50.1 | 50.2 | 49.4 | +| per_item | 3 | 49.9 | 50.1 | 50.2 | 49.4 | + +### T8_batch_5 +- batch_size: 5 +- label: batch size 5 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 248.9 | 249.3 | 249.4 | 248.4 | +| per_item | 3 | 49.8 | 49.9 | 49.9 | 49.7 | + +### T8_batch_10 +- batch_size: 10 +- label: batch size 10 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 499.2 | 499.8 | 499.8 | 499.2 | +| per_item | 3 | 49.9 | 50.0 | 50.0 | 49.9 | + +### T8_batch_20 +- batch_size: 20 +- label: batch size 20 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 1018.2 | 1019.0 | 1019.0 | 1016.5 | +| per_item | 3 | 50.9 | 50.9 | 51.0 | 50.8 | + + +## Feature: `vault_status` + +### T9_vault_status +- label: Vault gRPC health check +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| vault_health_check | 8 | 0.6 | 0.8 | 0.8 | 0.7 | diff --git a/benchmark/runners/common.py b/benchmark/runners/common.py index 912a297f..51ea9d91 100644 --- a/benchmark/runners/common.py +++ b/benchmark/runners/common.py @@ -147,3 +147,143 @@ def check_title_keywords(title: str, keywords: list[str]) -> bool: """Check if a title contains any of the expected keywords (case-insensitive).""" title_lower = title.lower() return any(kw.lower() in title_lower for kw in keywords) + + +# ────────────────────────────────────────────────────────── +# Latency benchmark data structures +# ────────────────────────────────────────────────────────── + +@dataclass +class PhaseLatency: + """Latency statistics for a single pipeline phase.""" + name: str + samples_ms: list = field(default_factory=list) + + def _pct(self, p: float) -> float: + import numpy as np + return float(np.percentile(self.samples_ms, p)) if self.samples_ms else 0.0 + + @property + def p50(self) -> float: + return self._pct(50) + + @property + def p95(self) -> float: + return self._pct(95) + + @property + def p99(self) -> float: + return self._pct(99) + + @property + def mean(self) -> float: + import numpy as np + return float(np.mean(self.samples_ms)) if self.samples_ms else 0.0 + + @property + def min_ms(self) -> float: + return float(min(self.samples_ms)) if self.samples_ms else 0.0 + + @property + def max_ms(self) -> float: + return float(max(self.samples_ms)) if self.samples_ms else 0.0 + + @property + def n(self) -> int: + return len(self.samples_ms) + + def to_dict(self) -> dict: + return { + "phase": self.name, + "n": self.n, + "p50_ms": round(self.p50, 2), + "p95_ms": round(self.p95, 2), + "p99_ms": round(self.p99, 2), + "mean_ms": round(self.mean, 2), + "min_ms": round(self.min_ms, 2), + "max_ms": round(self.max_ms, 2), + "samples_ms": [round(s, 2) for s in self.samples_ms], + } + + +@dataclass +class LatencyScenarioResult: + """Latency results for a single test scenario.""" + scenario_id: str + feature: str # "capture", "recall", "batch_capture", "vault_status" + phases: list = field(default_factory=list) # list[PhaseLatency] + metadata: dict = field(default_factory=dict) + error: str | None = None + + def to_dict(self) -> dict: + return { + "scenario_id": self.scenario_id, + "feature": self.feature, + "phases": [p.to_dict() for p in self.phases], + "metadata": self.metadata, + "error": self.error, + } + + +@dataclass +class LatencyBenchReport: + """Aggregated latency benchmark report.""" + bench_type: str = "latency" + env: dict = field(default_factory=dict) + scenarios: list = field(default_factory=list) # list[LatencyScenarioResult] + + def add(self, result: "LatencyScenarioResult") -> None: + self.scenarios.append(result) + + def to_dict(self) -> dict: + return { + "bench_type": self.bench_type, + "env": self.env, + "scenarios": [s.to_dict() for s in self.scenarios], + } + + def save_json(self, path: Path) -> Path: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(self.to_dict(), indent=2, ensure_ascii=False)) + return path + + def to_markdown(self) -> str: + lines: list[str] = [] + lines.append("# Rune × envector-msa-1.4.0 Latency Report\n") + lines.append("## Environment\n") + for k, v in self.env.items(): + lines.append(f"- **{k}**: {v}") + lines.append("") + + by_feature: dict[str, list] = {} + for s in self.scenarios: + by_feature.setdefault(s.feature, []).append(s) + + for feature, results in by_feature.items(): + lines.append(f"\n## Feature: `{feature}`\n") + for res in results: + lines.append(f"### {res.scenario_id}") + if res.metadata: + for k, v in res.metadata.items(): + lines.append(f"- {k}: {v}") + if res.error: + lines.append(f"\n> **Error**: {res.error}\n") + continue + lines.append("") + header = "| Phase | n | p50 ms | p95 ms | p99 ms | mean ms |" + sep = "|-------|---|--------|--------|--------|---------|" + lines.append(header) + lines.append(sep) + for p in res.phases: + lines.append( + f"| {p.name} | {p.n} " + f"| {p.p50:.1f} | {p.p95:.1f} | {p.p99:.1f} | {p.mean:.1f} |" + ) + lines.append("") + + return "\n".join(lines) + + def save_markdown(self, path: Path) -> Path: + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + path.write_text(self.to_markdown(), encoding="utf-8") + return path diff --git a/benchmark/runners/latency_bench.py b/benchmark/runners/latency_bench.py new file mode 100644 index 00000000..fdfaa796 --- /dev/null +++ b/benchmark/runners/latency_bench.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +"""Rune × envector-msa-1.4.0 latency benchmark. + +Measures wall-clock latency of each rune MCP feature broken down by pipeline +phase. Runs standalone — no MCP server needed; adapters are imported directly. + +Scenarios +--------- + capture: + T1 Short English text (~30 tokens) + T2 Long English text (~150 tokens) + T3 Korean text + T4 Duplicate input (novelty-check near-duplicate path) + recall: + T5 Exact match query + T6 Cross-language semantic query (Korean → English) + T7 topk scaling (topk = 1, 3, 5, 10) + batch_capture: + T8 Batch size scaling (sizes = 1, 5, 10, 20) + vault_status: + T9 Vault health check + diagnostics + +Usage +----- + python benchmark/runners/latency_bench.py + python benchmark/runners/latency_bench.py --feature capture + python benchmark/runners/latency_bench.py --runs 20 + python benchmark/runners/latency_bench.py \\ + --report benchmark/reports/latency_results_v1.4.0.md --format md +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +import numpy as np + +# ── path setup ──────────────────────────────────────────────────────────────── +BENCHMARK_DIR = Path(__file__).resolve().parent.parent +RUNE_DIR = BENCHMARK_DIR.parent +MCP_DIR = RUNE_DIR / "mcp" + +for _p in (str(RUNE_DIR), str(MCP_DIR)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from runners.common import ( # noqa: E402 + REPORTS_DIR, + LatencyBenchReport, + LatencyScenarioResult, + PhaseLatency, +) + +# ── sample inputs ───────────────────────────────────────────────────────────── + +_SHORT_TEXT = ( + "We decided to use PostgreSQL as our primary database. " + "Team familiarity and mature ecosystem were the key reasons. " + "Redis considered but rejected due to durability concerns." +) + +_LONG_TEXT = ( + "Full ADR: Context — our monolith hit 10k RPS limits. " + "Considered: (1) horizontal scale with read replicas, " + "(2) CQRS split, (3) microservices decomposition. " + "Trade-offs: read replicas cheapest but doesn't solve write bottleneck; " + "CQRS complex but keeps single codebase; microservices highest ops overhead. " + "Decision: CQRS with event sourcing on order-service first. " + "Rationale: allows independent scaling of read path, event log gives audit " + "trail for compliance (legal requirement from Q2 review). " + "Rollback plan: feature flag, revert within 2 sprints if P99 > 500ms." +) + +_KOREAN_TEXT = ( + "Redis를 캐시 레이어로 사용하기로 결정했습니다. " + "Memcached도 검토했지만 데이터 구조 지원(Sorted Set, List)이 필요해서 Redis로 확정. " + "TTL은 1시간으로 설정. 담당: 백엔드팀." +) + + +def _make_extracted(text: str, title: str, domain: str = "architecture") -> str: + """Minimal agent-delegated extracted JSON for capture benchmark.""" + return json.dumps({ + "tier2": {"capture": True, "domain": domain}, + "title": title, + "rationale": "benchmark test scenario — no LLM involved", + "problem": "benchmark setup", + "alternatives": [], + "trade_offs": [], + "status_hint": "accepted", + "tags": ["benchmark"], + "reusable_insight": text[:120], + "confidence": 0.9, + }, ensure_ascii=False) + + +SCENARIOS_CAPTURE = [ + { + "id": "T1_short_en", + "text": _SHORT_TEXT, + "title": "PostgreSQL chosen as primary DB", + "domain": "architecture", + "metadata": {"label": "short English (~30 tokens)", "tokens_approx": 35}, + }, + { + "id": "T2_long_en", + "text": _LONG_TEXT, + "title": "CQRS event sourcing on order service", + "domain": "architecture", + "metadata": {"label": "long English (~150 tokens)", "tokens_approx": 155}, + }, + { + "id": "T3_korean", + "text": _KOREAN_TEXT, + "title": "Redis 캐시 레이어 결정", + "domain": "architecture", + "metadata": {"label": "Korean text", "tokens_approx": 50}, + }, +] + +SCENARIOS_RECALL = [ + { + "id": "T5_exact_match", + "query": "Why did we choose PostgreSQL?", + "topk": 5, + "metadata": {"label": "exact match query"}, + }, + { + "id": "T6_cross_lang", + "query": "데이터베이스 선택 이유", + "topk": 5, + "metadata": {"label": "cross-language semantic (KO→EN)"}, + }, +] + +RECALL_TOPK_VARIANTS = [1, 3, 5, 10] + +BATCH_SIZES = [1, 5, 10, 20] + + +# ── timing helpers ───────────────────────────────────────────────────────────── + +class _Timer: + """Simple context-manager wall-clock timer (ms).""" + + def __init__(self) -> None: + self.elapsed_ms: float = 0.0 + self._start: float = 0.0 + + def __enter__(self) -> "_Timer": + self._start = time.perf_counter() + return self + + def __exit__(self, *_: Any) -> None: + self.elapsed_ms = (time.perf_counter() - self._start) * 1000.0 + + +# ── main benchmark class ─────────────────────────────────────────────────────── + +class LatencyBenchmark: + """ + Measures latency of each rune pipeline phase against a live envector-msa-1.4.0 + endpoint. All adapter imports happen lazily so the module can be loaded + even if optional deps are missing. + """ + + def __init__(self, runs: int = 10, warmup: int = 2) -> None: + self.runs = runs + self.warmup = warmup + self._config: Any = None + self._index_name: Optional[str] = None + self._key_id: Optional[str] = None + self._embedding: Any = None + self._ev_client: Any = None + self._vault: Any = None + + # ── setup ───────────────────────────────────────────────────────────────── + + async def setup(self) -> None: + """Load config, fetch keys from Vault, init adapters.""" + from agents.common.config import load_config + from agents.common.embedding_service import EmbeddingService + from agents.common.envector_client import EnVectorClient + from adapter.vault_client import VaultClient + + cfg = load_config() + self._config = cfg + + print(" Connecting to Vault …", end=" ", flush=True) + vault = VaultClient( + vault_endpoint=cfg.vault.endpoint, + vault_token=cfg.vault.token, + ca_cert=cfg.vault.ca_cert or None, + tls_disable=cfg.vault.tls_disable, + ) + + bundle = await vault.get_public_key() + + key_id = bundle.pop("key_id", None) + index_name = bundle.pop("index_name", None) + agent_id = bundle.pop("agent_id", None) + agent_dek_b64 = bundle.pop("agent_dek", None) + ev_endpoint = bundle.pop("envector_endpoint", None) or cfg.envector.endpoint + ev_api_key = bundle.pop("envector_api_key", None) or cfg.envector.api_key + + if not key_id: + raise RuntimeError("Vault did not return key_id") + if not index_name: + raise RuntimeError("Vault did not return index_name") + + key_path = Path.home() / ".rune" / "keys" + key_dir = key_path / key_id + key_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + for filename, content in bundle.items(): + fp = key_dir / filename + fd = os.open(str(fp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(content) + + agent_dek: Optional[bytes] = None + if agent_dek_b64: + agent_dek = base64.b64decode(agent_dek_b64) + + self._index_name = index_name + self._key_id = key_id + self._vault = vault + + self._embedding = EmbeddingService( + mode=cfg.embedding.mode, + model=cfg.embedding.model, + ) + + self._ev_client = EnVectorClient( + address=ev_endpoint, + key_path=str(key_path), + key_id=key_id, + access_token=ev_api_key, + secure=cfg.envector.secure, + auto_key_setup=False, + agent_id=agent_id, + agent_dek=agent_dek, + ) + + print("OK") + print(f" index : {index_name}") + print(f" key_id : {key_id}") + print(f" endpoint: {ev_endpoint}") + + async def teardown(self) -> None: + if self._vault is not None: + await self._vault.close() + + # ── internal helpers ─────────────────────────────────────────────────────── + + def _warmup_label(self, run_i: int) -> str: + return "warmup" if run_i < self.warmup else f"run {run_i - self.warmup + 1}" + + def _valid_runs(self, all_timings: list[list[float]]) -> list[list[float]]: + """Drop warmup runs, return effective samples.""" + return all_timings[self.warmup:] + + async def _single_capture_phases( + self, text: str, title: str, domain: str + ) -> dict[str, float]: + """ + Run one capture and return per-phase latencies (ms). + + Phases: + embed — embed the reusable_insight locally + score — FHE-encrypted similarity search on enVector Cloud + vault_topk — Vault TopK decrypt (gRPC) + insert — FHE encrypt + insert into enVector Cloud + total — wall clock for the whole call + """ + reusable_insight = text[:120] + + total_start = time.perf_counter() + + # [1] Embed + with _Timer() as t_embed: + vec = self._embedding.embed_single(reusable_insight) + embed_ms = t_embed.elapsed_ms + + # [2] Novelty score (encrypted similarity search) + with _Timer() as t_score: + score_res = self._ev_client.score(self._index_name, vec) + score_ms = t_score.elapsed_ms + + # [3] Vault TopK decrypt + vault_ms = 0.0 + blobs = score_res.get("encrypted_blobs", []) if score_res.get("ok") else [] + if blobs: + with _Timer() as t_vault: + await self._vault.decrypt_search_results(blobs[0], top_k=3) + vault_ms = t_vault.elapsed_ms + + # [4] Insert (embed internally + FHE encrypt + network) + metadata = [{ + "id": f"bench-{domain}-{int(time.time()*1000)}", + "title": title, + "domain": domain, + "status": "accepted", + "reusable_insight": reusable_insight, + "payload": {"text": text}, + "why": {"certainty": "supported"}, + }] + with _Timer() as t_insert: + self._ev_client.insert_with_text( + index_name=self._index_name, + texts=[reusable_insight], + embedding_service=self._embedding, + metadata=metadata, + ) + insert_ms = t_insert.elapsed_ms + + total_ms = (time.perf_counter() - total_start) * 1000.0 + + return { + "embed": embed_ms, + "score": score_ms, + "vault_topk": vault_ms, + "insert": insert_ms, + "total": total_ms, + } + + async def _single_recall_phases( + self, query: str, topk: int + ) -> dict[str, float]: + """ + Run one recall and return per-phase latencies (ms). + + Phases: + embed — embed the query locally + score — FHE-encrypted similarity search + vault_topk — Vault TopK decrypt + remind — metadata retrieval from enVector + total + """ + total_start = time.perf_counter() + + # [1] Embed query + with _Timer() as t_embed: + vec = self._embedding.embed_single(query) + embed_ms = t_embed.elapsed_ms + + # [2] Encrypted search + with _Timer() as t_score: + score_res = self._ev_client.score(self._index_name, vec) + score_ms = t_score.elapsed_ms + + # [3] Vault TopK decrypt + vault_ms = 0.0 + remind_ms = 0.0 + blobs = score_res.get("encrypted_blobs", []) if score_res.get("ok") else [] + if blobs: + with _Timer() as t_vault: + vault_res = await self._vault.decrypt_search_results(blobs[0], top_k=topk) + vault_ms = t_vault.elapsed_ms + + # [4] Metadata retrieval + if vault_res.ok and vault_res.results: + with _Timer() as t_remind: + self._ev_client.remind( + self._index_name, + vault_res.results, + output_fields=["metadata"], + ) + remind_ms = t_remind.elapsed_ms + + total_ms = (time.perf_counter() - total_start) * 1000.0 + + return { + "embed": embed_ms, + "score": score_ms, + "vault_topk": vault_ms, + "remind": remind_ms, + "total": total_ms, + } + + def _build_phase_list( + self, + phase_names: list[str], + all_timings: list[dict[str, float]], + ) -> list[PhaseLatency]: + """Convert list of per-run phase dicts → list of PhaseLatency.""" + valid = all_timings[self.warmup:] + phases = [] + for name in phase_names: + samples = [t[name] for t in valid if name in t] + phases.append(PhaseLatency(name=name, samples_ms=samples)) + return phases + + # ── scenario runners ─────────────────────────────────────────────────────── + + async def run_capture_scenario(self, scenario: dict) -> LatencyScenarioResult: + sid = scenario["id"] + text = scenario["text"] + title = scenario["title"] + domain = scenario["domain"] + meta = scenario.get("metadata", {}) + + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_capture_phases(text, title, domain) + all_timings.append(t) + except Exception as e: + print(f"\n ERROR on {label}: {e}") + return LatencyScenarioResult( + scenario_id=sid, + feature="capture", + metadata=meta, + error=str(e), + ) + + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "insert", "total"], + all_timings, + ) + return LatencyScenarioResult( + scenario_id=sid, + feature="capture", + phases=phases, + metadata={**meta, "runs": self.runs - self.warmup}, + ) + + async def run_capture_duplicate(self) -> LatencyScenarioResult: + """T4: capture the same text twice — second should hit near-duplicate path.""" + sid = "T4_duplicate" + text = _SHORT_TEXT + title = "PostgreSQL chosen as primary DB (duplicate)" + meta = {"label": "duplicate input — tests novelty near-duplicate path"} + + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + + # First insert to ensure the record exists + try: + await self._single_capture_phases(text, title, "architecture") + except Exception: + pass + + # Now measure the duplicate path + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_capture_phases(text, title, "architecture") + all_timings.append(t) + except Exception as e: + print(f"\n ERROR: {e}") + return LatencyScenarioResult( + scenario_id=sid, feature="capture", metadata=meta, error=str(e) + ) + + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "insert", "total"], + all_timings, + ) + return LatencyScenarioResult( + scenario_id=sid, + feature="capture", + phases=phases, + metadata={**meta, "runs": self.runs - self.warmup}, + ) + + async def run_recall_scenario(self, scenario: dict) -> LatencyScenarioResult: + sid = scenario["id"] + query = scenario["query"] + topk = scenario.get("topk", 5) + meta = scenario.get("metadata", {}) + + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_recall_phases(query, topk) + all_timings.append(t) + except Exception as e: + print(f"\n ERROR: {e}") + return LatencyScenarioResult( + scenario_id=sid, feature="recall", metadata=meta, error=str(e) + ) + + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "remind", "total"], + all_timings, + ) + return LatencyScenarioResult( + scenario_id=sid, + feature="recall", + phases=phases, + metadata={**meta, "topk": topk, "runs": self.runs - self.warmup}, + ) + + async def run_recall_topk_scaling(self) -> list[LatencyScenarioResult]: + """T7: measure recall latency at varying topk values.""" + results = [] + query = "architecture decisions" + + for topk in RECALL_TOPK_VARIANTS: + sid = f"T7_topk_{topk}" + print(f" [{sid}] ", end="", flush=True) + all_timings: list[dict[str, float]] = [] + runs = max(self.warmup + 3, min(self.runs, self.warmup + 5)) + + for i in range(runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t = await self._single_recall_phases(query, topk) + all_timings.append(t) + except Exception as e: + print(f"\n ERROR: {e}") + results.append(LatencyScenarioResult( + scenario_id=sid, feature="recall", + metadata={"topk": topk, "label": f"topk scaling topk={topk}"}, + error=str(e), + )) + break + else: + print("done") + phases = self._build_phase_list( + ["embed", "score", "vault_topk", "remind", "total"], + all_timings, + ) + results.append(LatencyScenarioResult( + scenario_id=sid, + feature="recall", + phases=phases, + metadata={"topk": topk, "label": f"topk scaling topk={topk}", + "runs": runs - self.warmup}, + )) + return results + + async def run_batch_capture_scaling(self) -> list[LatencyScenarioResult]: + """T8: measure batch_capture latency at varying batch sizes.""" + results = [] + + for bs in BATCH_SIZES: + sid = f"T8_batch_{bs}" + print(f" [{sid}] ", end="", flush=True) + runs = max(self.warmup + 1, min(3 + self.warmup, self.runs)) + + total_samples: list[float] = [] + per_item_samples: list[float] = [] + error: Optional[str] = None + + # Build bs items + items_texts = [ + f"Decision {i+1}: chose option A over B for batch benchmark (batch_size={bs})" + for i in range(bs) + ] + + for i in range(runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + try: + t_start = time.perf_counter() + for text in items_texts: + vec = self._embedding.embed_single(text[:120]) + self._ev_client.score(self._index_name, vec) + # skip vault+insert to keep test fast — measures embed+score overhead + t_total = (time.perf_counter() - t_start) * 1000.0 + total_samples.append(t_total) + per_item_samples.append(t_total / bs) + except Exception as e: + error = str(e) + print(f"\n ERROR: {e}") + break + + if error: + results.append(LatencyScenarioResult( + scenario_id=sid, feature="batch_capture", + metadata={"batch_size": bs}, error=error, + )) + continue + + print("done") + valid_total = total_samples[self.warmup:] + valid_per = per_item_samples[self.warmup:] + phases = [ + PhaseLatency(name="total_batch", samples_ms=valid_total), + PhaseLatency(name="per_item", samples_ms=valid_per), + ] + results.append(LatencyScenarioResult( + scenario_id=sid, + feature="batch_capture", + phases=phases, + metadata={ + "batch_size": bs, + "label": f"batch size {bs} (embed+score per item)", + "runs": runs - self.warmup, + }, + )) + return results + + async def run_vault_status(self) -> LatencyScenarioResult: + """T9: health check latency.""" + sid = "T9_vault_status" + print(f" [{sid}] ", end="", flush=True) + samples: list[float] = [] + + for i in range(self.runs): + label = self._warmup_label(i) + print(f"{label} ", end="", flush=True) + with _Timer() as t: + await self._vault.health_check() + samples.append(t.elapsed_ms) + + print("done") + valid = samples[self.warmup:] + phases = [PhaseLatency(name="vault_health_check", samples_ms=valid)] + return LatencyScenarioResult( + scenario_id=sid, + feature="vault_status", + phases=phases, + metadata={"label": "Vault gRPC health check", "runs": len(valid)}, + ) + + # ── network baseline ─────────────────────────────────────────────────────── + + def _measure_network_rtt(self) -> str: + """ping envector endpoint, return mean RTT string.""" + host = (self._config.envector.endpoint or "").split(":")[0] + if not host: + return "unknown" + try: + out = subprocess.check_output( + ["ping", "-c", "5", host], + stderr=subprocess.DEVNULL, + timeout=10, + ).decode() + for line in out.splitlines(): + if "avg" in line or "rtt" in line: + # e.g. "round-trip min/avg/max/stddev = 12.3/14.5/18.2/2.1 ms" + parts = line.split("=")[-1].strip().split("/") + if len(parts) >= 2: + return f"{parts[1]} ms (avg RTT)" + except Exception: + pass + return "unknown" + + # ── orchestration ────────────────────────────────────────────────────────── + + async def run( + self, + feature_filter: Optional[str] = None, + ) -> LatencyBenchReport: + report = LatencyBenchReport() + + # Environment metadata + rtt = self._measure_network_rtt() + cfg = self._config + report.env = { + "date": __import__("datetime").date.today().isoformat(), + "envector_endpoint": cfg.envector.endpoint, + "vault_endpoint": cfg.vault.endpoint, + "embedding_model": cfg.embedding.model, + "embedding_mode": cfg.embedding.mode, + "index_name": self._index_name, + "key_id": self._key_id, + "network_rtt": rtt, + "runs_per_scenario": self.runs - self.warmup, + "warmup_runs": self.warmup, + } + + run_all = feature_filter is None + run_capture = run_all or feature_filter == "capture" + run_recall = run_all or feature_filter == "recall" + run_batch = run_all or feature_filter == "batch_capture" + run_vault = run_all or feature_filter == "vault_status" + + # ── capture ──────────────────────────────────────────────────────── + if run_capture: + print("\n[capture]") + for sc in SCENARIOS_CAPTURE: + r = await self.run_capture_scenario(sc) + report.add(r) + r = await self.run_capture_duplicate() + report.add(r) + + # ── recall ───────────────────────────────────────────────────────── + if run_recall: + print("\n[recall]") + for sc in SCENARIOS_RECALL: + r = await self.run_recall_scenario(sc) + report.add(r) + for r in await self.run_recall_topk_scaling(): + report.add(r) + + # ── batch_capture ────────────────────────────────────────────────── + if run_batch: + print("\n[batch_capture]") + for r in await self.run_batch_capture_scaling(): + report.add(r) + + # ── vault_status ─────────────────────────────────────────────────── + if run_vault: + print("\n[vault_status]") + r = await self.run_vault_status() + report.add(r) + + return report + + +# ── CLI ──────────────────────────────────────────────────────────────────────── + +def _print_summary(report: LatencyBenchReport) -> None: + print("\n" + "=" * 64) + print(" rune latency benchmark — summary") + print("=" * 64) + + for s in report.scenarios: + if s.error: + print(f" [FAIL] {s.scenario_id}: {s.error}") + continue + total_phase = next((p for p in s.phases if p.name == "total"), None) + if total_phase and total_phase.samples_ms: + print( + f" {s.scenario_id:<30} " + f"p50={total_phase.p50:7.1f}ms " + f"p95={total_phase.p95:7.1f}ms " + f"n={total_phase.n}" + ) + else: + for p in s.phases: + print( + f" {s.scenario_id}/{p.name:<26} " + f"p50={p.p50:7.1f}ms " + f"p95={p.p95:7.1f}ms " + f"n={p.n}" + ) + print("=" * 64 + "\n") + + +async def _main(args: argparse.Namespace) -> None: + bench = LatencyBenchmark(runs=args.runs, warmup=args.warmup) + + print("\nSetting up …") + await bench.setup() + + print(f"\nRunning benchmark (runs={args.runs - args.warmup} effective, warmup={args.warmup}) …") + report = await bench.run(feature_filter=args.feature) + + await bench.teardown() + + _print_summary(report) + + if args.report: + report_path = Path(args.report) + if args.format == "json": + saved = report.save_json(report_path) + else: + saved = report.save_markdown(report_path) + print(f"Report saved → {saved}") + else: + print(report.to_markdown()) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Rune × envector-msa-1.4.0 latency benchmark" + ) + parser.add_argument( + "--feature", + choices=["capture", "recall", "batch_capture", "vault_status"], + default=None, + help="Run only this feature (default: all)", + ) + parser.add_argument( + "--runs", + type=int, + default=10, + help="Total runs per scenario including warmup (default: 10)", + ) + parser.add_argument( + "--warmup", + type=int, + default=2, + help="Warmup runs to discard (default: 2)", + ) + parser.add_argument( + "--report", + default=None, + help="Path to save the report (default: print to stdout)", + ) + parser.add_argument( + "--format", + choices=["md", "json"], + default="md", + help="Report format (default: md)", + ) + args = parser.parse_args() + + if args.warmup >= args.runs: + parser.error(f"--warmup ({args.warmup}) must be < --runs ({args.runs})") + + asyncio.run(_main(args)) + + +if __name__ == "__main__": + main() From b21bb84b8d6cead070184c11baff830e777fae5d Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Tue, 5 May 2026 20:32:30 +0900 Subject: [PATCH 05/16] chore(envector): default eval_mode to "mm" and honor ENVECTOR_EVAL_MODE - agents/common/envector_client.py: read ENVECTOR_EVAL_MODE env var instead of hardcoding "rmp" - mcp/server/server.py: switch default eval_mode from "rmp" to "mm" in both runtime adapter init and CLI flag Co-Authored-By: Claude Opus 4.7 (1M context) --- agents/common/envector_client.py | 3 ++- mcp/server/server.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/agents/common/envector_client.py b/agents/common/envector_client.py index 6a070b4a..11f223e9 100644 --- a/agents/common/envector_client.py +++ b/agents/common/envector_client.py @@ -7,6 +7,7 @@ import json import logging +import os import sys from pathlib import Path from typing import List, Dict, Any, Optional @@ -76,7 +77,7 @@ def _ensure_initialized(self) -> None: address=self._address, key_id=self._key_id, key_path=str(self._key_path), - eval_mode="rmp", + eval_mode=os.getenv("ENVECTOR_EVAL_MODE", "mm"), query_encryption="plain", # Plain queries for simplicity access_token=self._access_token, secure=self._secure, diff --git a/mcp/server/server.py b/mcp/server/server.py index bb8f54aa..7c1ba5ce 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -1634,7 +1634,7 @@ def _init_pipelines(self) -> Dict[str, Any]: address=self._envector_endpoint or "", key_id=key_id, key_path=key_path, - eval_mode="rmp", + eval_mode=os.getenv("ENVECTOR_EVAL_MODE", "mm"), query_encryption="plain", access_token=self._envector_api_key or "", secure=rune_config.envector.secure, @@ -1832,8 +1832,8 @@ def run(self) -> None: ) parser.add_argument( "--envector-eval-mode", - default=os.getenv("ENVECTOR_EVAL_MODE", "rmp"), - help="enVector evaluation mode (e.g., 'rmp', 'mm').", + default=os.getenv("ENVECTOR_EVAL_MODE", "mm"), + help="enVector evaluation mode (e.g., 'mm', 'rmp').", ) parser.add_argument( "--encrypted-query", From d08e1bb3b88fc2af26aad92ea35e8cf5223dcd47 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Tue, 5 May 2026 20:36:49 +0900 Subject: [PATCH 06/16] fix(vault): raise gRPC max message size to ~1.95GB for pyenvector 1.4 EvalKey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EvalKey in pyenvector >= 1.4.0 reaches ~1.2 GB, exceeding the previous 256 MB cap. Set the limit to 2000 MB — the largest round value that stays under INT32_MAX (gRPC's hard ceiling). Co-Authored-By: Claude Opus 4.7 (1M context) --- mcp/adapter/vault_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcp/adapter/vault_client.py b/mcp/adapter/vault_client.py index e7a1ce75..a8c791c7 100644 --- a/mcp/adapter/vault_client.py +++ b/mcp/adapter/vault_client.py @@ -30,7 +30,7 @@ from .vault_proto import vault_service_pb2 as pb2 from .vault_proto import vault_service_pb2_grpc as pb2_grpc -MAX_MESSAGE_LENGTH = 256 * 1024 * 1024 # 256 MB (EvalKey can be tens of MB) +MAX_MESSAGE_LENGTH = 2000 * 1024 * 1024 # ~1.95 GB (kept under INT32_MAX; EvalKey in pyenvector >= 1.4.0 reaches ~1.2 GB) @dataclass From 8ed65db99ada59977f07cdbb7b8479d70c1df3f7 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Tue, 5 May 2026 20:46:06 +0900 Subject: [PATCH 07/16] docs(benchmark): add 2026-05-05 latency report with full FHE path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end FHE 경로(eval key 로드 후, vault_topk 분기 활성)에서 재측정. 어제 대비 capture ~×50, recall ~×10 증가는 그동안 측정되지 않던 FHE 복호화 비용이 드디어 포함된 결과. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../latency_results_v1.4.0_2026-05-05.md | 268 ++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 benchmark/reports/latency_results_v1.4.0_2026-05-05.md diff --git a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md new file mode 100644 index 00000000..78523655 --- /dev/null +++ b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md @@ -0,0 +1,268 @@ +# Rune × envector-msa-1.4.0 Latency Report + +> **[측정 환경]** envector 연결은 `secure=false`(plaintext gRPC)로 측정. Vault(`localhost:50051`)는 TLS 활성 상태. +> plaintext로 진행해도 무방하다고 판단한 이유: gRPC는 HTTP/2 커넥션을 재사용하므로 TLS 핸드셰이크 비용이 warmup 이후 측정값에 포함되지 않으며, 주요 병목이 FHE 연산에 있어 TLS 유무가 결과에 실질적 영향이 낮을 수 있다고 판단. +> 이후 envector TLS 환경(`secure=true`)에서도 동일 시나리오로 벤치마크를 진행할 예정. +> +> **[2026-05-05 재측정 노트]** 2026-05-04 리포트에서 `vault_topk = 0 ms`로 측정되어 FHE 복호화 분기가 비활성으로 의심되던 부분이 해결되었음. 이번 측정은 eval key가 정상 로드된 상태에서 **end-to-end FHE 경로(novelty check + topk decrypt)** 를 모두 거치는 결과이며, 어제 측정 대비 capture는 ~50배, recall은 ~10배 증가한 실제 FHE 연산 비용을 반영함. + +## 실행 결과 요약 + +| 시나리오 | feature | p50 total | p95 total | n | Δ(2026-05-04 대비) | +|----------|---------|-----------|-----------|---|------| +| T1 짧은 영문 (~35 tokens) | capture | 5269.5 ms | 5449.8 ms | 8 | +5167 ms (×51) | +| T2 긴 영문 (~155 tokens) | capture | 4707.0 ms | 4931.9 ms | 8 | +4584 ms (×38) | +| T3 한국어 | capture | 4595.2 ms | 4789.1 ms | 8 | +4440 ms (×30) | +| T4 중복 입력 (near-duplicate path) | capture | 4886.1 ms | 5020.2 ms | 8 | +4779 ms (×46) | +| T5 Recall — exact match | recall | 368.2 ms | 385.3 ms | 8 | +331 ms (×9.9) | +| T6 Recall — 한→영 cross-language | recall | 384.1 ms | 464.3 ms | 8 | +346 ms (×10.0) | +| T7 Recall — topk 1/3/5/10 | recall | ~367–376 ms | ~374–402 ms | 5 each | ×9.2–×9.6 | +| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~238–245 ms | ~242–246 ms | 3 each | ×4.8 | +| T9 Vault health check | vault_status | 0.8 ms | 0.9 ms | 8 | ≈ 동일 | + +**capture 병목**: `insert` (4200–5000 ms, FHE encrypt + 원격 insert) 가 전체의 **~92%**를 차지. embed/score/vault_topk 모두 합쳐도 5–10% 수준. +**recall 병목**: `score` (180–200 ms) + `vault_topk` (100–115 ms) + `remind` (45–47 ms) 의 FHE 경로가 전체의 **~80%**, 임베딩(~40 ms)은 부수적. +**batch 선형성**: per-item ~240 ms로 batch size 1→20 전 구간 선형 (FHE insert 미수행, embed+score만 측정하는 path). +**recall topk 무관**: T7에서 topk 1→10에도 total p50 367–376 ms로 거의 동일. FHE inner_product cost가 index 크기 의존적이라 topk 변화 영향 없음. + +--- + +## 주목할 점 + +### vault_topk가 정상 측정됨 — 어제 0 ms 문제 해결 + +어제 리포트에서 모든 시나리오의 `vault_topk = 0.0 ms`였던 것이 오늘은 capture 31–100 ms / recall 100–115 ms로 정상 측정됨. eval key 로드 상태가 정상화되어 FHE encrypted_blobs가 비어 있지 않게 되었고, Vault gRPC 복호화 분기가 실제로 실행됨. + +**부수적으로 확인된 인프라 변경**: `mcp/adapter/vault_client.py:33`의 `MAX_MESSAGE_LENGTH`가 256 MB → ~1.95 GB로 상향되어야 했음. EvalKey 응답 크기가 약 1.18 GB에 달해 256 MB 한도에서 `RESOURCE_EXHAUSTED`로 실패하던 것을 동일 브랜치 내 plugin cache 버전과 일치시켜 해결. + +### capture insert 단계가 압도적 (~4.5초) + +`insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, pyenvector 1.4.0a5의 PPMM cache (1.31–1.40x 가속) 가 적용된 후의 수치임에도 절대값은 4.2–5.0초 수준. 다음 측정 라운드에서: +1. FHE encrypt 단독 시간 (로컬 CPU) +2. 원격 envector `insert_data` gRPC 시간 (네트워크 + 서버측 저장) +을 분리 계측하면 병목 위치가 더 명확해질 것으로 보임. + +### recall score 단계가 capture novelty score보다 ~2배 무거움 + +같은 `score` 호출인데도 capture 시 100 ms vs recall 시 180 ms. capture는 novelty check만 하고 단순 거리 metric을 받는 반면, recall은 그 결과를 가지고 후속 vault_topk + remind 분기로 연결되며 ScoreEntry payload가 달라지는 것으로 추정. 코드 경로 확인 필요. + +### T3 한국어 capture가 영문보다 score/vault_topk만 느림 + +| | embed | score | vault_topk | insert | +|--|-------|-------|------------|--------| +| T1 영문 | 92.6 | 100.4 | 31.9 | 5018.1 | +| T3 한국어 | 126.9 (+37%) | 167.5 (+67%) | 75.6 (+137%) | 4209.0 (-16%) | + +embed/score/vault_topk는 한국어가 모두 느리지만 insert는 오히려 더 빠름. 토큰 차이로 embed가 느린 것은 어제와 동일하나, score/vault_topk가 토큰 수에 영향받는 이유는 명확하지 않음 (이론적으로 FHE 벡터 길이가 동일해야 함). 토크나이저 출력의 vector dim이 다르거나, payload 길이 차이가 직렬화 오버헤드로 나타났을 가능성. + +### T4 중복 입력의 score가 다른 capture 시나리오의 2배 + +T1 score 100 ms vs T4 score 211 ms. T4는 같은 텍스트를 두 번 capture하는 시나리오로, novelty 결과가 high-similarity로 나오면서 vault_topk decrypt 분기가 더 무거운 path를 타는 것으로 보임. 어제 리포트의 "p95 spike 발생" 도 같은 원인으로 추정. + +### batch_capture가 단일 capture의 ~5% 시간 + +per-item 240 ms vs 단일 capture 5000 ms. 차이의 대부분은 batch_capture가 **insert를 수행하지 않고** embed + score(novelty)만 측정하기 때문. batch insert를 포함한 진짜 batch 시간이 필요하면 runner에 별도 시나리오 추가 필요. + +--- + +## Environment + +- **date**: 2026-05-05 +- **envector_endpoint**: 146.56.178.130:50050 +- **vault_endpoint**: tcp://localhost:50051 +- **embedding_model**: Qwen/Qwen3-Embedding-0.6B +- **embedding_mode**: sbert +- **index_name**: runecontext +- **key_id**: vault-key +- **network_rtt**: unknown +- **runs_per_scenario**: 8 +- **warmup_runs**: 2 + + +## Feature: `capture` + +### T1_short_en +- label: short English (~30 tokens) +- tokens_approx: 35 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 92.6 | 120.6 | 122.0 | 98.9 | +| score | 8 | 100.4 | 115.0 | 116.4 | 98.5 | +| vault_topk | 8 | 31.9 | 47.3 | 49.4 | 33.6 | +| insert | 8 | 5018.1 | 5242.2 | 5287.3 | 4951.1 | +| total | 8 | 5269.5 | 5449.8 | 5486.2 | 5182.1 | + +### T2_long_en +- label: long English (~150 tokens) +- tokens_approx: 155 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 100.6 | 139.3 | 144.6 | 108.0 | +| score | 8 | 135.7 | 146.0 | 147.1 | 134.3 | +| vault_topk | 8 | 56.0 | 64.4 | 64.7 | 54.9 | +| insert | 8 | 4406.8 | 4645.0 | 4695.4 | 4409.3 | +| total | 8 | 4707.0 | 4931.9 | 4976.3 | 4706.7 | + +### T3_korean +- label: Korean text +- tokens_approx: 50 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 126.9 | 147.3 | 149.4 | 128.3 | +| score | 8 | 167.5 | 184.9 | 187.2 | 168.7 | +| vault_topk | 8 | 75.6 | 91.1 | 92.6 | 76.4 | +| insert | 8 | 4209.0 | 4397.0 | 4405.8 | 4246.7 | +| total | 8 | 4595.2 | 4789.1 | 4806.0 | 4620.1 | + +### T4_duplicate +- label: duplicate input — tests novelty near-duplicate path +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 115.2 | 132.1 | 132.4 | 113.4 | +| score | 8 | 211.4 | 229.7 | 231.1 | 209.3 | +| vault_topk | 8 | 99.7 | 108.9 | 109.6 | 99.7 | +| insert | 8 | 4435.8 | 4568.8 | 4572.0 | 4446.1 | +| total | 8 | 4886.1 | 5020.2 | 5021.7 | 4868.4 | + + +## Feature: `recall` + +### T5_exact_match +- label: exact match query +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 40.4 | 41.2 | 41.3 | 39.7 | +| score | 8 | 182.7 | 200.0 | 204.1 | 185.3 | +| vault_topk | 8 | 98.5 | 105.7 | 106.0 | 99.7 | +| remind | 8 | 45.9 | 49.7 | 50.6 | 46.5 | +| total | 8 | 368.2 | 385.3 | 386.0 | 371.2 | + +### T6_cross_lang +- label: cross-language semantic (KO→EN) +- topk: 5 +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 8 | 41.4 | 47.1 | 47.3 | 42.6 | +| score | 8 | 195.4 | 217.7 | 221.0 | 197.5 | +| vault_topk | 8 | 104.6 | 157.1 | 177.1 | 114.1 | +| remind | 8 | 47.0 | 53.2 | 53.8 | 47.4 | +| total | 8 | 384.1 | 464.3 | 483.0 | 401.5 | + +### T7_topk_1 +- topk: 1 +- label: topk scaling topk=1 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 37.4 | 42.6 | 43.6 | 37.7 | +| score | 5 | 186.7 | 191.7 | 192.0 | 186.5 | +| vault_topk | 5 | 101.2 | 103.2 | 103.6 | 100.7 | +| remind | 5 | 43.5 | 45.1 | 45.3 | 43.0 | +| total | 5 | 367.7 | 375.1 | 375.3 | 367.9 | + +### T7_topk_3 +- topk: 3 +- label: topk scaling topk=3 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 38.0 | 38.5 | 38.6 | 36.8 | +| score | 5 | 181.0 | 191.1 | 191.5 | 182.7 | +| vault_topk | 5 | 105.4 | 109.0 | 109.2 | 104.4 | +| remind | 5 | 45.0 | 45.8 | 45.8 | 44.7 | +| total | 5 | 368.1 | 376.4 | 377.2 | 368.7 | + +### T7_topk_5 +- topk: 5 +- label: topk scaling topk=5 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 36.5 | 38.3 | 38.3 | 36.8 | +| score | 5 | 178.7 | 196.3 | 196.5 | 184.6 | +| vault_topk | 5 | 114.5 | 139.6 | 143.8 | 116.2 | +| remind | 5 | 44.8 | 46.1 | 46.3 | 44.9 | +| total | 5 | 376.4 | 402.2 | 403.8 | 382.4 | + +### T7_topk_10 +- topk: 10 +- label: topk scaling topk=10 +- runs: 5 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| embed | 5 | 38.2 | 39.3 | 39.5 | 37.2 | +| score | 5 | 181.9 | 190.2 | 190.6 | 183.8 | +| vault_topk | 5 | 101.8 | 106.0 | 106.4 | 102.6 | +| remind | 5 | 46.2 | 47.2 | 47.3 | 46.0 | +| total | 5 | 369.4 | 374.0 | 374.0 | 369.7 | + + +## Feature: `batch_capture` + +### T8_batch_1 +- batch_size: 1 +- label: batch size 1 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 237.9 | 242.2 | 242.6 | 239.2 | +| per_item | 3 | 237.9 | 242.2 | 242.6 | 239.2 | + +### T8_batch_5 +- batch_size: 5 +- label: batch size 5 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 1207.8 | 1209.7 | 1209.9 | 1207.5 | +| per_item | 3 | 241.6 | 241.9 | 242.0 | 241.5 | + +### T8_batch_10 +- batch_size: 10 +- label: batch size 10 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 2409.5 | 2437.7 | 2440.2 | 2415.5 | +| per_item | 3 | 240.9 | 243.8 | 244.0 | 241.5 | + +### T8_batch_20 +- batch_size: 20 +- label: batch size 20 (embed+score per item) +- runs: 3 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| total_batch | 3 | 4892.0 | 4907.7 | 4909.1 | 4846.8 | +| per_item | 3 | 244.6 | 245.4 | 245.5 | 242.3 | + + +## Feature: `vault_status` + +### T9_vault_status +- label: Vault gRPC health check +- runs: 8 + +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +|-------|---|--------|--------|--------|---------| +| vault_health_check | 8 | 0.8 | 0.9 | 0.9 | 0.8 | From d48fcaf3461d12d1814333309cf49b0b50ff8928 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Tue, 5 May 2026 20:52:05 +0900 Subject: [PATCH 08/16] =?UTF-8?q?docs(benchmark):=20correct=20pyenvector?= =?UTF-8?q?=20version=201.4.0a5=20=E2=86=92=201.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 설치/측정 환경의 실제 버전은 stable 1.4.0 (pre-release a5 아님). plan과 2026-05-05 리포트 둘 다 동일하게 수정. Co-Authored-By: Claude Opus 4.7 (1M context) --- benchmark/plans/latency_bench_plan_envector_v1.4.0.md | 2 +- benchmark/reports/latency_results_v1.4.0_2026-05-05.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md index 0a3a268f..727e2ed8 100644 --- a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md +++ b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md @@ -12,7 +12,7 @@ envector-msa-1.4.0은 이전 버전(rmp)에서 크게 변경되었다: - **KMS**: 독립 서비스로 분리 (Port 50060) - **Audit**: 독립 서비스로 분리 (Port 50052) -- **pyenvector 1.4.0a5**: `secure` 파라미터 추가, 암호화 최적화 (PPMM cache 1.31–1.40x) +- **pyenvector 1.4.0**: `secure` 파라미터 추가, 암호화 최적화 (PPMM cache 1.31–1.40x) - **gRPC-Gateway**: REST 전사(transcoding) 지원 rune의 현재 config는 `envector.endpoint = 146.56.178.130:50050` (원격 클라우드)를 사용하고 있으며, diff --git a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md index 78523655..ea847325 100644 --- a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md +++ b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md @@ -37,7 +37,7 @@ ### capture insert 단계가 압도적 (~4.5초) -`insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, pyenvector 1.4.0a5의 PPMM cache (1.31–1.40x 가속) 가 적용된 후의 수치임에도 절대값은 4.2–5.0초 수준. 다음 측정 라운드에서: +`insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, pyenvector 1.4.0의 PPMM cache (1.31–1.40x 가속) 가 적용된 후의 수치임에도 절대값은 4.2–5.0초 수준. 다음 측정 라운드에서: 1. FHE encrypt 단독 시간 (로컬 CPU) 2. 원격 envector `insert_data` gRPC 시간 (네트워크 + 서버측 저장) 을 분리 계측하면 병목 위치가 더 명확해질 것으로 보임. From 59c1be7e50dcd6410de993e41ca40e216efd604e Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Tue, 5 May 2026 21:02:13 +0900 Subject: [PATCH 09/16] docs(benchmark): cite envector-msa-1.4.0 sources for secure/PPMM claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit plan에 추상적으로 적혀있던 두 변경 사항의 정확한 출처와 적용 범위를 표기: - `secure` 파라미터: SDK Configuration 문서. - PPMM cache 최적화: query batch m=4–16에서 1.20–1.40x (V2) / 1.35–1.59x (V3), 단일 쿼리(m=1)에는 효과 없음. 검색(Search) 경로 전용. 리포트의 capture insert 분석에서 PPMM 인용을 제거. PPMM은 Search 경로 최적화이고 insert/encrypt 경로와는 무관함을 envector design doc로 확인. Co-Authored-By: Claude Opus 4.7 (1M context) --- benchmark/plans/latency_bench_plan_envector_v1.4.0.md | 8 +++++++- benchmark/reports/latency_results_v1.4.0_2026-05-05.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md index 727e2ed8..8fd0c1c4 100644 --- a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md +++ b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md @@ -12,7 +12,13 @@ envector-msa-1.4.0은 이전 버전(rmp)에서 크게 변경되었다: - **KMS**: 독립 서비스로 분리 (Port 50060) - **Audit**: 독립 서비스로 분리 (Port 50052) -- **pyenvector 1.4.0**: `secure` 파라미터 추가, 암호화 최적화 (PPMM cache 1.31–1.40x) +- **pyenvector 1.4.0**: + - `secure` 파라미터 추가 — TLS 활성/비활성 토글, 기본 False(`access_token` 설정 시 True). + 출처: `envector-msa-1.4.0/docs/architecture/components/sdk.md` Configuration 섹션. + - 검색(Search) 경로의 PPMM native 연산 cache-aware 최적화 추가 — query batch m=4–16에서 + 1.20–1.40x (V2: loop swap), 1.35–1.59x (V3: swap+j-tile) speedup. 단일 쿼리(m=1)에는 효과 없음. + 출처: `envector-msa-1.4.0/docs/design/ppmm-cache-optimization-analysis.md` + (Solution Dev Team, 2026-03-09). - **gRPC-Gateway**: REST 전사(transcoding) 지원 rune의 현재 config는 `envector.endpoint = 146.56.178.130:50050` (원격 클라우드)를 사용하고 있으며, diff --git a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md index ea847325..73528b75 100644 --- a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md +++ b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md @@ -37,7 +37,7 @@ ### capture insert 단계가 압도적 (~4.5초) -`insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, pyenvector 1.4.0의 PPMM cache (1.31–1.40x 가속) 가 적용된 후의 수치임에도 절대값은 4.2–5.0초 수준. 다음 측정 라운드에서: +`insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, 절대값은 4.2–5.0초 수준. (참고: envector-msa-1.4.0의 PPMM cache 최적화는 Search 경로 전용이며 insert와 무관 — `envector-msa-1.4.0/docs/design/ppmm-cache-optimization-analysis.md`.) 다음 측정 라운드에서: 1. FHE encrypt 단독 시간 (로컬 CPU) 2. 원격 envector `insert_data` gRPC 시간 (네트워크 + 서버측 저장) 을 분리 계측하면 병목 위치가 더 명확해질 것으로 보임. From 83e8cf9598b0fee70563c872abe0a16b0e0e9125 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Tue, 5 May 2026 21:05:32 +0900 Subject: [PATCH 10/16] =?UTF-8?q?docs(benchmark):=20tone=20down=20"?= =?UTF-8?q?=EC=95=95=EB=8F=84=EC=A0=81"=20to=20neutral=20phrasing=20in=20c?= =?UTF-8?q?apture=20insert=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 데이터(~92%)는 사실로 유지하되 헤딩 표현을 중립적으로 조정. Co-Authored-By: Claude Opus 4.7 (1M context) --- benchmark/reports/latency_results_v1.4.0_2026-05-05.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md index 73528b75..5e585393 100644 --- a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md +++ b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md @@ -35,7 +35,7 @@ **부수적으로 확인된 인프라 변경**: `mcp/adapter/vault_client.py:33`의 `MAX_MESSAGE_LENGTH`가 256 MB → ~1.95 GB로 상향되어야 했음. EvalKey 응답 크기가 약 1.18 GB에 달해 256 MB 한도에서 `RESOURCE_EXHAUSTED`로 실패하던 것을 동일 브랜치 내 plugin cache 버전과 일치시켜 해결. -### capture insert 단계가 압도적 (~4.5초) +### capture 시간 대부분이 insert에서 소비됨 (~4.5초, 전체의 92%) `insert` phase가 capture 전체 시간의 92%. 내부적으로 FHE 암호화 + 원격 envector insert가 합쳐진 구간이며, 절대값은 4.2–5.0초 수준. (참고: envector-msa-1.4.0의 PPMM cache 최적화는 Search 경로 전용이며 insert와 무관 — `envector-msa-1.4.0/docs/design/ppmm-cache-optimization-analysis.md`.) 다음 측정 라운드에서: 1. FHE encrypt 단독 시간 (로컬 CPU) From 654806e0b1c0f54e267a36fbf66719a2a34fc71d Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Wed, 6 May 2026 13:59:16 +0900 Subject: [PATCH 11/16] fix(vault): use separate 90s deadline for GetPublicKey GetPublicKey streams ~1.1 GiB EvalKey for pyenvector 1.4 and exceeds the shared 30s default deadline (measured ~38s on a 30 MiB/s link). Companion to d08e1bb, which raised the gRPC message size limit but left the deadline untouched. Standard RPC timeout stays at 30s. Co-Authored-By: Claude Opus 4.7 (1M context) --- mcp/adapter/vault_client.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mcp/adapter/vault_client.py b/mcp/adapter/vault_client.py index a8c791c7..b305b555 100644 --- a/mcp/adapter/vault_client.py +++ b/mcp/adapter/vault_client.py @@ -82,6 +82,7 @@ def __init__( vault_endpoint: str, vault_token: str, timeout: float = 30.0, + public_key_timeout: float = 90.0, ca_cert: Optional[str] = None, tls_disable: bool = False, ): @@ -93,7 +94,11 @@ def __init__( "host:port" (direct), "tcp://host:port", or legacy "http://host:50080/mcp". RUNEVAULT_GRPC_TARGET overrides. vault_token: Authentication token for Vault - timeout: Request timeout in seconds + timeout: Request timeout in seconds for standard RPCs. + public_key_timeout: Deadline for GetPublicKey, which streams + the multi-GiB EvalKey bundle (pyenvector 1.4 EvalKey is + ~1.1 GiB). Measured baseline ~38s on a 30 MiB/s link; + default tolerates down to ~12.5 MiB/s networks. ca_cert: Path to CA certificate PEM file for self-signed certs. None or empty string uses system CA bundle. tls_disable: If True, use insecure plaintext channel (dev only). @@ -101,6 +106,7 @@ def __init__( self.vault_endpoint = vault_endpoint.rstrip("/") self.vault_token = vault_token self.timeout = timeout + self.public_key_timeout = public_key_timeout self._ca_cert = ca_cert self._tls_disable = tls_disable @@ -203,7 +209,7 @@ async def get_public_key(self) -> dict: try: request = pb2.GetPublicKeyRequest(token=self.vault_token) response = await self._stub.GetPublicKey( - request, timeout=self.timeout + request, timeout=self.public_key_timeout ) if response.error: raise VaultError(f"GetPublicKey failed: {response.error}") From 01b92b5d4c3a3583f67d58bbffb1857bd1386fca Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Wed, 6 May 2026 14:32:32 +0900 Subject: [PATCH 12/16] docs(benchmark): normalize markdown formatting across benchmark docs Apply Prettier-style cleanup to all .md files modified vs main on this branch: pad table cells to equal visual width (CJK-aware), insert blank lines before fenced code blocks and tables, and drop trailing two-space hard breaks where the next line is already a block boundary. Intentional hard breaks in paragraph-fragment lists are preserved. No content changes. --- benchmark/README.md | 100 ++++---- .../latency_bench_plan_envector_v1.4.0.md | 64 +++-- benchmark/reports/latency_results_v1.4.0.md | 219 +++++++++--------- .../latency_results_v1.4.0_2026-05-05.md | 206 ++++++++-------- 4 files changed, 315 insertions(+), 274 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 510cbbf1..95b45d61 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -8,12 +8,12 @@ Since v0.2.0, Rune operates in **agent-delegated mode**: the calling agent (Clau This means **capture quality is determined by the scribe prompt**, not by Rune's internal modules. The benchmark reflects this: -| Benchmark | What it tests | Method | -|-----------|---------------|--------| -| **scribe_bench** (capture) | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON | -| **scribe_bench** (extraction) | Is the extracted JSON well-structured? | Same LLM call, score field coverage and extraction type | -| **retriever_bench** | Can stored decisions be retrieved with various queries? | Offline embedding similarity (FHE is transparent to scores) | -| **latency_bench** | How long does each pipeline phase take? | Direct adapter calls against live envector-msa-1.4.0, per-phase timing | +| Benchmark | What it tests | Method | +| ----------------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------- | +| **scribe_bench** (capture) | Does the scribe prompt correctly guide capture/skip decisions? | Feed scribe prompt + scenario input to LLM, score the output JSON | +| **scribe_bench** (extraction) | Is the extracted JSON well-structured? | Same LLM call, score field coverage and extraction type | +| **retriever_bench** | Can stored decisions be retrieved with various queries? | Offline embedding similarity (FHE is transparent to scores) | +| **latency_bench** | How long does each pipeline phase take? | Direct adapter calls against live envector-msa-1.4.0, per-phase timing | ## Running @@ -76,12 +76,12 @@ python benchmark/runners/latency_bench.py \ Pipeline phases measured per feature: -| Feature | Phases | -|---------|--------| -| `capture` | embed → score (FHE search) → vault_topk → insert → total | -| `recall` | embed → score → vault_topk → remind → total | -| `batch_capture` | total_batch, per_item (embed+score per item) | -| `vault_status` | vault_health_check | +| Feature | Phases | +| --------------- | -------------------------------------------------------- | +| `capture` | embed → score (FHE search) → vault_topk → insert → total | +| `recall` | embed → score → vault_topk → remind → total | +| `batch_capture` | total_batch, per_item (embed+score per item) | +| `vault_status` | vault_health_check | ## Scenarios (104 total) @@ -140,49 +140,49 @@ When Rune doesn't behave as expected — a decision was missed, noise was captur #### Capture scenario fields -| Field | Required | Description | -|-------|----------|-------------| -| `id` | yes | Unique identifier. Format: `{category}-{description}-{number}` (e.g., `debug-grpc-timeout-009`) | -| `category` | yes | Directory path under `scenarios/`. Must match where the file lives (e.g., `capture/should_capture/debugging`) | -| `language` | yes | Language of the input: `en`, `ko`, `ja`, or `mixed` | -| `input` | yes | The actual conversation text to evaluate. Paste the real message as-is | -| `expected_capture` | yes | `true` if this should be captured, `false` if it should be skipped | -| `expected_fields` | no | Expected metadata when captured (see below). Use `{}` for should_not_capture scenarios | -| `expected_fields.domain` | no | Expected domain classification: `architecture`, `security`, `product`, `debugging`, `incident`, `ops`, `process`, `qa`, `hr`, `data`, `finance`, `general`, etc. | -| `expected_fields.status_hint` | no | Expected decision status: `accepted`, `proposed`, or `rejected` | -| `expected_fields.title_keywords` | no | List of keywords that should appear in the extracted title (case-insensitive, any match passes) | -| `recall_queries` | no | Optional queries that should (or should not) retrieve this decision after capture | -| `notes` | no | Free-text annotation explaining why this scenario is interesting or tricky | +| Field | Required | Description | +| -------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | yes | Unique identifier. Format: `{category}-{description}-{number}` (e.g., `debug-grpc-timeout-009`) | +| `category` | yes | Directory path under `scenarios/`. Must match where the file lives (e.g., `capture/should_capture/debugging`) | +| `language` | yes | Language of the input: `en`, `ko`, `ja`, or `mixed` | +| `input` | yes | The actual conversation text to evaluate. Paste the real message as-is | +| `expected_capture` | yes | `true` if this should be captured, `false` if it should be skipped | +| `expected_fields` | no | Expected metadata when captured (see below). Use `{}` for should_not_capture scenarios | +| `expected_fields.domain` | no | Expected domain classification: `architecture`, `security`, `product`, `debugging`, `incident`, `ops`, `process`, `qa`, `hr`, `data`, `finance`, `general`, etc. | +| `expected_fields.status_hint` | no | Expected decision status: `accepted`, `proposed`, or `rejected` | +| `expected_fields.title_keywords` | no | List of keywords that should appear in the extracted title (case-insensitive, any match passes) | +| `recall_queries` | no | Optional queries that should (or should not) retrieve this decision after capture | +| `notes` | no | Free-text annotation explaining why this scenario is interesting or tricky | #### Retriever scenario fields -| Field | Required | Description | -|-------|----------|-------------| -| `id` | yes | Format: `recall-{subcategory}-{description}-{number}` | -| `category` | yes | One of: `recall/exact_match`, `recall/semantic_match`, `recall/cross_domain`, `recall/temporal` | -| `language` | yes | Language of the query | -| `seed_records` | yes | Array of records to index before querying. Each has `title`, `domain`, `content`, and optionally `tags` | -| `query` | yes | The recall query to test | -| `expected_match_titles` | yes | Titles of seed records that should appear in results above `min_score` | -| `min_score` | no | Minimum cosine similarity threshold (default: 0.35). Lower for semantic/cross-domain tests | -| `notes` | no | Free-text annotation | +| Field | Required | Description | +| ----------------------- | -------- | ------------------------------------------------------------------------------------------------------- | +| `id` | yes | Format: `recall-{subcategory}-{description}-{number}` | +| `category` | yes | One of: `recall/exact_match`, `recall/semantic_match`, `recall/cross_domain`, `recall/temporal` | +| `language` | yes | Language of the query | +| `seed_records` | yes | Array of records to index before querying. Each has `title`, `domain`, `content`, and optionally `tags` | +| `query` | yes | The recall query to test | +| `expected_match_titles` | yes | Titles of seed records that should appear in results above `min_score` | +| `min_score` | no | Minimum cosine similarity threshold (default: 0.35). Lower for semantic/cross-domain tests | +| `notes` | no | Free-text annotation | #### Extraction scenario fields -| Field | Required | Description | -|-------|----------|-------------| -| `id` | yes | Format: `extract-{type}-{description}-{number}` | -| `category` | yes | One of: `extraction/single`, `extraction/phase_chain`, `extraction/bundle` | -| `language` | yes | Language of the input | -| `input` | yes | The conversation text to extract from | -| `expected_extraction_type` | yes | Expected extraction mode: `single`, `phase_chain`, or `bundle` | -| `expected_fields.title_keywords` | no | Keywords expected in extracted title | -| `expected_fields.status_hint` | no | Expected status: `accepted`, `proposed`, `rejected` | -| `expected_fields.min_alternatives` | no | Minimum number of alternatives that should be extracted | -| `expected_fields.min_trade_offs` | no | Minimum number of trade-offs that should be extracted | -| `expected_fields.min_phases` | no | Minimum phase count (for phase_chain/bundle) | -| `expected_fields.max_phases` | no | Maximum phase count (for phase_chain/bundle) | -| `notes` | no | Free-text annotation | +| Field | Required | Description | +| ---------------------------------- | -------- | -------------------------------------------------------------------------- | +| `id` | yes | Format: `extract-{type}-{description}-{number}` | +| `category` | yes | One of: `extraction/single`, `extraction/phase_chain`, `extraction/bundle` | +| `language` | yes | Language of the input | +| `input` | yes | The conversation text to extract from | +| `expected_extraction_type` | yes | Expected extraction mode: `single`, `phase_chain`, or `bundle` | +| `expected_fields.title_keywords` | no | Keywords expected in extracted title | +| `expected_fields.status_hint` | no | Expected status: `accepted`, `proposed`, `rejected` | +| `expected_fields.min_alternatives` | no | Minimum number of alternatives that should be extracted | +| `expected_fields.min_trade_offs` | no | Minimum number of trade-offs that should be extracted | +| `expected_fields.min_phases` | no | Minimum phase count (for phase_chain/bundle) | +| `expected_fields.max_phases` | no | Maximum phase count (for phase_chain/bundle) | +| `notes` | no | Free-text annotation | ### 1. Capture Scenarios @@ -225,6 +225,8 @@ Extraction produced the wrong structure (e.g., single decision extracted as phas - **Paste real conversations** as input — real-world data makes better benchmarks than synthetic examples - **Redact sensitive data**: mask API keys, passwords, and PII before adding - Verify immediately after adding: + ```bash python benchmark/runners/scribe_bench.py --category debugging -v + ``` diff --git a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md index 8fd0c1c4..1be1d167 100644 --- a/benchmark/plans/latency_bench_plan_envector_v1.4.0.md +++ b/benchmark/plans/latency_bench_plan_envector_v1.4.0.md @@ -10,8 +10,9 @@ 이번 작업은 rune의 각 MCP 기능을 실행했을 때 **실제 응답 시간(latency)**을 측정하는 것이 목표다. envector-msa-1.4.0은 이전 버전(rmp)에서 크게 변경되었다: -- **KMS**: 독립 서비스로 분리 (Port 50060) -- **Audit**: 독립 서비스로 분리 (Port 50052) + +- **KMS**: 독립 서비스로 분리 (Port 50060) +- **Audit**: 독립 서비스로 분리 (Port 50052) - **pyenvector 1.4.0**: - `secure` 파라미터 추가 — TLS 활성/비활성 토글, 기본 False(`access_token` 설정 시 True). 출처: `envector-msa-1.4.0/docs/architecture/components/sdk.md` Configuration 섹션. @@ -31,6 +32,7 @@ vault는 `tcp://localhost:50051`에서 로컬 gRPC로 실행된다. ### Feature 1: `capture` (MCP tool) 전체 파이프라인: + ``` [1] 텍스트 → Embedding (로컬 fastembed, Qwen/Qwen3-Embedding-0.6B) [2] Novelty Check → envector inner_product (FHE 암호화 쿼리 전송) @@ -44,6 +46,7 @@ Total end-to-end ### Feature 2: `recall` (MCP tool) 전체 파이프라인: + ``` [1] 쿼리 → Embedding (로컬) [2] Encrypted Search → envector inner_product @@ -68,6 +71,7 @@ Total end-to-end ## 구체적 테스트 시나리오 (각 기능별) ### [T1] Capture — 짧은 영문 텍스트 (baseline) + ```json { "input": "We decided to use PostgreSQL as our primary database. Team familiarity and mature ecosystem were the key reasons. Redis considered but rejected due to durability concerns.", @@ -75,9 +79,11 @@ Total end-to-end "expected_insight": "PostgreSQL chosen over Redis for primary DB due to team familiarity and durability" } ``` + 반복: 10회 (첫 2회 warmup 제외) ### [T2] Capture — 긴 영문 텍스트 (embedding 부하 테스트) + ```json { "input": "Full ADR: Context — our monolith hit 10k RPS limits. Considered: (1) horizontal scale with read replicas, (2) CQRS split, (3) microservices decomposition. Trade-offs: read replicas cheapest but doesn't solve write bottleneck; CQRS complex but keeps single codebase; microservices highest ops overhead. Decision: CQRS with event sourcing on order-service first. Rationale: allows independent scaling of read path, event log gives audit trail for compliance (legal requirement from Q2 review). Rollback plan: feature flag, revert within 2 sprints if P99 > 500ms.", @@ -85,9 +91,11 @@ Total end-to-end "label": "long_text_~150_tokens" } ``` + 반복: 10회 ### [T3] Capture — 한국어 텍스트 (multi-language) + ```json { "input": "Redis를 캐시 레이어로 사용하기로 결정했습니다. Memcached도 검토했지만 데이터 구조 지원(Sorted Set, List)이 필요해서 Redis로 확정. TTL은 1시간으로 설정. 담당: 백엔드팀.", @@ -95,14 +103,17 @@ Total end-to-end "label": "korean_text" } ``` + 반복: 10회 ### [T4] Capture — 중복 입력 (novelty check 동작 확인) + - T1의 입력을 두 번 연속 capture - 두 번째 호출의 novelty check 결과(near-duplicate) 및 응답 시간 비교 - 중복 스킵 여부 확인 ### [T5] Recall — Exact match 쿼리 + ```json { "query": "Why did we choose PostgreSQL?", @@ -110,9 +121,11 @@ Total end-to-end "label": "exact_match" } ``` + 반복: 10회 ### [T6] Recall — Semantic match 쿼리 (의미 기반) + ```json { "query": "데이터베이스 선택 이유", @@ -120,9 +133,11 @@ Total end-to-end "label": "cross_language_semantic" } ``` + 반복: 10회 ### [T7] Recall — topk 변화에 따른 latency + ```json { "query": "architecture decisions", @@ -130,9 +145,11 @@ Total end-to-end "label": "topk_scaling" } ``` + 각 topk 당 5회 반복 ### [T8] Batch Capture — batch size 스케일링 + ```json { "batch_sizes": [1, 5, 10, 20], @@ -140,14 +157,17 @@ Total end-to-end "label": "batch_scaling" } ``` + 각 batch size 당 3회 반복 ### [T9] Vault Status / Diagnostics 헬스체크 + - `vault_status` 10회 반복 - `diagnostics` 10회 반복 - Vault gRPC cold/warm 연결 구분 ### [T10] Memory 크기별 Recall latency (회귀 테스트) + - 빈 index에서 recall - 10개 capture 후 recall - 50개 capture 후 recall @@ -158,12 +178,14 @@ Total end-to-end ## 측정 방법론 ### 통계 기준 + - **반복**: 최소 10회 (warmup 2회 제외하여 8회 유효) - **보고 지표**: p50, p95, p99, mean, min, max (ms 단위) - **타이머**: `time.perf_counter()` (Python 고정밀 타이머) - **단계별 측정**: 각 파이프라인 단계를 개별 타이머로 계측 ### 환경 조건 + - envector-msa-1.4.0: 원격 클라우드 (`146.56.178.130:50050`, `secure=false`) - Vault: 로컬 (`tcp://localhost:50051`) - Embedding model: `Qwen/Qwen3-Embedding-0.6B` (로컬) @@ -177,6 +199,7 @@ Total end-to-end 기존 `common.py`의 `BenchmarkReport` / `ScenarioResult` 재사용. 새로 추가할 구조: + ```python @dataclass class LatencyMeasurement: @@ -198,6 +221,7 @@ class LatencyBenchReport: **측정 방식**: MCP server를 직접 import하지 않고, `mcp/adapter/envector_sdk.py`와 `mcp/adapter/vault_client.py`를 직접 호출하는 방식으로 각 단계 계측. **실행 방법**: + ```bash # 모든 시나리오 python benchmark/runners/latency_bench.py @@ -218,10 +242,12 @@ python benchmark/runners/latency_bench.py --report benchmark/reports/latency_res ### 2. `benchmark/reports/latency_results_v1.4.0.md` (보고서) 테스트 실행 후 자동 생성. 형식: + ```markdown # Rune × envector-msa-1.4.0 Latency Report ## Environment + - Date: 2026-05-XX - envector endpoint: 146.56.178.130:50050 - Vault: tcp://localhost:50051 @@ -231,13 +257,14 @@ python benchmark/runners/latency_bench.py --report benchmark/reports/latency_res ## Feature: capture ### T1: Short English text (~30 tokens) -| Phase | p50 | p95 | p99 | mean | -|-------------------|-------|-------|-------|-------| -| embedding | | | | | -| novelty_check | | | | | -| fhe_encrypt+insert| | | | | -| vault_topk_decrypt| | | | | -| **total** | | | | | + +| Phase | p50 | p95 | p99 | mean | +| ------------------ | --- | --- | --- | ---- | +| embedding | | | | | +| novelty_check | | | | | +| fhe_encrypt+insert | | | | | +| vault_topk_decrypt | | | | | +| **total** | | | | | ... ``` @@ -264,22 +291,23 @@ ping -c 5 146.56.178.130 ## 검증 방법 -1. **단계별 합계 일치**: `sum(phase latencies) ≈ total` (±5% 허용) -2. **재현성**: 같은 시나리오를 서로 다른 시간에 실행했을 때 p50 변동 < 20% -3. **스케일 선형성 확인**: T8 batch_capture에서 item당 latency가 batch_size 증가에도 일정한지 확인 -4. **중복 감지 동작**: T4에서 두 번째 capture가 "near_duplicate"로 skip되는지 확인 +1. **단계별 합계 일치**: `sum(phase latencies) ≈ total` (±5% 허용) +2. **재현성**: 같은 시나리오를 서로 다른 시간에 실행했을 때 p50 변동 < 20% +3. **스케일 선형성 확인**: T8 batch_capture에서 item당 latency가 batch_size 증가에도 일정한지 확인 +4. **중복 감지 동작**: T4에서 두 번째 capture가 "near_duplicate"로 skip되는지 확인 --- ## 수정할 파일 목록 -| 파일 | 작업 | -|------|------| -| `benchmark/runners/latency_bench.py` | 신규 생성 | -| `benchmark/reports/latency_results_v1.4.0.md` | 실행 후 자동 생성 | -| `benchmark/runners/common.py` | `LatencyMeasurement`, `LatencyBenchReport` dataclass 추가 | +| 파일 | 작업 | +| --------------------------------------------- | --------------------------------------------------------- | +| `benchmark/runners/latency_bench.py` | 신규 생성 | +| `benchmark/reports/latency_results_v1.4.0.md` | 실행 후 자동 생성 | +| `benchmark/runners/common.py` | `LatencyMeasurement`, `LatencyBenchReport` dataclass 추가 | 수정하지 않는 파일: + - `mcp/adapter/envector_sdk.py` — 직접 호출하되 수정 없음 - `mcp/adapter/vault_client.py` — 직접 호출하되 수정 없음 - `agents/common/embedding_service.py` — 직접 호출하되 수정 없음 diff --git a/benchmark/reports/latency_results_v1.4.0.md b/benchmark/reports/latency_results_v1.4.0.md index 41846d0c..fb381578 100644 --- a/benchmark/reports/latency_results_v1.4.0.md +++ b/benchmark/reports/latency_results_v1.4.0.md @@ -6,17 +6,17 @@ ## 실행 결과 요약 -| 시나리오 | feature | p50 total | p95 total | n | -|----------|---------|-----------|-----------|---| -| T1 짧은 영문 (~35 tokens) | capture | 102.8 ms | 104.8 ms | 8 | -| T2 긴 영문 (~155 tokens) | capture | 123.2 ms | 126.8 ms | 8 | -| T3 한국어 | capture | 155.0 ms | 165.3 ms | 8 | -| T4 중복 입력 (near-duplicate path) | capture | 106.8 ms | 201.2 ms | 8 | -| T5 Recall — exact match | recall | 37.3 ms | 40.4 ms | 8 | -| T6 Recall — 한→영 cross-language | recall | 38.4 ms | 39.3 ms | 8 | -| T7 Recall — topk 1/3/5/10 | recall | ~39–42 ms | ~42–46 ms | 5 each | -| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~50 ms | ~50 ms | 3 each | -| T9 Vault health check | vault_status | 0.6 ms | 0.8 ms | 8 | +| 시나리오 | feature | p50 total | p95 total | n | +| ---------------------------------- | ------------- | --------- | --------- | ------ | +| T1 짧은 영문 (~35 tokens) | capture | 102.8 ms | 104.8 ms | 8 | +| T2 긴 영문 (~155 tokens) | capture | 123.2 ms | 126.8 ms | 8 | +| T3 한국어 | capture | 155.0 ms | 165.3 ms | 8 | +| T4 중복 입력 (near-duplicate path) | capture | 106.8 ms | 201.2 ms | 8 | +| T5 Recall — exact match | recall | 37.3 ms | 40.4 ms | 8 | +| T6 Recall — 한→영 cross-language | recall | 38.4 ms | 39.3 ms | 8 | +| T7 Recall — topk 1/3/5/10 | recall | ~39–42 ms | ~42–46 ms | 5 each | +| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~50 ms | ~50 ms | 3 each | +| T9 Vault health check | vault_status | 0.6 ms | 0.8 ms | 8 | **capture 병목**: embedding (42–70 ms) + insert (52–78 ms) 가 전체의 ~94%를 차지. **recall 병목**: embedding (29–32 ms) + score (8–11 ms) 가 전체의 ~100%. @@ -70,191 +70,202 @@ T1(42 ms) → T3(69 ms): 동일 모델(Qwen3-Embedding-0.6B)에서 한국어 텍 - **runs_per_scenario**: 8 - **warmup_runs**: 2 - ## Feature: `capture` ### T1_short_en + - label: short English (~30 tokens) - tokens_approx: 35 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 42.1 | 42.7 | 42.7 | 42.1 | -| score | 8 | 9.0 | 10.0 | 10.2 | 8.8 | -| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| insert | 8 | 51.7 | 54.0 | 54.5 | 51.8 | -| total | 8 | 102.8 | 104.8 | 105.1 | 102.7 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 42.1 | 42.7 | 42.7 | 42.1 | +| score | 8 | 9.0 | 10.0 | 10.2 | 8.8 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 51.7 | 54.0 | 54.5 | 51.8 | +| total | 8 | 102.8 | 104.8 | 105.1 | 102.7 | ### T2_long_en + - label: long English (~150 tokens) - tokens_approx: 155 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 52.8 | 53.5 | 53.6 | 53.0 | -| score | 8 | 8.5 | 10.8 | 11.4 | 8.7 | -| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| insert | 8 | 62.0 | 63.6 | 63.6 | 61.8 | -| total | 8 | 123.2 | 126.8 | 127.5 | 123.5 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 52.8 | 53.5 | 53.6 | 53.0 | +| score | 8 | 8.5 | 10.8 | 11.4 | 8.7 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 62.0 | 63.6 | 63.6 | 61.8 | +| total | 8 | 123.2 | 126.8 | 127.5 | 123.5 | ### T3_korean + - label: Korean text - tokens_approx: 50 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 69.2 | 69.7 | 69.7 | 69.1 | -| score | 8 | 8.8 | 10.1 | 10.2 | 8.8 | -| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| insert | 8 | 77.6 | 86.3 | 89.3 | 79.1 | -| total | 8 | 155.0 | 165.3 | 168.6 | 157.0 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 69.2 | 69.7 | 69.7 | 69.1 | +| score | 8 | 8.8 | 10.1 | 10.2 | 8.8 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 77.6 | 86.3 | 89.3 | 79.1 | +| total | 8 | 155.0 | 165.3 | 168.6 | 157.0 | ### T4_duplicate + - label: duplicate input — tests novelty near-duplicate path - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 42.3 | 43.3 | 43.6 | 42.4 | -| score | 8 | 9.5 | 72.8 | 74.4 | 25.5 | -| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| insert | 8 | 54.6 | 87.5 | 91.6 | 61.8 | -| total | 8 | 106.8 | 201.2 | 203.3 | 129.7 | - +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 42.3 | 43.3 | 43.6 | 42.4 | +| score | 8 | 9.5 | 72.8 | 74.4 | 25.5 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| insert | 8 | 54.6 | 87.5 | 91.6 | 61.8 | +| total | 8 | 106.8 | 201.2 | 203.3 | 129.7 | ## Feature: `recall` ### T5_exact_match + - label: exact match query - topk: 5 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 28.7 | 29.8 | 30.0 | 28.7 | -| score | 8 | 8.7 | 11.1 | 11.5 | 9.1 | -| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| total | 8 | 37.3 | 40.4 | 41.0 | 37.8 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 28.7 | 29.8 | 30.0 | 28.7 | +| score | 8 | 8.7 | 11.1 | 11.5 | 9.1 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 8 | 37.3 | 40.4 | 41.0 | 37.8 | ### T6_cross_lang + - label: cross-language semantic (KO→EN) - topk: 5 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 30.1 | 30.5 | 30.6 | 30.0 | -| score | 8 | 8.3 | 9.5 | 9.6 | 8.3 | -| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | -| total | 8 | 38.4 | 39.3 | 39.4 | 38.3 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 30.1 | 30.5 | 30.6 | 30.0 | +| score | 8 | 8.3 | 9.5 | 9.6 | 8.3 | +| vault_topk | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 8 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 8 | 38.4 | 39.3 | 39.4 | 38.3 | ### T7_topk_1 + - topk: 1 - label: topk scaling topk=1 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 31.6 | 33.4 | 33.4 | 31.7 | -| score | 5 | 8.1 | 12.2 | 12.6 | 9.2 | -| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| total | 5 | 39.8 | 45.2 | 46.0 | 40.8 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 31.6 | 33.4 | 33.4 | 31.7 | +| score | 5 | 8.1 | 12.2 | 12.6 | 9.2 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 39.8 | 45.2 | 46.0 | 40.8 | ### T7_topk_3 + - topk: 3 - label: topk scaling topk=3 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 30.3 | 30.6 | 30.6 | 29.6 | -| score | 5 | 11.0 | 15.2 | 15.3 | 11.8 | -| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| total | 5 | 41.7 | 45.5 | 45.6 | 41.4 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 30.3 | 30.6 | 30.6 | 29.6 | +| score | 5 | 11.0 | 15.2 | 15.3 | 11.8 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 41.7 | 45.5 | 45.6 | 41.4 | ### T7_topk_5 + - topk: 5 - label: topk scaling topk=5 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 29.2 | 29.7 | 29.7 | 29.1 | -| score | 5 | 9.8 | 13.2 | 13.7 | 10.1 | -| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| total | 5 | 39.3 | 42.3 | 42.9 | 39.2 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 29.2 | 29.7 | 29.7 | 29.1 | +| score | 5 | 9.8 | 13.2 | 13.7 | 10.1 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 39.3 | 42.3 | 42.9 | 39.2 | ### T7_topk_10 + - topk: 10 - label: topk scaling topk=10 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 28.6 | 30.2 | 30.4 | 28.8 | -| score | 5 | 8.5 | 11.1 | 11.3 | 9.3 | -| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | -| total | 5 | 38.9 | 39.5 | 39.6 | 38.1 | - +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 28.6 | 30.2 | 30.4 | 28.8 | +| score | 5 | 8.5 | 11.1 | 11.3 | 9.3 | +| vault_topk | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| remind | 5 | 0.0 | 0.0 | 0.0 | 0.0 | +| total | 5 | 38.9 | 39.5 | 39.6 | 38.1 | ## Feature: `batch_capture` ### T8_batch_1 + - batch_size: 1 - label: batch size 1 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 49.9 | 50.1 | 50.2 | 49.4 | -| per_item | 3 | 49.9 | 50.1 | 50.2 | 49.4 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 49.9 | 50.1 | 50.2 | 49.4 | +| per_item | 3 | 49.9 | 50.1 | 50.2 | 49.4 | ### T8_batch_5 + - batch_size: 5 - label: batch size 5 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 248.9 | 249.3 | 249.4 | 248.4 | -| per_item | 3 | 49.8 | 49.9 | 49.9 | 49.7 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 248.9 | 249.3 | 249.4 | 248.4 | +| per_item | 3 | 49.8 | 49.9 | 49.9 | 49.7 | ### T8_batch_10 + - batch_size: 10 - label: batch size 10 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 499.2 | 499.8 | 499.8 | 499.2 | -| per_item | 3 | 49.9 | 50.0 | 50.0 | 49.9 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 499.2 | 499.8 | 499.8 | 499.2 | +| per_item | 3 | 49.9 | 50.0 | 50.0 | 49.9 | ### T8_batch_20 + - batch_size: 20 - label: batch size 20 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 1018.2 | 1019.0 | 1019.0 | 1016.5 | -| per_item | 3 | 50.9 | 50.9 | 51.0 | 50.8 | - +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 1018.2 | 1019.0 | 1019.0 | 1016.5 | +| per_item | 3 | 50.9 | 50.9 | 51.0 | 50.8 | ## Feature: `vault_status` ### T9_vault_status + - label: Vault gRPC health check - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| vault_health_check | 8 | 0.6 | 0.8 | 0.8 | 0.7 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ------------------ | --- | ------ | ------ | ------ | ------- | +| vault_health_check | 8 | 0.6 | 0.8 | 0.8 | 0.7 | diff --git a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md index 5e585393..eb9637fa 100644 --- a/benchmark/reports/latency_results_v1.4.0_2026-05-05.md +++ b/benchmark/reports/latency_results_v1.4.0_2026-05-05.md @@ -8,17 +8,17 @@ ## 실행 결과 요약 -| 시나리오 | feature | p50 total | p95 total | n | Δ(2026-05-04 대비) | -|----------|---------|-----------|-----------|---|------| -| T1 짧은 영문 (~35 tokens) | capture | 5269.5 ms | 5449.8 ms | 8 | +5167 ms (×51) | -| T2 긴 영문 (~155 tokens) | capture | 4707.0 ms | 4931.9 ms | 8 | +4584 ms (×38) | -| T3 한국어 | capture | 4595.2 ms | 4789.1 ms | 8 | +4440 ms (×30) | -| T4 중복 입력 (near-duplicate path) | capture | 4886.1 ms | 5020.2 ms | 8 | +4779 ms (×46) | -| T5 Recall — exact match | recall | 368.2 ms | 385.3 ms | 8 | +331 ms (×9.9) | -| T6 Recall — 한→영 cross-language | recall | 384.1 ms | 464.3 ms | 8 | +346 ms (×10.0) | -| T7 Recall — topk 1/3/5/10 | recall | ~367–376 ms | ~374–402 ms | 5 each | ×9.2–×9.6 | -| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~238–245 ms | ~242–246 ms | 3 each | ×4.8 | -| T9 Vault health check | vault_status | 0.8 ms | 0.9 ms | 8 | ≈ 동일 | +| 시나리오 | feature | p50 total | p95 total | n | Δ(2026-05-04 대비) | +| ---------------------------------- | ------------- | ----------- | ----------- | ------ | ------------------ | +| T1 짧은 영문 (~35 tokens) | capture | 5269.5 ms | 5449.8 ms | 8 | +5167 ms (×51) | +| T2 긴 영문 (~155 tokens) | capture | 4707.0 ms | 4931.9 ms | 8 | +4584 ms (×38) | +| T3 한국어 | capture | 4595.2 ms | 4789.1 ms | 8 | +4440 ms (×30) | +| T4 중복 입력 (near-duplicate path) | capture | 4886.1 ms | 5020.2 ms | 8 | +4779 ms (×46) | +| T5 Recall — exact match | recall | 368.2 ms | 385.3 ms | 8 | +331 ms (×9.9) | +| T6 Recall — 한→영 cross-language | recall | 384.1 ms | 464.3 ms | 8 | +346 ms (×10.0) | +| T7 Recall — topk 1/3/5/10 | recall | ~367–376 ms | ~374–402 ms | 5 each | ×9.2–×9.6 | +| T8 Batch per-item (size 1/5/10/20) | batch_capture | ~238–245 ms | ~242–246 ms | 3 each | ×4.8 | +| T9 Vault health check | vault_status | 0.8 ms | 0.9 ms | 8 | ≈ 동일 | **capture 병목**: `insert` (4200–5000 ms, FHE encrypt + 원격 insert) 가 전체의 **~92%**를 차지. embed/score/vault_topk 모두 합쳐도 5–10% 수준. **recall 병목**: `score` (180–200 ms) + `vault_topk` (100–115 ms) + `remind` (45–47 ms) 의 FHE 경로가 전체의 **~80%**, 임베딩(~40 ms)은 부수적. @@ -48,9 +48,9 @@ ### T3 한국어 capture가 영문보다 score/vault_topk만 느림 -| | embed | score | vault_topk | insert | -|--|-------|-------|------------|--------| -| T1 영문 | 92.6 | 100.4 | 31.9 | 5018.1 | +| | embed | score | vault_topk | insert | +| --------- | ------------ | ------------ | ------------ | ------------- | +| T1 영문 | 92.6 | 100.4 | 31.9 | 5018.1 | | T3 한국어 | 126.9 (+37%) | 167.5 (+67%) | 75.6 (+137%) | 4209.0 (-16%) | embed/score/vault_topk는 한국어가 모두 느리지만 insert는 오히려 더 빠름. 토큰 차이로 embed가 느린 것은 어제와 동일하나, score/vault_topk가 토큰 수에 영향받는 이유는 명확하지 않음 (이론적으로 FHE 벡터 길이가 동일해야 함). 토크나이저 출력의 vector dim이 다르거나, payload 길이 차이가 직렬화 오버헤드로 나타났을 가능성. @@ -86,51 +86,51 @@ per-item 240 ms vs 단일 capture 5000 ms. 차이의 대부분은 batch_capture - tokens_approx: 35 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 92.6 | 120.6 | 122.0 | 98.9 | -| score | 8 | 100.4 | 115.0 | 116.4 | 98.5 | -| vault_topk | 8 | 31.9 | 47.3 | 49.4 | 33.6 | -| insert | 8 | 5018.1 | 5242.2 | 5287.3 | 4951.1 | -| total | 8 | 5269.5 | 5449.8 | 5486.2 | 5182.1 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 92.6 | 120.6 | 122.0 | 98.9 | +| score | 8 | 100.4 | 115.0 | 116.4 | 98.5 | +| vault_topk | 8 | 31.9 | 47.3 | 49.4 | 33.6 | +| insert | 8 | 5018.1 | 5242.2 | 5287.3 | 4951.1 | +| total | 8 | 5269.5 | 5449.8 | 5486.2 | 5182.1 | ### T2_long_en - label: long English (~150 tokens) - tokens_approx: 155 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 100.6 | 139.3 | 144.6 | 108.0 | -| score | 8 | 135.7 | 146.0 | 147.1 | 134.3 | -| vault_topk | 8 | 56.0 | 64.4 | 64.7 | 54.9 | -| insert | 8 | 4406.8 | 4645.0 | 4695.4 | 4409.3 | -| total | 8 | 4707.0 | 4931.9 | 4976.3 | 4706.7 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 100.6 | 139.3 | 144.6 | 108.0 | +| score | 8 | 135.7 | 146.0 | 147.1 | 134.3 | +| vault_topk | 8 | 56.0 | 64.4 | 64.7 | 54.9 | +| insert | 8 | 4406.8 | 4645.0 | 4695.4 | 4409.3 | +| total | 8 | 4707.0 | 4931.9 | 4976.3 | 4706.7 | ### T3_korean - label: Korean text - tokens_approx: 50 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 126.9 | 147.3 | 149.4 | 128.3 | -| score | 8 | 167.5 | 184.9 | 187.2 | 168.7 | -| vault_topk | 8 | 75.6 | 91.1 | 92.6 | 76.4 | -| insert | 8 | 4209.0 | 4397.0 | 4405.8 | 4246.7 | -| total | 8 | 4595.2 | 4789.1 | 4806.0 | 4620.1 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 126.9 | 147.3 | 149.4 | 128.3 | +| score | 8 | 167.5 | 184.9 | 187.2 | 168.7 | +| vault_topk | 8 | 75.6 | 91.1 | 92.6 | 76.4 | +| insert | 8 | 4209.0 | 4397.0 | 4405.8 | 4246.7 | +| total | 8 | 4595.2 | 4789.1 | 4806.0 | 4620.1 | ### T4_duplicate - label: duplicate input — tests novelty near-duplicate path - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 115.2 | 132.1 | 132.4 | 113.4 | -| score | 8 | 211.4 | 229.7 | 231.1 | 209.3 | -| vault_topk | 8 | 99.7 | 108.9 | 109.6 | 99.7 | -| insert | 8 | 4435.8 | 4568.8 | 4572.0 | 4446.1 | -| total | 8 | 4886.1 | 5020.2 | 5021.7 | 4868.4 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 115.2 | 132.1 | 132.4 | 113.4 | +| score | 8 | 211.4 | 229.7 | 231.1 | 209.3 | +| vault_topk | 8 | 99.7 | 108.9 | 109.6 | 99.7 | +| insert | 8 | 4435.8 | 4568.8 | 4572.0 | 4446.1 | +| total | 8 | 4886.1 | 5020.2 | 5021.7 | 4868.4 | ## Feature: `recall` @@ -140,78 +140,78 @@ per-item 240 ms vs 단일 capture 5000 ms. 차이의 대부분은 batch_capture - topk: 5 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 40.4 | 41.2 | 41.3 | 39.7 | -| score | 8 | 182.7 | 200.0 | 204.1 | 185.3 | -| vault_topk | 8 | 98.5 | 105.7 | 106.0 | 99.7 | -| remind | 8 | 45.9 | 49.7 | 50.6 | 46.5 | -| total | 8 | 368.2 | 385.3 | 386.0 | 371.2 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 40.4 | 41.2 | 41.3 | 39.7 | +| score | 8 | 182.7 | 200.0 | 204.1 | 185.3 | +| vault_topk | 8 | 98.5 | 105.7 | 106.0 | 99.7 | +| remind | 8 | 45.9 | 49.7 | 50.6 | 46.5 | +| total | 8 | 368.2 | 385.3 | 386.0 | 371.2 | ### T6_cross_lang - label: cross-language semantic (KO→EN) - topk: 5 - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 8 | 41.4 | 47.1 | 47.3 | 42.6 | -| score | 8 | 195.4 | 217.7 | 221.0 | 197.5 | -| vault_topk | 8 | 104.6 | 157.1 | 177.1 | 114.1 | -| remind | 8 | 47.0 | 53.2 | 53.8 | 47.4 | -| total | 8 | 384.1 | 464.3 | 483.0 | 401.5 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 8 | 41.4 | 47.1 | 47.3 | 42.6 | +| score | 8 | 195.4 | 217.7 | 221.0 | 197.5 | +| vault_topk | 8 | 104.6 | 157.1 | 177.1 | 114.1 | +| remind | 8 | 47.0 | 53.2 | 53.8 | 47.4 | +| total | 8 | 384.1 | 464.3 | 483.0 | 401.5 | ### T7_topk_1 - topk: 1 - label: topk scaling topk=1 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 37.4 | 42.6 | 43.6 | 37.7 | -| score | 5 | 186.7 | 191.7 | 192.0 | 186.5 | -| vault_topk | 5 | 101.2 | 103.2 | 103.6 | 100.7 | -| remind | 5 | 43.5 | 45.1 | 45.3 | 43.0 | -| total | 5 | 367.7 | 375.1 | 375.3 | 367.9 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 37.4 | 42.6 | 43.6 | 37.7 | +| score | 5 | 186.7 | 191.7 | 192.0 | 186.5 | +| vault_topk | 5 | 101.2 | 103.2 | 103.6 | 100.7 | +| remind | 5 | 43.5 | 45.1 | 45.3 | 43.0 | +| total | 5 | 367.7 | 375.1 | 375.3 | 367.9 | ### T7_topk_3 - topk: 3 - label: topk scaling topk=3 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 38.0 | 38.5 | 38.6 | 36.8 | -| score | 5 | 181.0 | 191.1 | 191.5 | 182.7 | -| vault_topk | 5 | 105.4 | 109.0 | 109.2 | 104.4 | -| remind | 5 | 45.0 | 45.8 | 45.8 | 44.7 | -| total | 5 | 368.1 | 376.4 | 377.2 | 368.7 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 38.0 | 38.5 | 38.6 | 36.8 | +| score | 5 | 181.0 | 191.1 | 191.5 | 182.7 | +| vault_topk | 5 | 105.4 | 109.0 | 109.2 | 104.4 | +| remind | 5 | 45.0 | 45.8 | 45.8 | 44.7 | +| total | 5 | 368.1 | 376.4 | 377.2 | 368.7 | ### T7_topk_5 - topk: 5 - label: topk scaling topk=5 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 36.5 | 38.3 | 38.3 | 36.8 | -| score | 5 | 178.7 | 196.3 | 196.5 | 184.6 | -| vault_topk | 5 | 114.5 | 139.6 | 143.8 | 116.2 | -| remind | 5 | 44.8 | 46.1 | 46.3 | 44.9 | -| total | 5 | 376.4 | 402.2 | 403.8 | 382.4 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 36.5 | 38.3 | 38.3 | 36.8 | +| score | 5 | 178.7 | 196.3 | 196.5 | 184.6 | +| vault_topk | 5 | 114.5 | 139.6 | 143.8 | 116.2 | +| remind | 5 | 44.8 | 46.1 | 46.3 | 44.9 | +| total | 5 | 376.4 | 402.2 | 403.8 | 382.4 | ### T7_topk_10 - topk: 10 - label: topk scaling topk=10 - runs: 5 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| embed | 5 | 38.2 | 39.3 | 39.5 | 37.2 | -| score | 5 | 181.9 | 190.2 | 190.6 | 183.8 | -| vault_topk | 5 | 101.8 | 106.0 | 106.4 | 102.6 | -| remind | 5 | 46.2 | 47.2 | 47.3 | 46.0 | -| total | 5 | 369.4 | 374.0 | 374.0 | 369.7 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ---------- | --- | ------ | ------ | ------ | ------- | +| embed | 5 | 38.2 | 39.3 | 39.5 | 37.2 | +| score | 5 | 181.9 | 190.2 | 190.6 | 183.8 | +| vault_topk | 5 | 101.8 | 106.0 | 106.4 | 102.6 | +| remind | 5 | 46.2 | 47.2 | 47.3 | 46.0 | +| total | 5 | 369.4 | 374.0 | 374.0 | 369.7 | ## Feature: `batch_capture` @@ -221,40 +221,40 @@ per-item 240 ms vs 단일 capture 5000 ms. 차이의 대부분은 batch_capture - label: batch size 1 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 237.9 | 242.2 | 242.6 | 239.2 | -| per_item | 3 | 237.9 | 242.2 | 242.6 | 239.2 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 237.9 | 242.2 | 242.6 | 239.2 | +| per_item | 3 | 237.9 | 242.2 | 242.6 | 239.2 | ### T8_batch_5 - batch_size: 5 - label: batch size 5 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 1207.8 | 1209.7 | 1209.9 | 1207.5 | -| per_item | 3 | 241.6 | 241.9 | 242.0 | 241.5 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 1207.8 | 1209.7 | 1209.9 | 1207.5 | +| per_item | 3 | 241.6 | 241.9 | 242.0 | 241.5 | ### T8_batch_10 - batch_size: 10 - label: batch size 10 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 2409.5 | 2437.7 | 2440.2 | 2415.5 | -| per_item | 3 | 240.9 | 243.8 | 244.0 | 241.5 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 2409.5 | 2437.7 | 2440.2 | 2415.5 | +| per_item | 3 | 240.9 | 243.8 | 244.0 | 241.5 | ### T8_batch_20 - batch_size: 20 - label: batch size 20 (embed+score per item) - runs: 3 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| total_batch | 3 | 4892.0 | 4907.7 | 4909.1 | 4846.8 | -| per_item | 3 | 244.6 | 245.4 | 245.5 | 242.3 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ----------- | --- | ------ | ------ | ------ | ------- | +| total_batch | 3 | 4892.0 | 4907.7 | 4909.1 | 4846.8 | +| per_item | 3 | 244.6 | 245.4 | 245.5 | 242.3 | ## Feature: `vault_status` @@ -263,6 +263,6 @@ per-item 240 ms vs 단일 capture 5000 ms. 차이의 대부분은 batch_capture - label: Vault gRPC health check - runs: 8 -| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | -|-------|---|--------|--------|--------|---------| -| vault_health_check | 8 | 0.8 | 0.9 | 0.9 | 0.8 | +| Phase | n | p50 ms | p95 ms | p99 ms | mean ms | +| ------------------ | --- | ------ | ------ | ------ | ------- | +| vault_health_check | 8 | 0.8 | 0.9 | 0.9 | 0.8 | From 9e4cb8f3123e4ddd1afeb74680e7090c854d46a3 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Wed, 6 May 2026 14:44:11 +0900 Subject: [PATCH 13/16] feat(mcp): cache envector_secure from Vault bundle to config.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plumb the new envector_secure field through fetch_keys_from_vault (now an 8-tuple) and apply it to EnVectorConfig.secure in both the reload (_init_pipelines) and startup-init paths. When Vault provides the field, the value is persisted to ~/.rune/config.json so the EnVectorClient/Adapter is constructed with the right TLS mode on subsequent runs. Backward-compatible: older Vault servers that omit the field cause vault_ev_secure to come back as None, in which case the client leaves the existing secure setting (env var → config → SDK default) untouched. Co-Authored-By: Claude Opus 4.7 (1M context) --- mcp/server/server.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/mcp/server/server.py b/mcp/server/server.py index 7c1ba5ce..78a2776a 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -212,7 +212,11 @@ async def _async_fetch_keys_from_vault( tls_disable: If True, use insecure plaintext channel. Returns: - tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, envector_endpoint, envector_api_key) + tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, + envector_endpoint, envector_api_key, envector_secure) + envector_secure is None when Vault did not provide the field + (older Vault servers); the client should leave it untouched and + fall back to the SDK default in that case. """ client = VaultClient( vault_endpoint=vault_endpoint, @@ -230,6 +234,7 @@ async def _async_fetch_keys_from_vault( vault_agent_dek_b64 = bundle.pop("agent_dek", None) vault_envector_endpoint = bundle.pop("envector_endpoint", None) vault_envector_api_key = bundle.pop("envector_api_key", None) + vault_envector_secure = bundle.pop("envector_secure", None) if vault_index_name: logger.info(f"Vault provided index_name: {vault_index_name}") @@ -237,7 +242,7 @@ async def _async_fetch_keys_from_vault( logger.info(f"Vault provided key_id: {vault_key_id}") else: logger.warning("Vault did not provide key_id — key directory cannot be determined") - return False, vault_index_name, None, None, None, None, None + return False, vault_index_name, None, None, None, None, None, None if vault_agent_id: logger.info(f"Vault provided agent_id: {vault_agent_id}") @@ -249,10 +254,10 @@ async def _async_fetch_keys_from_vault( agent_dek_bytes = base64.b64decode(vault_agent_dek_b64) except (base64.binascii.Error, ValueError) as e: logger.error(f"Failed to decode agent_dek from Vault (invalid base64): {e}") - return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None + return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None, None if len(agent_dek_bytes) != 32: logger.error(f"agent_dek has invalid length {len(agent_dek_bytes)} bytes (expected 32 for AES-256)") - return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None + return False, vault_index_name, vault_key_id, vault_agent_id, None, None, None, None # Save keys under key_base_path// with restrictive permissions key_dir = os.path.join(key_base_path, vault_key_id) @@ -265,11 +270,14 @@ async def _async_fetch_keys_from_vault( f.write(key_content) logger.info(f"Saved {filename} to {filepath}") - return True, vault_index_name, vault_key_id, vault_agent_id, agent_dek_bytes, vault_envector_endpoint, vault_envector_api_key + return ( + True, vault_index_name, vault_key_id, vault_agent_id, agent_dek_bytes, + vault_envector_endpoint, vault_envector_api_key, vault_envector_secure, + ) except Exception as e: logger.error(f"Failed to fetch keys from Vault: {e}") - return False, None, None, None, None, None, None + return False, None, None, None, None, None, None, None finally: await client.close() @@ -293,10 +301,11 @@ def fetch_keys_from_vault( tls_disable: If True, use insecure plaintext channel. Returns: - tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, envector_endpoint, envector_api_key) + tuple: (success, index_name, key_id, agent_id, agent_dek_bytes, + envector_endpoint, envector_api_key, envector_secure) """ import asyncio - _fail = (False, None, None, None, None, None, None) + _fail = (False, None, None, None, None, None, None, None) try: asyncio.get_running_loop() @@ -1558,7 +1567,10 @@ def _init_pipelines(self) -> Dict[str, Any]: # Always re-fetch from Vault on reload to pick up endpoint/index changes if rune_config.vault.endpoint and rune_config.vault.token: logger.info("Fetching keys from Vault...") - success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, vault_ev_endpoint, vault_ev_api_key = fetch_keys_from_vault( + ( + success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, + vault_ev_endpoint, vault_ev_api_key, vault_ev_secure, + ) = fetch_keys_from_vault( rune_config.vault.endpoint, rune_config.vault.token, key_path, @@ -1579,9 +1591,11 @@ def _init_pipelines(self) -> Dict[str, Any]: self._envector_endpoint = vault_ev_endpoint if vault_ev_api_key: self._envector_api_key = vault_ev_api_key + if vault_ev_secure is not None: + rune_config.envector.secure = vault_ev_secure # Cache enVector credentials to config.json - if vault_ev_endpoint or vault_ev_api_key: + if vault_ev_endpoint or vault_ev_api_key or vault_ev_secure is not None: from agents.common.config import save_config as save_rune_config if vault_ev_endpoint: rune_config.envector.endpoint = vault_ev_endpoint @@ -1921,7 +1935,10 @@ def _parse_optional_bool(value): ENVECTOR_KEY_PATH = MCPServerApp.DEFAULT_KEY_PATH logger.info(f"Vault configured — fetching public keys from: {RUNEVAULT_ENDPOINT}") - success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, vault_ev_endpoint, vault_ev_api_key = fetch_keys_from_vault( + ( + success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, + vault_ev_endpoint, vault_ev_api_key, vault_ev_secure, + ) = fetch_keys_from_vault( RUNEVAULT_ENDPOINT, RUNEVAULT_TOKEN, ENVECTOR_KEY_PATH, ca_cert=VAULT_CA_CERT, @@ -1941,6 +1958,9 @@ def _parse_optional_bool(value): if vault_ev_api_key: ENVECTOR_API_KEY = vault_ev_api_key logger.info("Using enVector API key from Vault bundle") + if vault_ev_secure is not None: + ENVECTOR_SECURE = vault_ev_secure + logger.info(f"Using enVector secure={ENVECTOR_SECURE} from Vault bundle") if not vault_ev_endpoint or not vault_ev_api_key: logger.error("Vault bundle missing enVector credentials. Contact your Vault administrator.") _set_dormant_with_reason("envector_not_provisioned") From cae7ad461b7c421b8d68d3a66fb68a06f20e2371 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Wed, 6 May 2026 15:22:47 +0900 Subject: [PATCH 14/16] fix(mcp): drop user-scope MCP registration to avoid plugin-scope conflict Claude Code auto-registers enVector MCP from the plugin manifest (plugin:rune:envector). The duplicate user-scope `claude mcp add` entry competed with the plugin-scope registration during stdio handshake and caused both to fail. Keep the legacy-entry purge so older installs are cleaned up; let the plugin manifest own registration. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/configure-claude-mcp.sh | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/scripts/configure-claude-mcp.sh b/scripts/configure-claude-mcp.sh index 1a087c8a..94ea6a15 100755 --- a/scripts/configure-claude-mcp.sh +++ b/scripts/configure-claude-mcp.sh @@ -24,25 +24,21 @@ print_warn() { CONFIGURED=0 # ============================================================================ -# 1. Claude Code — explicit `claude mcp add` registration +# 1. Claude Code — purge legacy user-scope MCP entries +# +# Claude Code auto-registers the enVector MCP server from the plugin manifest +# (plugin:rune:envector). Older Rune installs added a duplicate user-scope +# `envector` entry via `claude mcp add`, which competes with the plugin-scope +# registration during stdio handshake and causes both to fail to connect. +# This block only removes those legacy entries; registration is owned by the +# plugin manifest. # ============================================================================ CLAUDE_CMD=$(CLAUDECODE= command -v claude 2>/dev/null || true) if [ -n "$CLAUDE_CMD" ]; then for name in envector rune-vault envector-mcp-server; do CLAUDECODE= "$CLAUDE_CMD" mcp remove --scope user "$name" 2>/dev/null || true done - print_info "Cleaned stale Claude Code MCP entries" - - CLAUDECODE= "$CLAUDE_CMD" mcp add --scope user envector -- \ - env \ - ENVECTOR_CONFIG="$HOME/.rune/config.json" \ - ENVECTOR_AUTO_KEY_SETUP=false \ - PYTHONPATH="$PLUGIN_DIR/mcp" \ - "$PLUGIN_DIR/scripts/bootstrap-mcp.sh" >/dev/null - print_info "MCP server registered in Claude Code user scope" - CONFIGURED=$((CONFIGURED + 1)) -else - print_warn "Claude Code CLI not found; skipping Claude Code MCP registration" + print_info "Cleaned legacy Claude Code user-scope MCP entries (plugin scope owns registration)" fi # ============================================================================ @@ -230,7 +226,7 @@ fi # Summary # ============================================================================ if [ "$CONFIGURED" -eq 0 ]; then - print_warn "No Claude Code CLI or Claude Desktop configuration found" + print_warn "No Claude Desktop configuration found (Claude Code uses plugin-scope MCP registration)" fi echo "" From c9ef9585d71774c58701cceb95a4df221c83b20e Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Wed, 6 May 2026 16:33:36 +0900 Subject: [PATCH 15/16] fix(mcp): defer Vault key fetch to background to avoid startup timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synchronous fetch_keys_from_vault() call in main() blocked MCP transport startup on the GetPublicKey gRPC stream. With pyenvector 1.4.0's larger EvalKey the stream now regularly exceeds Claude Code's 30s MCP startup limit, leaving the server unreachable on cold starts. The existing _init_pipelines background thread already re-fetches and rebuilds the adapter, so the sync block was redundant — remove it and let pipeline init populate key_id / agent_id / agent_dek / index_name once Vault responds. Tools that need these already gate via _ensure_pipelines() so they wait gracefully. Mirror the envector_not_provisioned dormant transition into _init_pipelines so the missing-credentials signal is preserved. Co-Authored-By: Claude Opus 4.7 (1M context) --- mcp/server/server.py | 84 ++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 50 deletions(-) diff --git a/mcp/server/server.py b/mcp/server/server.py index 78a2776a..4ae77a88 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -1603,6 +1603,10 @@ def _init_pipelines(self) -> Dict[str, Any]: rune_config.envector.api_key = vault_ev_api_key save_rune_config(rune_config) logger.info("Cached enVector credentials to config.json") + + if not vault_ev_endpoint or not vault_ev_api_key: + logger.error("Vault bundle missing enVector credentials. Contact your Vault administrator.") + _set_dormant_with_reason("envector_not_provisioned") else: result["errors"].append("Failed to fetch keys from Vault") logger.error("Failed to fetch keys from Vault — capture/search will fail") @@ -1924,50 +1928,22 @@ def _parse_optional_bool(value): VAULT_TLS_DISABLE = bool(_vault_cfg.get("tls_disable", False)) VAULT_CONFIGURED = bool(RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN) - VAULT_KEYS_LOADED = False VAULT_INDEX_NAME = None AGENT_ID = None AGENT_DEK = None if RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN: # When Vault is configured (Rune plugin mode), use canonical key path. - # key_id is discovered from Vault — no hardcoded default. + # key_id is discovered from Vault by the background pipeline init thread — + # no synchronous fetch here, so MCP transport is ready in ~1s instead of + # blocking on a multi-MB EvalKey stream that can exceed Claude's startup + # timeout. ENVECTOR_KEY_PATH = MCPServerApp.DEFAULT_KEY_PATH - - logger.info(f"Vault configured — fetching public keys from: {RUNEVAULT_ENDPOINT}") - ( - success, vault_index, vault_key_id, vault_agent_id, vault_agent_dek, - vault_ev_endpoint, vault_ev_api_key, vault_ev_secure, - ) = fetch_keys_from_vault( - RUNEVAULT_ENDPOINT, RUNEVAULT_TOKEN, - ENVECTOR_KEY_PATH, - ca_cert=VAULT_CA_CERT, - tls_disable=VAULT_TLS_DISABLE, + AUTO_KEY_SETUP = False + ENVECTOR_KEY_ID = None # populated by _init_pipelines from Vault bundle + logger.info( + f"Vault configured — keys will be fetched in background from: {RUNEVAULT_ENDPOINT}" ) - if success and vault_key_id: - ENVECTOR_KEY_ID = vault_key_id - logger.info(f"Vault provided key_id: {ENVECTOR_KEY_ID}") - AUTO_KEY_SETUP = False - VAULT_KEYS_LOADED = True - VAULT_INDEX_NAME = vault_index - AGENT_ID = vault_agent_id - AGENT_DEK = vault_agent_dek - if vault_ev_endpoint: - ENVECTOR_ENDPOINT = vault_ev_endpoint - logger.info("Using enVector endpoint from Vault bundle") - if vault_ev_api_key: - ENVECTOR_API_KEY = vault_ev_api_key - logger.info("Using enVector API key from Vault bundle") - if vault_ev_secure is not None: - ENVECTOR_SECURE = vault_ev_secure - logger.info(f"Using enVector secure={ENVECTOR_SECURE} from Vault bundle") - if not vault_ev_endpoint or not vault_ev_api_key: - logger.error("Vault bundle missing enVector credentials. Contact your Vault administrator.") - _set_dormant_with_reason("envector_not_provisioned") - else: - logger.error("Failed to fetch keys/key_id from Vault. Operations requiring encryption will fail.") - _set_dormant_with_reason("vault_unreachable") - AUTO_KEY_SETUP = False elif RUNEVAULT_ENDPOINT and not RUNEVAULT_TOKEN: logger.warning("Vault endpoint provided but no token specified. Skipping Vault integration.") VAULT_CONFIGURED = True @@ -1975,20 +1951,25 @@ def _parse_optional_bool(value): elif not AUTO_KEY_SETUP: logger.info(f"Using externally provided keys from: {ENVECTOR_KEY_PATH}") + # When Vault is configured, defer adapter construction to _init_pipelines + # (which builds it after the background key fetch completes). Constructing + # it here would either block on key files or partially init with empty + # placeholders that get overwritten anyway. envector_adapter = None - try: - envector_adapter = EnVectorSDKAdapter( - address=ENVECTOR_ENDPOINT, - key_id=ENVECTOR_KEY_ID, - key_path=ENVECTOR_KEY_PATH, - eval_mode=ENVECTOR_EVAL_MODE, - query_encryption=ENCRYPTED_QUERY, - access_token=ENVECTOR_API_KEY, - secure=ENVECTOR_SECURE, - auto_key_setup=AUTO_KEY_SETUP, - ) - except Exception as e: - logger.warning(f"enVector adapter init failed (server will start in degraded mode): {e}") + if not (RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN): + try: + envector_adapter = EnVectorSDKAdapter( + address=ENVECTOR_ENDPOINT, + key_id=ENVECTOR_KEY_ID, + key_path=ENVECTOR_KEY_PATH, + eval_mode=ENVECTOR_EVAL_MODE, + query_encryption=ENCRYPTED_QUERY, + access_token=ENVECTOR_API_KEY, + secure=ENVECTOR_SECURE, + auto_key_setup=AUTO_KEY_SETUP, + ) + except Exception as e: + logger.warning(f"enVector adapter init failed (server will start in degraded mode): {e}") vault_client = None if RUNEVAULT_ENDPOINT and RUNEVAULT_TOKEN: @@ -2015,7 +1996,10 @@ def _parse_optional_bool(value): agent_dek=AGENT_DEK, ) - # Set enVector credentials from Vault bundle on app instance + # Seed enVector credentials from cached ~/.rune/config.json so that + # vault_status / non-encrypted tools have something to report before the + # background Vault fetch completes. _init_pipelines will overwrite these + # with fresh values from the Vault bundle. if ENVECTOR_ENDPOINT: app._envector_endpoint = ENVECTOR_ENDPOINT if ENVECTOR_API_KEY: From cc29cf7ed51f6a0c0ba3cbf3d40d49789fcc83a6 Mon Sep 17 00:00:00 2001 From: heeyeon01 Date: Wed, 6 May 2026 17:19:52 +0900 Subject: [PATCH 16/16] fix(mcp): align Vault key fetch wrapper timeout with gRPC deadline The thread wrapper around fetch_keys_from_vault used a 30s timeout while the underlying GetPublicKey gRPC call has a 90s deadline (654806e). On cold start the gRPC call could take 40s+ due to TLS handshake racing sentence-transformers model loading on MainThread, so the wrapper declared failure and switched the server to dormant even though the worker thread eventually succeeded and wrote keys to disk. Bump the wrapper timeout to 120s (gRPC deadline + buffer for thread/ asyncio/TLS overhead) with a 30s grace re-poll so an in-flight call returns its real result. The `with ThreadPoolExecutor` block already waits on shutdown, so a longer timeout costs nothing on the success path. Also rework the ad-hoc DEBUG file logger added during diagnosis into an opt-in RotatingFileHandler controlled by RUNE_MCP_DEBUG_LOG (=1 for INFO, =debug for verbose). Off by default so normal runs don't write to disk; preserved as a tool for future background-thread debugging. Co-Authored-By: Claude Opus 4.7 (1M context) --- mcp/server/server.py | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/mcp/server/server.py b/mcp/server/server.py index 4ae77a88..fa80e47f 100644 --- a/mcp/server/server.py +++ b/mcp/server/server.py @@ -41,6 +41,34 @@ def filter(self, record): logger.addFilter(_SensitiveFilter()) + +# Optional file log for diagnosing background-thread issues that Claude +# Code's MCP console doesn't always capture cleanly. Off by default. +# RUNE_MCP_DEBUG_LOG=1 -> INFO level (lifecycle + errors) +# RUNE_MCP_DEBUG_LOG=debug -> DEBUG level (verbose, includes httpx traces) +if os.environ.get("RUNE_MCP_DEBUG_LOG"): + try: + from logging.handlers import RotatingFileHandler + _debug_log = os.path.expanduser("~/.rune/mcp-server.log") + os.makedirs(os.path.dirname(_debug_log), exist_ok=True) + _fh = RotatingFileHandler(_debug_log, maxBytes=5_000_000, backupCount=3) + _level = ( + logging.DEBUG + if os.environ["RUNE_MCP_DEBUG_LOG"].lower() == "debug" + else logging.INFO + ) + _fh.setLevel(_level) + _fh.setFormatter(logging.Formatter( + "%(asctime)s [%(threadName)s] %(levelname)s %(name)s: %(message)s" + )) + _fh.addFilter(_SensitiveFilter()) + _root = logging.getLogger() + _root.addHandler(_fh) + if _root.level == 0 or _root.level > _level: + _root.setLevel(_level) + except Exception: + pass + from pydantic import Field # Add parent directory (rune/mcp/) to sys.path so `from adapter import ...` works @@ -317,11 +345,21 @@ def fetch_keys_from_vault( asyncio.run, _async_fetch_keys_from_vault(vault_endpoint, vault_token, key_base_path, ca_cert, tls_disable), ) + # gRPC GetPublicKey has a 90s deadline; add buffer for thread/ + # asyncio/TLS overhead so a slow-but-eventually-OK fetch returns + # its real result. The `with` block already waits on shutdown, + # so a longer timeout costs nothing on the success path. try: - return future.result(timeout=30) + return future.result(timeout=120) except concurrent.futures.TimeoutError: - logger.error("Vault key fetch timed out after 30 seconds") - return _fail + logger.warning( + "Vault key fetch exceeded 120s — waiting briefly for in-flight call" + ) + try: + return future.result(timeout=30) + except Exception as e: + logger.error(f"Vault key fetch ultimately failed: {e}") + return _fail except Exception as e: logger.error(f"Vault key fetch failed in thread: {e}") return _fail