feat(test): add orcarouter/ provider for skill-eval models

.test/README.md

@@ -46,6 +46,18 @@ export GEPA_REFLECTION_LM="openai/gpt-4o"
 export GEPA_GEN_LM="openai/gpt-4o"
 ```
 
+**OrcaRouter**
+
+[OrcaRouter](https://www.orcarouter.ai) is an OpenAI-compatible gateway. Use the `orcarouter/` provider prefix with a fully namespaced model id (e.g. `orcarouter/openai/gpt-4o-mini`):
+
+```bash
+export ORCAROUTER_API_KEY="sk-orca-..."
+export GEPA_REFLECTION_LM="orcarouter/openai/gpt-4o-mini"
+export GEPA_GEN_LM="orcarouter/openai/gpt-4o-mini"
+```
+
+When using a non-Databricks provider, set `GEPA_FALLBACK_MODELS` to models on the same provider (the default fallback chain points at Databricks endpoints):
+
 ### 3. Configure the Claude Code agent (for `--agent-eval`)
 
 Agent evaluation runs a real Claude Code instance via the [Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk-python). The agent's environment is configured in `.test/claude_agent_settings.json`:
@@ -189,7 +201,7 @@ uv run python .test/scripts/optimize.py <skill_name> [options]
 | `--reflection-lm` | `GEPA_REFLECTION_LM` | `databricks/databricks-claude-opus-4-6` | GEPA's reflection/mutation model |
 | `--judge-model` | `GEPA_JUDGE_LM` | `databricks/databricks-claude-sonnet-4-6` | MLflow quality judge |
 
-Proxy evaluator models use [litellm provider prefixes](https://docs.litellm.ai/docs/providers): `databricks/`, `openai/`, `anthropic/`.
+Proxy evaluator models use [litellm provider prefixes](https://docs.litellm.ai/docs/providers): `databricks/`, `openai/`, `anthropic/`. OrcaRouter is supported through the `orcarouter/` prefix (e.g. `orcarouter/openai/gpt-4o-mini`), which routes to `https://api.orcarouter.ai/v1` with `ORCAROUTER_API_KEY`.
 
 ### Tool Optimization
 

.test/src/skill_test/optimize/config.py

@@ -54,7 +54,7 @@ def _configure_litellm_retries() -> None:
 
 
 def _register_litellm_models() -> None:
-    """Register Databricks model context windows with litellm."""
+    """Register Databricks and OrcaRouter model context windows with litellm."""
     try:
         import litellm
 
@@ -122,6 +122,36 @@ def _register_litellm_models() -> None:
                 "input_cost_per_token": 0,
                 "output_cost_per_token": 0,
             },
+            # OrcaRouter: OpenAI-compatible gateway (https://api.orcarouter.ai/v1).
+            # Model ids are fully namespaced (openai/..., deepseek/...); see
+            # judges.py `_to_litellm_model` for routing and ORCAROUTER_API_KEY auth.
+            "orcarouter/openai/gpt-4o-mini": {
+                "max_tokens": 16_384,
+                "max_input_tokens": 128_000,
+                "max_output_tokens": 16_384,
+                "litellm_provider": "openai",
+                "mode": "chat",
+                "input_cost_per_token": 0,
+                "output_cost_per_token": 0,
+            },
+            "orcarouter/openai/gpt-4o": {
+                "max_tokens": 16_384,
+                "max_input_tokens": 128_000,
+                "max_output_tokens": 16_384,
+                "litellm_provider": "openai",
+                "mode": "chat",
+                "input_cost_per_token": 0,
+                "output_cost_per_token": 0,
+            },
+            "orcarouter/deepseek/deepseek-v4-flash-0731": {
+                "max_tokens": 32_768,
+                "max_input_tokens": 128_000,
+                "max_output_tokens": 32_768,
+                "litellm_provider": "openai",
+                "mode": "chat",
+                "input_cost_per_token": 0,
+                "output_cost_per_token": 0,
+            },
         }
         for model_name, model_info in _models.items():
             litellm.model_cost[model_name] = model_info
@@ -254,12 +284,13 @@ def validate_reflection_context(
             f"Fix: use a model with a larger context window:\n"
             f"  --reflection-lm 'databricks/databricks-claude-opus-4-6'   (200K)\n"
             f"  --reflection-lm 'openai/gpt-4o'                           (128K)\n"
+            f"  --reflection-lm 'orcarouter/openai/gpt-4o'                (128K)\n"
             f"  --reflection-lm 'anthropic/claude-sonnet-4-5-20250514'    (200K)\n\n"
             f"Or set the environment variable:\n"
             f"  export GEPA_REFLECTION_LM='databricks/databricks-claude-opus-4-6'\n\n"
             f"If you already use a large-context model and still see 'max_model_len'\n"
             f"errors, the Databricks serving endpoint itself has a low context limit.\n"
-            f"Switch to a non-Databricks provider (openai/ or anthropic/) instead.\n\n"
+            f"Switch to a non-Databricks provider (openai/, orcarouter/, or anthropic/) instead.\n\n"
             f"  Current GEPA_REFLECTION_LM={os.environ.get('GEPA_REFLECTION_LM', '(not set)')}"
         )
 

.test/src/skill_test/optimize/judges.py

@@ -192,6 +192,17 @@ def _get_gateway_base_url() -> str | None:
     return url.rstrip("/")
 
 
+# OrcaRouter: OpenAI-compatible gateway. Model ids are fully namespaced
+# (e.g. ``openai/gpt-4o-mini``, ``deepseek/deepseek-v4-flash-0731``), so the
+# id after the ``orcarouter/`` prefix is passed through verbatim.
+ORCAROUTER_BASE_URL = "https://api.orcarouter.ai/v1"
+
+
+def _orcarouter_api_key() -> str:
+    """Return the OrcaRouter API key, or an empty string if not configured."""
+    return os.environ.get("ORCAROUTER_API_KEY", "")
+
+
 def _to_litellm_model(model: str) -> tuple[str, str | None, str | None]:
     """Convert a model string to (litellm_model, base_url, api_key) for completion calls.
 
@@ -200,6 +211,10 @@ def _to_litellm_model(model: str) -> tuple[str, str | None, str | None]:
     provider in litellm does not auto-read ``DATABRICKS_TOKEN``, so we
     pass it explicitly as ``api_key``.
 
+    ``orcarouter/<namespaced-model>`` models (e.g. ``orcarouter/openai/gpt-4o-mini``)
+    are routed through OrcaRouter as an OpenAI-compatible endpoint using
+    ``ORCAROUTER_API_KEY``.
+
     Returns:
         (model_string, base_url_or_None, api_key_or_None)
     """
@@ -209,6 +224,14 @@ def _to_litellm_model(model: str) -> tuple[str, str | None, str | None]:
         endpoint_name = model.split("/", 1)[1]
         api_key = os.environ.get("DATABRICKS_TOKEN") or os.environ.get("DATABRICKS_API_KEY", "")
         return f"openai/{endpoint_name}", gateway, api_key or None
+    if model.startswith("orcarouter/"):
+        # Route through OrcaRouter as an OpenAI-compatible endpoint.
+        # litellm appends /chat/completions to the base URL, and OrcaRouter
+        # requires the fully namespaced model id (openai/..., anthropic/...,
+        # deepseek/...), so the rest of the id is passed through verbatim.
+        model_id = model.split("/", 1)[1]
+        api_key = _orcarouter_api_key()
+        return f"openai/{model_id}", ORCAROUTER_BASE_URL, api_key or None
     return model, None, None
 
 
@@ -249,6 +272,9 @@ def _to_judge_model_and_params(model: str) -> tuple[str, dict[str, Any] | None]:
     If AI Gateway is configured, uses ``openai:/endpoint-name`` with
     ``inference_params.base_url`` pointing to the gateway. Otherwise
     uses standard ``provider:/model`` format.
+
+    ``orcarouter/<namespaced-model>`` models map to ``openai:/<model-id>``
+    with ``inference_params.base_url`` pointing at OrcaRouter.
     """
     gateway = _get_gateway_base_url()
     if gateway and model.startswith(("databricks/", "databricks:/")):
@@ -262,6 +288,13 @@ def _to_judge_model_and_params(model: str) -> tuple[str, dict[str, Any] | None]:
         if api_key:
             params["api_key"] = api_key
         return f"openai:/{endpoint_name}", params
+    if model.startswith("orcarouter/"):
+        model_id = model.split("/", 1)[1]
+        api_key = _orcarouter_api_key()
+        params = {"base_url": ORCAROUTER_BASE_URL}
+        if api_key:
+            params["api_key"] = api_key
+        return f"openai:/{model_id}", params
     return _to_judge_uri(model), _judge_inference_params()