diff --git a/.test/README.md b/.test/README.md index eb448826..cd48eb5b 100644 --- a/.test/README.md +++ b/.test/README.md @@ -46,6 +46,18 @@ export GEPA_REFLECTION_LM="openai/gpt-4o" export GEPA_GEN_LM="openai/gpt-4o" ``` +**OrcaRouter** + +[OrcaRouter](https://www.orcarouter.ai) is an OpenAI-compatible gateway. Use the `orcarouter/` provider prefix with a fully namespaced model id (e.g. `orcarouter/openai/gpt-4o-mini`): + +```bash +export ORCAROUTER_API_KEY="sk-orca-..." +export GEPA_REFLECTION_LM="orcarouter/openai/gpt-4o-mini" +export GEPA_GEN_LM="orcarouter/openai/gpt-4o-mini" +``` + +When using a non-Databricks provider, set `GEPA_FALLBACK_MODELS` to models on the same provider (the default fallback chain points at Databricks endpoints): + ### 3. Configure the Claude Code agent (for `--agent-eval`) Agent evaluation runs a real Claude Code instance via the [Claude Agent SDK](https://github.com/anthropics/claude-agent-sdk-python). The agent's environment is configured in `.test/claude_agent_settings.json`: @@ -189,7 +201,7 @@ uv run python .test/scripts/optimize.py [options] | `--reflection-lm` | `GEPA_REFLECTION_LM` | `databricks/databricks-claude-opus-4-6` | GEPA's reflection/mutation model | | `--judge-model` | `GEPA_JUDGE_LM` | `databricks/databricks-claude-sonnet-4-6` | MLflow quality judge | -Proxy evaluator models use [litellm provider prefixes](https://docs.litellm.ai/docs/providers): `databricks/`, `openai/`, `anthropic/`. +Proxy evaluator models use [litellm provider prefixes](https://docs.litellm.ai/docs/providers): `databricks/`, `openai/`, `anthropic/`. OrcaRouter is supported through the `orcarouter/` prefix (e.g. `orcarouter/openai/gpt-4o-mini`), which routes to `https://api.orcarouter.ai/v1` with `ORCAROUTER_API_KEY`. ### Tool Optimization diff --git a/.test/src/skill_test/optimize/config.py b/.test/src/skill_test/optimize/config.py index 7ea0db0f..847e938d 100644 --- a/.test/src/skill_test/optimize/config.py +++ b/.test/src/skill_test/optimize/config.py @@ -54,7 +54,7 @@ def _configure_litellm_retries() -> None: def _register_litellm_models() -> None: - """Register Databricks model context windows with litellm.""" + """Register Databricks and OrcaRouter model context windows with litellm.""" try: import litellm @@ -122,6 +122,36 @@ def _register_litellm_models() -> None: "input_cost_per_token": 0, "output_cost_per_token": 0, }, + # OrcaRouter: OpenAI-compatible gateway (https://api.orcarouter.ai/v1). + # Model ids are fully namespaced (openai/..., deepseek/...); see + # judges.py `_to_litellm_model` for routing and ORCAROUTER_API_KEY auth. + "orcarouter/openai/gpt-4o-mini": { + "max_tokens": 16_384, + "max_input_tokens": 128_000, + "max_output_tokens": 16_384, + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + }, + "orcarouter/openai/gpt-4o": { + "max_tokens": 16_384, + "max_input_tokens": 128_000, + "max_output_tokens": 16_384, + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + }, + "orcarouter/deepseek/deepseek-v4-flash-0731": { + "max_tokens": 32_768, + "max_input_tokens": 128_000, + "max_output_tokens": 32_768, + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + }, } for model_name, model_info in _models.items(): litellm.model_cost[model_name] = model_info @@ -254,12 +284,13 @@ def validate_reflection_context( f"Fix: use a model with a larger context window:\n" f" --reflection-lm 'databricks/databricks-claude-opus-4-6' (200K)\n" f" --reflection-lm 'openai/gpt-4o' (128K)\n" + f" --reflection-lm 'orcarouter/openai/gpt-4o' (128K)\n" f" --reflection-lm 'anthropic/claude-sonnet-4-5-20250514' (200K)\n\n" f"Or set the environment variable:\n" f" export GEPA_REFLECTION_LM='databricks/databricks-claude-opus-4-6'\n\n" f"If you already use a large-context model and still see 'max_model_len'\n" f"errors, the Databricks serving endpoint itself has a low context limit.\n" - f"Switch to a non-Databricks provider (openai/ or anthropic/) instead.\n\n" + f"Switch to a non-Databricks provider (openai/, orcarouter/, or anthropic/) instead.\n\n" f" Current GEPA_REFLECTION_LM={os.environ.get('GEPA_REFLECTION_LM', '(not set)')}" ) diff --git a/.test/src/skill_test/optimize/judges.py b/.test/src/skill_test/optimize/judges.py index d48c5324..c971e993 100644 --- a/.test/src/skill_test/optimize/judges.py +++ b/.test/src/skill_test/optimize/judges.py @@ -192,6 +192,17 @@ def _get_gateway_base_url() -> str | None: return url.rstrip("/") +# OrcaRouter: OpenAI-compatible gateway. Model ids are fully namespaced +# (e.g. ``openai/gpt-4o-mini``, ``deepseek/deepseek-v4-flash-0731``), so the +# id after the ``orcarouter/`` prefix is passed through verbatim. +ORCAROUTER_BASE_URL = "https://api.orcarouter.ai/v1" + + +def _orcarouter_api_key() -> str: + """Return the OrcaRouter API key, or an empty string if not configured.""" + return os.environ.get("ORCAROUTER_API_KEY", "") + + def _to_litellm_model(model: str) -> tuple[str, str | None, str | None]: """Convert a model string to (litellm_model, base_url, api_key) for completion calls. @@ -200,6 +211,10 @@ def _to_litellm_model(model: str) -> tuple[str, str | None, str | None]: provider in litellm does not auto-read ``DATABRICKS_TOKEN``, so we pass it explicitly as ``api_key``. + ``orcarouter/`` models (e.g. ``orcarouter/openai/gpt-4o-mini``) + are routed through OrcaRouter as an OpenAI-compatible endpoint using + ``ORCAROUTER_API_KEY``. + Returns: (model_string, base_url_or_None, api_key_or_None) """ @@ -209,6 +224,14 @@ def _to_litellm_model(model: str) -> tuple[str, str | None, str | None]: endpoint_name = model.split("/", 1)[1] api_key = os.environ.get("DATABRICKS_TOKEN") or os.environ.get("DATABRICKS_API_KEY", "") return f"openai/{endpoint_name}", gateway, api_key or None + if model.startswith("orcarouter/"): + # Route through OrcaRouter as an OpenAI-compatible endpoint. + # litellm appends /chat/completions to the base URL, and OrcaRouter + # requires the fully namespaced model id (openai/..., anthropic/..., + # deepseek/...), so the rest of the id is passed through verbatim. + model_id = model.split("/", 1)[1] + api_key = _orcarouter_api_key() + return f"openai/{model_id}", ORCAROUTER_BASE_URL, api_key or None return model, None, None @@ -249,6 +272,9 @@ def _to_judge_model_and_params(model: str) -> tuple[str, dict[str, Any] | None]: If AI Gateway is configured, uses ``openai:/endpoint-name`` with ``inference_params.base_url`` pointing to the gateway. Otherwise uses standard ``provider:/model`` format. + + ``orcarouter/`` models map to ``openai:/`` + with ``inference_params.base_url`` pointing at OrcaRouter. """ gateway = _get_gateway_base_url() if gateway and model.startswith(("databricks/", "databricks:/")): @@ -262,6 +288,13 @@ def _to_judge_model_and_params(model: str) -> tuple[str, dict[str, Any] | None]: if api_key: params["api_key"] = api_key return f"openai:/{endpoint_name}", params + if model.startswith("orcarouter/"): + model_id = model.split("/", 1)[1] + api_key = _orcarouter_api_key() + params = {"base_url": ORCAROUTER_BASE_URL} + if api_key: + params["api_key"] = api_key + return f"openai:/{model_id}", params return _to_judge_uri(model), _judge_inference_params()