fix: repair LLM test-connection and make silent model/template fallbacks visible (#281)

* fix: repair admin LLM test-connection endpoint

POST /settings/test-connection calls LLMService.test_connection(), which has
never existed, so the endpoint always raised AttributeError and the UI reported
a generic failure regardless of whether the key was valid.

Add the missing method. It probes the provider's models endpoint without
spending completion tokens and without falling back to other providers, so a
failure identifies this provider/base URL/key combination specifically.

Credentials are sent only in headers, never in the URL. OpenRouter's /models is
public and therefore cannot validate a key, so use /auth/key there and flag
provisioning/management keys, which pass /auth/key but are rejected by chat
completions.

Also surface the template fallback from indicator aiGenerate through
debug.generation_error: the built-in template passes validation, so previously
a failed model call was indistinguishable from a successful response and the
client reported success.

Refs #280

* test: cover LLM test_connection probe

Cases: unconfigured key, OpenRouter authenticated endpoint selection, the
provisioning/management key trap, HTTP failure surfacing, and that the Google
key is sent as a header rather than in the URL.

* feat: report the model that actually answered an LLM call

A model-level fallback (requested model -> configured default -> static fallback)
was only observable in the provider dashboard, so a misconfigured or unavailable
model id silently produced answers from a different model while the client
reported success.

LLMService now records last_model_used / last_provider_used / model_fallback_used,
the indicator generation path carries them in its edit plan, and aiGenerate
returns them as generation_model / generation_provider /
generation_model_fallback in the streamed debug frame.

---------

Co-authored-by: TIANHE <TIANHE@GMAIL.COM>

Author: guofeng201507

backend_api_python/app/routes/indicator.py

@@ -1188,6 +1188,15 @@ def stream():
 
         code_text, debug_info, edit_plan = _generate_final_code()
 
+        # Signal the template fallback to the client; otherwise a failed model
+        # call is indistinguishable from a successful template-shaped answer.
+        if edit_plan.get("error"):
+            debug_info["generation_error"] = str(edit_plan["error"])[:500]
+        if edit_plan.get("model"):
+            debug_info["generation_model"] = edit_plan["model"]
+            debug_info["generation_provider"] = edit_plan.get("provider") or ""
+            debug_info["generation_model_fallback"] = bool(edit_plan.get("model_fallback"))
+
         if workspace_context:
             validation = _validate_indicator_code_internal(code_text)
             validation["edit_plan"] = edit_plan

backend_api_python/app/routes/settings.py

@@ -2451,13 +2451,10 @@ def test_connection():
         service = data.get('service')
 
         if service == 'openrouter':
-            from app.services.llm import LLMService
-            llm = LLMService()
-            result = llm.test_connection()
-            if result:
-                return jsonify({'code': 1, 'msg': 'OpenRouter connection successful'})
-            else:
-                return jsonify({'code': 0, 'msg': 'OpenRouter connection failed'})
+            from app.services.llm import LLMProvider, LLMService
+            llm = LLMService(provider=LLMProvider.OPENROUTER)
+            ok, detail = llm.test_connection(LLMProvider.OPENROUTER)
+            return jsonify({'code': 1 if ok else 0, 'msg': detail})
 
         elif service == 'finnhub':
             import requests

backend_api_python/app/services/indicator_ai_generation.py

@@ -156,7 +156,7 @@ def generate_indicator_code_candidate(
     )
     if use_patch_response:
         try:
-            return apply_model_code_edits(existing, content)
+            code, plan = apply_model_code_edits(existing, content)
         except CodeEditError as exc:
             logger.warning("indicator model patch rejected, retrying full candidate: %s", exc)
             fallback_prompt = (
@@ -182,7 +182,13 @@ def generate_indicator_code_candidate(
                 "operation": "generate_candidate",
                 "patch_error": str(exc),
             }
+            code = _strip_code_fences(content) or template_factory()
     else:
         plan = {"executor": "model", "operation": "generate_candidate"}
+        code = _strip_code_fences(content) or template_factory()
 
-    return _strip_code_fences(content) or template_factory(), plan
+    # A model-level fallback is otherwise only visible in the provider dashboard.
+    plan["model"] = llm.last_model_used
+    plan["provider"] = llm.last_provider_used
+    plan["model_fallback"] = llm.model_fallback_used
+    return code, plan

backend_api_python/app/services/llm.py

@@ -178,6 +178,11 @@ def __init__(self, provider: str = None):
             provider: Override the default provider (openrouter, openai, google, deepseek, grok, atlascloud, custom, minimax)
         """
         self._provider_override = provider
+        # Populated by call_llm_api() so callers can report what actually answered;
+        # a model-level fallback is otherwise only visible in provider dashboards.
+        self.last_model_used = ""
+        self.last_provider_used = ""
+        self.model_fallback_used = False
 
     @property
     def provider(self) -> LLMProvider:
@@ -299,6 +304,66 @@ def is_configured(self, provider: LLMProvider = None) -> bool:
             return bool((self.get_base_url(p) or "").strip())
         return p == LLMProvider.LITELLM
 
+    def _record_model_use(
+        self,
+        used_model: str,
+        requested_model: str,
+        provider: LLMProvider,
+    ) -> None:
+        """Remember which model/provider answered the most recent call."""
+        self.last_model_used = used_model
+        self.last_provider_used = provider.value
+        self.model_fallback_used = used_model != requested_model
+
+    def test_connection(self, provider: LLMProvider = None) -> tuple[bool, str]:
+        """Probe the configured provider's models endpoint.
+
+        Intentionally does not spend completion tokens and does not fall back to
+        other providers, so a failure here means *this* provider/base URL/key
+        combination is unreachable or rejected. Returns ``(ok, detail)``.
+
+        Credentials are only ever sent in headers, never in the URL, so provider
+        errors and request exceptions cannot leak the key.
+        """
+        p = provider or self.provider
+        model = self.get_default_model(p)
+        if not self.is_configured(p):
+            return False, f"{p.value}: API key is not configured"
+        key = (self.get_api_key(p) or "").strip()
+        base = (self.get_base_url(p) or "").rstrip("/")
+        headers: Dict[str, str] = {}
+        if p == LLMProvider.OPENROUTER:
+            # OpenRouter's /models is public, so it cannot validate the key.
+            # /auth/key requires the bearer token and 401s on a bad key.
+            url = f"{base}/auth/key"
+            headers["Authorization"] = f"Bearer {key}"
+        elif p == LLMProvider.GOOGLE:
+            url = f"{base}/models"
+            headers["x-goog-api-key"] = key
+        else:
+            url = f"{base}/models"
+            headers["Authorization"] = f"Bearer {key}"
+        try:
+            resp = requests.get(url, headers=headers, timeout=15)
+        except Exception as exc:
+            return False, f"{p.value}: {redact_secrets(exc)}"
+        if resp.status_code == 200:
+            if p == LLMProvider.OPENROUTER:
+                # A provisioning/management key passes /auth/key but is rejected
+                # (401 "User not found") by chat completions, so flag it here.
+                try:
+                    info = (resp.json() or {}).get("data") or {}
+                except Exception:
+                    info = {}
+                if info.get("is_provisioning_key") or info.get("is_management_key"):
+                    return False, (
+                        f"{p.value}: this is a provisioning/management key, "
+                        "which cannot run inference. Create a normal API key instead."
+                    )
+            return True, f"{p.value} reachable · model={model}"
+        body = redact_secrets(resp.text[:200]) if resp.text else ""
+        return False, f"{p.value}: HTTP {resp.status_code} {body}".strip()
+
     # Legacy properties for backward compatibility
     @property
     def api_key(self):
@@ -1230,23 +1295,25 @@ def call_llm_api(self, messages: list, model: str = None, temperature: float = 0
         for current_model in models_to_try:
             try:
                 if p == LLMProvider.LITELLM:
-                    return self._call_litellm(
+                    result = self._call_litellm(
                         messages, current_model, temperature,
                         api_key, base_url, timeout,
                         use_json_mode=use_json_mode
                     )
                 elif p == LLMProvider.GOOGLE:
-                    return self._call_google_gemini(
+                    result = self._call_google_gemini(
                         messages, current_model, temperature,
                         api_key, base_url, timeout
                     )
                 else:
                     # OpenAI-compatible providers
-                    return self._call_openai_compatible(
+                    result = self._call_openai_compatible(
                         messages, current_model, temperature,
                         api_key, base_url, timeout,
                         use_json_mode=use_json_mode
                     )
+                self._record_model_use(current_model, models_to_try[0], p)
+                return result
 
             except LLMAPIError as e:
                 status_code = e.status_code

backend_api_python/tests/test_llm_litellm_provider.py

@@ -841,3 +841,163 @@ def test_call_llm_api_redacts_request_error_before_reraising(monkeypatch):
     assert "Bearer <redacted>" in message
     assert exc_info.value.__cause__ is None
     assert exc_info.value.retryable is True
+
+
+class _FakeHttpGetResponse:
+    def __init__(self, status_code, payload=None, text=""):
+        self.status_code = status_code
+        self._payload = payload
+        self.text = text
+
+    def json(self):
+        if self._payload is None:
+            raise ValueError("not json")
+        return self._payload
+
+
+def test_test_connection_requires_configured_key(monkeypatch):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "")
+
+    ok, detail = service.test_connection()
+
+    assert ok is False
+    assert "not configured" in detail
+
+
+def test_test_connection_uses_authenticated_endpoint_for_openrouter(monkeypatch):
+    captured = {}
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key")
+    monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1")
+
+    def fake_get(url, headers=None, timeout=None):
+        captured.update({"url": url, "headers": headers, "timeout": timeout})
+        return _FakeHttpGetResponse(200, {"data": {}})
+
+    monkeypatch.setattr("app.services.llm.requests.get", fake_get)
+
+    ok, detail = service.test_connection()
+
+    assert ok is True
+    # /models is public on OpenRouter, so it cannot validate a key.
+    assert captured["url"] == "https://openrouter.ai/api/v1/auth/key"
+    assert captured["headers"]["Authorization"] == "Bearer openrouter-key"
+    assert "reachable" in detail
+
+
+def test_test_connection_flags_openrouter_provisioning_key(monkeypatch):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "provisioning-key")
+    monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1")
+    monkeypatch.setattr(
+        "app.services.llm.requests.get",
+        lambda *args, **kwargs: _FakeHttpGetResponse(200, {"data": {"is_provisioning_key": True}}),
+    )
+
+    ok, detail = service.test_connection()
+
+    assert ok is False
+    # Such a key passes /auth/key but chat completions reject it with 401.
+    assert "provisioning/management" in detail
+
+
+def test_test_connection_reports_http_failure(monkeypatch):
+    secret = "sk-response-secret-123"
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: secret)
+    monkeypatch.setattr(service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1")
+    monkeypatch.setattr(
+        "app.services.llm.requests.get",
+        lambda *args, **kwargs: _FakeHttpGetResponse(
+            401,
+            None,
+            f'{{"error":"unauthorized ?api_key={secret}"}}',
+        ),
+    )
+
+    ok, detail = service.test_connection()
+
+    assert ok is False
+    assert "HTTP 401" in detail
+    assert secret not in detail
+    assert "api_key=<redacted>" in detail
+
+
+def test_test_connection_never_puts_google_key_in_url(monkeypatch):
+    captured = {}
+    service = LLMService(provider="google")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "google-key")
+    monkeypatch.setattr(
+        service,
+        "get_base_url",
+        lambda provider=None: "https://generativelanguage.googleapis.com/v1beta",
+    )
+
+    def fake_get(url, headers=None, timeout=None):
+        captured.update({"url": url, "headers": headers})
+        return _FakeHttpGetResponse(200, {})
+
+    monkeypatch.setattr("app.services.llm.requests.get", fake_get)
+
+    ok, _ = service.test_connection()
+
+    assert ok is True
+    assert captured["headers"]["x-goog-api-key"] == "google-key"
+    assert "google-key" not in captured["url"]
+
+
+def _openrouter_service_with(monkeypatch, default_model="deepseek/deepseek-v4.1-flash"):
+    service = LLMService(provider="openrouter")
+    monkeypatch.setattr(service, "get_api_key", lambda provider=None: "openrouter-key")
+    monkeypatch.setattr(
+        service, "get_base_url", lambda provider=None: "https://openrouter.ai/api/v1"
+    )
+    monkeypatch.setattr(service, "get_default_model", lambda provider=None: default_model)
+    return service
+
+
+def test_model_fallback_is_recorded_when_primary_model_fails(monkeypatch):
+    attempted = []
+    service = _openrouter_service_with(monkeypatch)
+
+    def fake_call(messages, model, *args, **kwargs):
+        attempted.append(model)
+        if model == "does/not-exist":
+            raise LLMAPIError(
+                "OpenRouter API 400 (model=does/not-exist): not a valid model ID",
+                status_code=400,
+            )
+        return "generated code"
+
+    monkeypatch.setattr(service, "_call_openai_compatible", fake_call)
+
+    out = service.call_llm_api(
+        [{"role": "user", "content": "hi"}],
+        model="does/not-exist",
+        use_json_mode=False,
+        try_alternative_providers=False,
+    )
+
+    assert out == "generated code"
+    assert attempted == ["does/not-exist", "deepseek/deepseek-v4.1-flash"]
+    # The caller must be able to tell that a different model answered.
+    assert service.last_model_used == "deepseek/deepseek-v4.1-flash"
+    assert service.last_provider_used == "openrouter"
+    assert service.model_fallback_used is True
+
+
+def test_model_fallback_flag_is_false_when_primary_model_answers(monkeypatch):
+    service = _openrouter_service_with(monkeypatch)
+    monkeypatch.setattr(service, "_call_openai_compatible", lambda *args, **kwargs: "generated code")
+
+    out = service.call_llm_api(
+        [{"role": "user", "content": "hi"}],
+        model="deepseek/deepseek-v4.1-flash",
+        use_json_mode=False,
+        try_alternative_providers=False,
+    )
+
+    assert out == "generated code"
+    assert service.last_model_used == "deepseek/deepseek-v4.1-flash"
+    assert service.model_fallback_used is False

backend_api_python/tests/test_settings_llm_connection.py

@@ -0,0 +1,35 @@
+from flask import Flask
+
+
+def _unwrap_handler(handler):
+    while hasattr(handler, "__wrapped__"):
+        handler = handler.__wrapped__
+    return handler
+
+
+def test_openrouter_connection_route_pins_the_requested_provider(monkeypatch):
+    import app.routes.settings as settings_route
+    import app.services.llm as llm_module
+
+    calls = []
+
+    class FakeLLMService:
+        def __init__(self, provider=None):
+            calls.append(("init", provider))
+
+        def test_connection(self, provider=None):
+            calls.append(("test", provider))
+            return True, "ok"
+
+    monkeypatch.setattr(llm_module, "LLMService", FakeLLMService)
+    app = Flask(__name__)
+    handler = _unwrap_handler(settings_route.test_connection)
+
+    with app.test_request_context(json={"service": "openrouter"}):
+        response = handler()
+
+    assert response.get_json()["code"] == 1
+    assert calls == [
+        ("init", llm_module.LLMProvider.OPENROUTER),
+        ("test", llm_module.LLMProvider.OPENROUTER),
+    ]