fix(router): persist and apply retry_policy via update_settings (#31308)

litellm/router.py

@@ -10134,6 +10134,7 @@ def update_settings(self, **kwargs):
             "fallbacks",
             "context_window_fallbacks",
             "model_group_retry_policy",
+            "retry_policy",
             "model_group_alias",
             "enable_weighted_failover",
         ]
@@ -10169,6 +10170,8 @@ def update_settings(self, **kwargs):
                                 ),
                             )
                             rebuild_routing_groups = True
+                    elif var == "retry_policy" and isinstance(value, dict):
+                        value = RetryPolicy(**value)
                     setattr(self, var, value)
             else:
                 verbose_router_logger.debug("Setting {} is not allowed".format(var))

litellm/types/router.py

@@ -83,6 +83,18 @@ class RouterConfig(BaseModel):
     model_config = ConfigDict(protected_namespaces=())
 
 
+# Defined here (above UpdateRouterConfig) so it can be used as a typed field.
+class RetryPolicy(BaseModel):
+    """Custom number of retries per exception type. https://docs.litellm.ai/docs/exception_mapping"""
+
+    BadRequestErrorRetries: Optional[int] = None
+    AuthenticationErrorRetries: Optional[int] = None
+    TimeoutErrorRetries: Optional[int] = None
+    RateLimitErrorRetries: Optional[int] = None
+    ContentPolicyViolationErrorRetries: Optional[int] = None
+    InternalServerErrorRetries: Optional[int] = None
+
+
 class UpdateRouterConfig(BaseModel):
     """
     Set of params that you can modify via `router.update_settings()`.
@@ -101,6 +113,7 @@ class UpdateRouterConfig(BaseModel):
     retry_after: Optional[float] = None
     fallbacks: Optional[List[dict]] = None
     context_window_fallbacks: Optional[List[dict]] = None
+    retry_policy: Optional[RetryPolicy] = None
     model_group_alias: Optional[Dict[str, Union[str, Dict]]] = {}
 
     model_config = ConfigDict(protected_namespaces=())
@@ -494,22 +507,6 @@ class AllowedFailsPolicy(BaseModel):
     InternalServerErrorAllowedFails: Optional[int] = None
 
 
-class RetryPolicy(BaseModel):
-    """
-    Use this to set a custom number of retries per exception type
-    If RateLimitErrorRetries = 3, then 3 retries will be made for RateLimitError
-    Mapping of Exception type to number of retries
-    https://docs.litellm.ai/docs/exception_mapping
-    """
-
-    BadRequestErrorRetries: Optional[int] = None
-    AuthenticationErrorRetries: Optional[int] = None
-    TimeoutErrorRetries: Optional[int] = None
-    RateLimitErrorRetries: Optional[int] = None
-    ContentPolicyViolationErrorRetries: Optional[int] = None
-    InternalServerErrorRetries: Optional[int] = None
-
-
 class AlertingConfig(BaseModel):
     """
     Use this configure alerting for the router. Receive alerts on the following events

tests/router_unit_tests/test_router_helper_utils.py

@@ -977,6 +977,20 @@ def test_update_settings(model_list):
     assert router.allowed_fails == 20
 
 
+def test_update_settings_retry_policy(model_list):
+    """update_settings must accept retry_policy and coerce a dict to RetryPolicy."""
+    from litellm.types.router import RetryPolicy
+
+    router = Router(model_list=model_list)
+    assert router.retry_policy is None
+
+    router.update_settings(retry_policy={"RateLimitErrorRetries": 3})
+
+    assert isinstance(router.retry_policy, RetryPolicy)
+    assert router.retry_policy.RateLimitErrorRetries == 3
+    assert router.get_settings()["retry_policy"].RateLimitErrorRetries == 3
+
+
 def test_common_checks_available_deployment(model_list):
     """Test if the 'common_checks_available_deployment' function is working correctly"""
     router = Router(model_list=model_list)
@@ -997,9 +1011,7 @@ def test_filter_cooldown_deployments(model_list):
         healthy_deployments=router._get_all_deployments(model_name="gpt-5-mini"),  # type: ignore
         cooldown_deployments=[],
     )
-    assert len(deployments) == len(
-        router._get_all_deployments(model_name="gpt-5-mini")
-    )
+    assert len(deployments) == len(router._get_all_deployments(model_name="gpt-5-mini"))
 
 
 def test_track_deployment_metrics(model_list):

tests/test_litellm/proxy/test_proxy_server.py

@@ -605,9 +605,7 @@ def test_ui_extensionless_route_requires_restructure(tmp_path):
 
 
 def test_admin_ui_export_serves_nested_extensionless_routes():
-    out_dir = (
-        Path(litellm.__file__).parent / "proxy" / "_experimental" / "out"
-    )
+    out_dir = Path(litellm.__file__).parent / "proxy" / "_experimental" / "out"
     assert out_dir.is_dir(), f"missing UI export at {out_dir}"
 
     nested_html_offenders = [
@@ -619,8 +617,7 @@ def test_admin_ui_export_serves_nested_extensionless_routes():
         and "litellm-asset-prefix" not in path.parts
     ]
     assert not nested_html_offenders, (
-        "Nested routes must be named index.html. Offenders: "
-        f"{nested_html_offenders}"
+        "Nested routes must be named index.html. Offenders: " f"{nested_html_offenders}"
     )
 
     callback_index = out_dir / "mcp" / "oauth" / "callback" / "index.html"
@@ -630,17 +627,17 @@ def test_admin_ui_export_serves_nested_extensionless_routes():
     )
 
     fastapi_app = FastAPI()
-    fastapi_app.mount(
-        "/ui", StaticFiles(directory=str(out_dir), html=True), name="ui"
-    )
+    fastapi_app.mount("/ui", StaticFiles(directory=str(out_dir), html=True), name="ui")
     client = TestClient(fastapi_app)
 
     redirect = client.get(
         "/ui/mcp/oauth/callback?code=abc&state=xyz",
         follow_redirects=False,
     )
     assert redirect.status_code == 307
-    assert redirect.headers["location"].endswith("/ui/mcp/oauth/callback/?code=abc&state=xyz")
+    assert redirect.headers["location"].endswith(
+        "/ui/mcp/oauth/callback/?code=abc&state=xyz"
+    )
 
     landed = client.get("/ui/mcp/oauth/callback?code=abc&state=xyz")
     assert landed.status_code == 200
@@ -3619,6 +3616,51 @@ async def test_add_router_settings_from_db_config_merge_logic():
     assert combined_settings["nested_config"] == expected_nested
 
 
+@pytest.mark.asyncio
+async def test_add_router_settings_from_db_config_applies_retry_policy_to_live_router():
+    """A retry_policy in the router_settings DB row must be applied to the live Router on startup."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.router import Router
+    from litellm.types.router import RetryPolicy
+
+    proxy_config = ProxyConfig()
+    router = Router(
+        model_list=[
+            {
+                "model_name": "m",
+                "litellm_params": {
+                    "model": "openai/gpt-3.5-turbo",
+                    "api_key": "sk-fake",
+                },
+            }
+        ]
+    )
+
+    retry_policy = {"RateLimitErrorRetries": 3, "InternalServerErrorRetries": 5}
+    mock_db_config = MagicMock()
+    mock_db_config.param_value = {"retry_policy": retry_policy}
+
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.litellm_config.find_first = AsyncMock(
+        return_value=mock_db_config
+    )
+
+    await proxy_config._add_router_settings_from_db_config(
+        config_data={"router_settings": {}},
+        llm_router=router,
+        prisma_client=mock_prisma_client,
+    )
+
+    assert isinstance(router.retry_policy, RetryPolicy)
+    assert router.retry_policy.RateLimitErrorRetries == 3
+    assert router.retry_policy.InternalServerErrorRetries == 5
+    # And the value round-trips back through get_settings (the read path used
+    # by /get/config/callbacks).
+    assert router.get_settings()["retry_policy"].RateLimitErrorRetries == 3
+
+
 @pytest.mark.asyncio
 async def test_add_router_settings_from_db_config_edge_cases():
     """
@@ -5902,15 +5944,16 @@ async def fresh_lock(_counter_key):
         if call.kwargs.get("nx") is True
     ]
     assert len(nx_writes) == 2
-    assert sorted(set_results) == [False, True], (
-        f"expected exactly one SET NX winner and one loser, got {set_results}"
-    )
+    assert sorted(set_results) == [
+        False,
+        True,
+    ], f"expected exactly one SET NX winner and one loser, got {set_results}"
     # Loser path executed: after the winner's SET NX returned True, the
     # losing coalesced() call falls back to async_get_cache to read the
     # winner's value rather than re-seeding.
-    assert get_after_set_count >= 1, (
-        "loser branch (else: read back winner's value) was never exercised"
-    )
+    assert (
+        get_after_set_count >= 1
+    ), "loser branch (else: read back winner's value) was never exercised"
 
 
 @pytest.mark.asyncio
@@ -7108,6 +7151,27 @@ def test_update_config_success_callback_normalizes_existing_mixed_case(
         restore()
 
 
+def test_update_config_persists_router_settings_retry_policy(_update_config_setup):
+    """/config/update must persist router_settings.retry_policy, merged with existing keys."""
+    client, prisma, restore = _update_config_setup(
+        initial_rows={"router_settings": {"num_retries": 2}}
+    )
+    try:
+        retry_policy = {"RateLimitErrorRetries": 3, "InternalServerErrorRetries": 5}
+        resp = client.post(
+            "/config/update",
+            json={"router_settings": {"retry_policy": retry_policy}},
+        )
+        assert resp.status_code == 200
+        stored = prisma.db.litellm_config.rows["router_settings"]
+        # The retry policy round-trips to the DB ...
+        assert stored["retry_policy"] == retry_policy
+        # ... and the pre-existing, untouched key is preserved.
+        assert stored["num_retries"] == 2
+    finally:
+        restore()
+
+
 # ---------------------------------------------------------------------------
 # Lazy feature loading (LazyFeatureMiddleware) — verifies that optional
 # routers are NOT imported at module load and ARE imported on first request

tests/test_litellm/types/test_router_types.py

@@ -0,0 +1,28 @@
+"""Unit tests for litellm.types.router config models."""
+
+from litellm.types.router import RetryPolicy, UpdateRouterConfig
+
+
+def test_update_router_config_retains_retry_policy():
+    """UpdateRouterConfig must parse retry_policy into RetryPolicy and serialize it back out."""
+    cfg = UpdateRouterConfig(
+        retry_policy={"RateLimitErrorRetries": 3, "InternalServerErrorRetries": 5}
+    )
+
+    assert isinstance(cfg.retry_policy, RetryPolicy)
+    assert cfg.retry_policy.RateLimitErrorRetries == 3
+    assert cfg.retry_policy.InternalServerErrorRetries == 5
+
+    serialized = cfg.dict(exclude_none=True)
+    assert serialized["retry_policy"] == {
+        "RateLimitErrorRetries": 3,
+        "InternalServerErrorRetries": 5,
+    }
+
+
+def test_update_router_config_retry_policy_defaults_to_none():
+    """retry_policy is optional and absent from exclude_none serialization."""
+    cfg = UpdateRouterConfig()
+
+    assert cfg.retry_policy is None
+    assert "retry_policy" not in cfg.dict(exclude_none=True)