fix: harden live strategy order recovery

Author: Henry

backend_api_python/app/services/live_trading/account_configuration.py

@@ -13,15 +13,42 @@ def requires_derivatives_account_configuration(*, market_type: str, reduce_only:
     return market in {"swap", "future", "futures", "perp", "perpetual"} and not bool(reduce_only)
 
 
-def _is_binance_unknown_timeout(exc: BaseException | str) -> bool:
+def _is_binance_ambiguous_configuration_error(exc: BaseException | str) -> bool:
     text = str(exc or "").lower()
     return (
-        "-1007" in text
+        "-1000" in text
+        or "unknown error occurred while processing the request" in text
+        or "-1006" in text
+        or "-1007" in text
         or "execution status unknown" in text
         or ("http 408" in text and "timeout" in text)
     )
 
 
+def resolve_derivatives_margin_mode(
+    *,
+    payload: Dict[str, Any],
+    strategy_config: Dict[str, Any],
+    exchange_config: Dict[str, Any] | None = None,
+) -> str:
+    """Resolve the persisted strategy margin mode for order dispatch."""
+    trading_config = strategy_config.get("trading_config") or {}
+    if not isinstance(trading_config, dict):
+        trading_config = {}
+    venue_config = exchange_config or {}
+    if not isinstance(venue_config, dict):
+        venue_config = {}
+    return normalize_margin_mode(
+        payload.get("margin_mode")
+        or payload.get("marginMode")
+        or trading_config.get("margin_mode")
+        or trading_config.get("marginMode")
+        or venue_config.get("margin_mode")
+        or venue_config.get("marginMode")
+        or "cross"
+    )
+
+
 def _confirm_binance_configuration(
     client: Any,
     *,
@@ -39,12 +66,12 @@ def _confirm_binance_configuration(
         observed_leverage = 0
     if require_margin and observed_mode != margin_mode:
         raise LiveTradingError(
-            f"Binance configuration timeout could not be confirmed: "
+            f"Binance configuration outcome could not be confirmed: "
             f"margin_mode expected={margin_mode}, observed={observed_mode}"
         )
     if require_leverage and observed_leverage != int(leverage):
         raise LiveTradingError(
-            f"Binance configuration timeout could not be confirmed: "
+            f"Binance configuration outcome could not be confirmed: "
             f"leverage expected={int(leverage)}, observed={observed_leverage}"
         )
     return observed
@@ -92,40 +119,66 @@ def configure_derivatives_account(
     }
 
     if isinstance(client, BinanceFuturesClient):
-        margin_confirmed_after_timeout = False
+        observed_before: Dict[str, Any] | None = None
+        try:
+            observed_before = client.get_symbol_configuration(symbol=symbol) or None
+        except Exception:
+            observed_before = None
+        if observed_before:
+            details["configuration_before"] = observed_before
+
+        observed_mode = ""
+        if observed_before:
+            try:
+                observed_mode = normalize_margin_mode(
+                    str(observed_before.get("margin_mode") or "")
+                )
+            except LiveTradingError:
+                observed_mode = ""
+        margin_changed = observed_mode != mode
+        observed_after_margin: Dict[str, Any] | None = None
+        if margin_changed:
+            try:
+                client.set_margin_type(symbol=symbol, margin_mode=mode)
+            except Exception as exc:
+                text = str(exc).lower()
+                if "-4046" not in text and "no need to change margin type" not in text:
+                    if not _is_binance_ambiguous_configuration_error(exc):
+                        raise LiveTradingError(f"Binance margin mode setup failed: {exc}") from exc
+                    observed_after_margin = _confirm_binance_configuration(
+                        client,
+                        symbol=symbol,
+                        margin_mode=mode,
+                        leverage=target_leverage,
+                        require_margin=True,
+                        require_leverage=False,
+                    )
+                    details["readback_after_margin_error"] = observed_after_margin
+        else:
+            details["margin_mode_already_configured"] = True
+
+        effective_observed = observed_after_margin or observed_before or {}
         try:
-            client.set_margin_type(symbol=symbol, margin_mode=mode)
-        except Exception as exc:
-            text = str(exc).lower()
-            if "-4046" not in text and "no need to change margin type" not in text:
-                if not _is_binance_unknown_timeout(exc):
-                    raise LiveTradingError(f"Binance margin mode setup failed: {exc}") from exc
+            effective_leverage = int(float(effective_observed.get("leverage") or 0))
+        except (TypeError, ValueError):
+            effective_leverage = 0
+        if margin_changed or effective_leverage != target_leverage:
+            try:
+                client.set_leverage(symbol=symbol, leverage=target_leverage)
+            except Exception as exc:
+                if not _is_binance_ambiguous_configuration_error(exc):
+                    raise
                 observed = _confirm_binance_configuration(
                     client,
                     symbol=symbol,
                     margin_mode=mode,
                     leverage=target_leverage,
                     require_margin=True,
-                    require_leverage=False,
+                    require_leverage=True,
                 )
-                details["readback_after_margin_timeout"] = observed
-                margin_confirmed_after_timeout = True
-        try:
-            client.set_leverage(symbol=symbol, leverage=target_leverage)
-        except Exception as exc:
-            if not _is_binance_unknown_timeout(exc):
-                raise
-            observed = _confirm_binance_configuration(
-                client,
-                symbol=symbol,
-                margin_mode=mode,
-                leverage=target_leverage,
-                require_margin=True,
-                require_leverage=True,
-            )
-            details["readback_after_leverage_timeout"] = observed
-        if margin_confirmed_after_timeout:
-            details["margin_mode_confirmed_after_timeout"] = True
+                details["readback_after_leverage_error"] = observed
+        else:
+            details["leverage_already_configured"] = True
         return details
 
     if isinstance(client, OkxClient):

backend_api_python/app/services/pending_order_worker.py

@@ -29,6 +29,7 @@
 )
 from app.services.live_trading.account_configuration import (
     requires_derivatives_account_configuration,
+    resolve_derivatives_margin_mode,
 )
 from app.services.live_trading.strategy_position_sync import (
     strategy_uses_fill_ledger,
@@ -1937,12 +1938,10 @@ def _execute_live_order(self, *, order_id: int, order_row: Dict[str, Any], paylo
             try:
                 from app.services.live_trading.account_configuration import configure_derivatives_account
 
-                margin_mode = str(
-                    payload.get("margin_mode")
-                    or payload.get("marginMode")
-                    or cfg.get("margin_mode")
-                    or cfg.get("marginMode")
-                    or "cross"
+                margin_mode = resolve_derivatives_margin_mode(
+                    payload=payload,
+                    strategy_config=cfg,
+                    exchange_config=exchange_config,
                 )
                 phases["account_configuration"] = configure_derivatives_account(
                     client,

backend_api_python/app/services/strategy_v2/live_execution.py

@@ -25,6 +25,7 @@ class LiveOrderRequest:
     market_type: str
     execution_mode: str
     leverage: float = 1.0
+    margin_mode: str = "cross"
     reason: str = ""
     notification_config: dict[str, Any] | None = None
     order_type: str = "market"
@@ -286,6 +287,7 @@ def submit(self, request: LiveOrderRequest) -> int | None:
             "price": request.limit_price or request.reference_price,
             "ref_price": request.reference_price,
             "leverage": request.leverage,
+            "margin_mode": request.margin_mode,
             "execution_mode": request.execution_mode,
             "notification_config": request.notification_config or {},
             "signal_ts": request.signal_timestamp,

backend_api_python/app/services/strategy_v2/runtime.py

@@ -2626,15 +2626,22 @@ def restore_protection_snapshot(self, values: Mapping[str, Any] | None) -> None:
                 states[str(symbol)] = state
         self.protection_specs = specs
         self.protection_states = states
-        self._protection_exit_pending = {str(item) for item in (raw.get("exitPending") or [])}
+        # Older snapshots treated every zero-target strategy order as a
+        # protection exit.  A rejected ordinary close could therefore leave an
+        # orphaned latch that suppressed all future orders for the symbol, even
+        # after a runtime restart.  A genuine protection exit always has both a
+        # persisted spec and state, so discard legacy latches that do not.
+        self._protection_exit_pending = {
+            str(item)
+            for item in (raw.get("exitPending") or [])
+            if str(item) in specs and str(item) in states
+        }
 
     def _capture_protection_intents(self, orders: Iterable[OrderIntent]) -> None:
         for order in orders:
             position_key = _position_key(order.symbol, order.position_side)
             if order.protection is not None:
                 self.protection_specs[position_key] = order.protection
-            if order.kind in {"target_quantity", "target_value", "target_percent"} and abs(order.value) <= 1e-12:
-                self._protection_exit_pending.add(position_key)
 
     def _bind_runtime_api(self) -> None:
         ctx = self.context

backend_api_python/app/services/trading_executor.py

@@ -1772,6 +1772,11 @@ def _execute_signal(self, **values: Any) -> bool:
             market_type=str(values.get("market_type") or "spot"),
             execution_mode=str(values.get("execution_mode") or "signal"),
             leverage=leverage,
+            margin_mode=str(
+                trading_config.get("margin_mode")
+                or trading_config.get("marginMode")
+                or "cross"
+            ),
             reason=str(values.get("signal_reason") or ""),
             notification_config=dict(values.get("notification_config") or {}),
             order_type=str(values.get("order_type") or "market"),

backend_api_python/tests/test_derivatives_account_configuration.py

@@ -5,6 +5,7 @@
 from app.services.live_trading.account_configuration import (
     configure_derivatives_account,
     requires_derivatives_account_configuration,
+    resolve_derivatives_margin_mode,
 )
 from app.services.live_trading.base import LiveTradingError
 from app.services.live_trading.bybit import BybitClient
@@ -77,19 +78,49 @@ def test_reduce_only_swap_skips_derivatives_configuration():
     assert requires_derivatives_account_configuration(market_type="spot", reduce_only=False) is False
 
 
+def test_strategy_margin_mode_is_resolved_from_nested_trading_config():
+    assert resolve_derivatives_margin_mode(
+        payload={},
+        strategy_config={"trading_config": {"margin_mode": "isolated"}},
+        exchange_config={},
+    ) == "isolated"
+
+
+def test_binance_matching_configuration_skips_mutating_requests():
+    client = BinanceFuturesClient.__new__(BinanceFuturesClient)
+    client.get_symbol_configuration = lambda **_kwargs: {
+        "margin_mode": "cross",
+        "leverage": 5,
+    }
+    client.set_margin_type = lambda **_kwargs: pytest.fail("margin mode must not be changed")
+    client.set_leverage = lambda **_kwargs: pytest.fail("leverage must not be changed")
+
+    result = configure_derivatives_account(
+        client,
+        exchange_id="binance",
+        symbol="BTC/USDT",
+        leverage=5,
+        margin_mode="cross",
+    )
+
+    assert result["margin_mode_already_configured"] is True
+    assert result["leverage_already_configured"] is True
+
+
 def test_binance_margin_timeout_continues_after_configuration_readback():
     client = BinanceFuturesClient.__new__(BinanceFuturesClient)
+    snapshots = iter((
+        {"margin_mode": "isolated", "leverage": 5},
+        {"margin_mode": "cross", "leverage": 5},
+    ))
     client.set_margin_type = lambda **_kwargs: (_ for _ in ()).throw(
         LiveTradingError(
             'Binance HTTP 408: {"code":-1007,"msg":"Timeout waiting for response; execution status unknown."}'
         )
     )
     leverage_calls = []
     client.set_leverage = lambda **kwargs: leverage_calls.append(kwargs) or {"leverage": 5}
-    client.get_symbol_configuration = lambda **_kwargs: {
-        "margin_mode": "cross",
-        "leverage": 5,
-    }
+    client.get_symbol_configuration = lambda **_kwargs: next(snapshots)
 
     result = configure_derivatives_account(
         client,
@@ -99,20 +130,48 @@ def test_binance_margin_timeout_continues_after_configuration_readback():
         margin_mode="cross",
     )
 
-    assert result["margin_mode_confirmed_after_timeout"] is True
+    assert result["readback_after_margin_error"]["margin_mode"] == "cross"
     assert leverage_calls == [{"symbol": "BTC/USDT", "leverage": 5}]
 
 
+def test_binance_margin_unknown_error_continues_after_configuration_readback():
+    client = BinanceFuturesClient.__new__(BinanceFuturesClient)
+    snapshots = iter((
+        {"margin_mode": "isolated", "leverage": 2},
+        {"margin_mode": "cross", "leverage": 2},
+    ))
+    client.get_symbol_configuration = lambda **_kwargs: next(snapshots)
+    client.set_margin_type = lambda **_kwargs: (_ for _ in ()).throw(
+        LiveTradingError(
+            'Binance HTTP 400: {"code":-1000,"msg":"An unknown error occurred while processing the request."}'
+        )
+    )
+    leverage_calls = []
+    client.set_leverage = lambda **kwargs: leverage_calls.append(kwargs) or {"leverage": 2}
+
+    result = configure_derivatives_account(
+        client,
+        exchange_id="binance",
+        symbol="DOGE/USDT",
+        leverage=2,
+        margin_mode="cross",
+    )
+
+    assert result["readback_after_margin_error"]["margin_mode"] == "cross"
+    assert leverage_calls == [{"symbol": "DOGE/USDT", "leverage": 2}]
+
+
 def test_binance_margin_timeout_fails_when_readback_differs():
     client = BinanceFuturesClient.__new__(BinanceFuturesClient)
+    snapshots = iter((
+        {"margin_mode": "isolated", "leverage": 5},
+        {"margin_mode": "isolated", "leverage": 5},
+    ))
     client.set_margin_type = lambda **_kwargs: (_ for _ in ()).throw(
         LiveTradingError("Binance HTTP 408: code=-1007 execution status unknown")
     )
     client.set_leverage = lambda **_kwargs: pytest.fail("leverage must not be changed")
-    client.get_symbol_configuration = lambda **_kwargs: {
-        "margin_mode": "isolated",
-        "leverage": 5,
-    }
+    client.get_symbol_configuration = lambda **_kwargs: next(snapshots)
 
     with pytest.raises(LiveTradingError, match="could not be confirmed"):
         configure_derivatives_account(

backend_api_python/tests/test_strategy_v2_order_gateway.py

@@ -1,3 +1,4 @@
+import json
 from types import SimpleNamespace
 
 from app.services.strategy_v2 import live_execution
@@ -49,6 +50,7 @@ def _request(action="open_long", **overrides):
         "signal_timestamp": 123,
         "market_type": "swap",
         "execution_mode": "live",
+        "margin_mode": "isolated",
     }
     values.update(overrides)
     return LiveOrderRequest(
@@ -136,6 +138,28 @@ def create_intent(self, **_kwargs):
     assert StrategyV2OrderGateway().submit(_request()) is None
 
 
+def test_submit_persists_margin_mode_in_pending_order_payload(monkeypatch):
+    class _IntentService:
+        def __init__(self, **_kwargs):
+            pass
+
+        @staticmethod
+        def build_signal_idempotency_key(**_kwargs):
+            return "margin-mode-signal"
+
+        @staticmethod
+        def create_intent(**_kwargs):
+            return SimpleNamespace(id=91, existing=False, status="intent_created")
+
+    cursor = _Cursor({"id": 123})
+    monkeypatch.setattr(live_execution, "OrderIntentService", _IntentService)
+    monkeypatch.setattr(live_execution, "get_db_connection", lambda: _Db(cursor))
+
+    assert StrategyV2OrderGateway().submit(_request(margin_mode="isolated")) == 123
+    payload = json.loads(cursor.params[10])
+    assert payload["margin_mode"] == "isolated"
+
+
 def test_ai_rejection_is_latched_until_the_signal_disappears(monkeypatch):
     decisions = []
     next_intent_id = iter((91, 92))