fix: harden grid order and fill recovery

Author: Henry

backend_api_python/app/services/grid/exchange_orders.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import secrets
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -182,7 +183,12 @@ def make_grid_initial_client_order_id(strategy_id: int, leg: str = "") -> str:
 
 
 def make_grid_client_order_id(strategy_id: int, cell_index: int, purpose: str) -> str:
-    """Short client oid (OKX max 32). purpose: e/l/x/s = entry long/exit/short entry."""
+    """Return a collision-resistant grid order id within OKX's 32-char limit.
+
+    Resting orders for the same cell can be replaced more than once in one
+    second. A second-resolution suffix is therefore not unique enough and is
+    rejected by exchanges such as Binance after an id has been used once.
+    """
     p = (purpose or "x")[:1].lower()
     if "long_entry" in purpose:
         p = "e"
@@ -192,8 +198,8 @@ def make_grid_client_order_id(strategy_id: int, cell_index: int, purpose: str) -
         p = "s"
     elif "short_exit" in purpose:
         p = "c"
-    ts = int(__import__("time").time()) % 1000000
-    return f"g{int(strategy_id) % 10000:04d}c{int(cell_index):03d}{p}{ts % 99999:05d}"[:32]
+    nonce = secrets.token_hex(6)
+    return f"g{int(strategy_id) % 10000:04d}c{int(cell_index):03d}{p}{nonce}"[:32]
 
 
 def place_grid_limit_order(

backend_api_python/app/services/grid/fill_handler.py

@@ -91,40 +91,39 @@ def apply_grid_fill_to_local_state(
         market_type=str(tc.get("market_type") or "swap"),
         fill_source="grid_poller",
     )
-    try:
-        profit, _pos, matched_entry = apply_fill_to_local_position(
-            strategy_id=int(strategy_id),
-            symbol=sym,
-            signal_type=signal_type,
-            filled=qty,
-            avg_price=px,
-            leg=leg,
-        )
-        if grid_profit is not None:
-            profit = grid_profit
-            matched_entry = grid_entry_price
-        record_trade(
-            strategy_id=int(strategy_id),
-            symbol=sym,
-            trade_type=signal_type,
-            price=px,
-            amount=qty,
-            commission=float(commission or 0.0),
-            commission_ccy=str(commission_ccy or ""),
-            commission_quote=commission_quote,
-            profit=profit,
-            close_reason=purpose,
-            matched_entry_price=matched_entry,
-            grid_matched_profit=profit if purpose in ("long_exit", "short_exit") and profit is not None else None,
-            leg=leg,
-            exchange_fill_id=str(exchange_fill_id or ""),
-            execution_event_id=int(execution_event_id or 0),
-            grid_order_id=int(order.id or 0),
-            fee_status=str(fee_status or "pending"),
-            fee_source=str(fee_source or ""),
-        )
-    except Exception as e:
-        logger.warning("apply_grid_fill sid=%s: %s", strategy_id, e)
+    profit, _pos, matched_entry = apply_fill_to_local_position(
+        strategy_id=int(strategy_id),
+        symbol=sym,
+        signal_type=signal_type,
+        filled=qty,
+        avg_price=px,
+        leg=leg,
+    )
+    if grid_profit is not None:
+        profit = grid_profit
+        matched_entry = grid_entry_price
+    record_trade(
+        strategy_id=int(strategy_id),
+        symbol=sym,
+        trade_type=signal_type,
+        price=px,
+        amount=qty,
+        commission=float(commission or 0.0),
+        commission_ccy=str(commission_ccy or ""),
+        commission_quote=commission_quote,
+        profit=profit,
+        close_reason=purpose,
+        matched_entry_price=matched_entry,
+        grid_matched_profit=(
+            profit if purpose in ("long_exit", "short_exit") and profit is not None else None
+        ),
+        leg=leg,
+        exchange_fill_id=str(exchange_fill_id or ""),
+        execution_event_id=int(execution_event_id or 0),
+        grid_order_id=int(order.id or 0),
+        fee_status=str(fee_status or "pending"),
+        fee_source=str(fee_source or ""),
+    )
 
 
 def record_grid_market_fill(
@@ -162,56 +161,54 @@ def record_grid_market_fill(
         market_type=str(tc.get("market_type") or "swap"),
         fill_source="grid_market",
     )
-    try:
-        profit, _pos, matched_entry = apply_fill_to_local_position(
-            strategy_id=int(strategy_id),
-            symbol=sym,
-            signal_type=sig,
-            filled=qty,
-            avg_price=px,
-            leg=leg,
-        )
-        trade_id = record_trade(
-            strategy_id=int(strategy_id),
-            symbol=sym,
-            trade_type=sig,
-            price=px,
-            amount=qty,
-            commission=float(commission or 0.0),
-            commission_ccy=str(commission_ccy or ""),
-            commission_quote=commission_quote,
-            profit=profit,
-            close_reason=str(reason or sig),
-            matched_entry_price=matched_entry,
-            leg=leg,
-            fee_status=str(fee_status or "pending"),
-            fee_source=str(fee_source or "rest"),
-        )
-        if trade_id and (exchange_order_id or client_order_id):
-            try:
-                from app.services.execution_streams.repository import ExecutionEventRepository
+    profit, _pos, matched_entry = apply_fill_to_local_position(
+        strategy_id=int(strategy_id),
+        symbol=sym,
+        signal_type=sig,
+        filled=qty,
+        avg_price=px,
+        leg=leg,
+    )
+    trade_id = record_trade(
+        strategy_id=int(strategy_id),
+        symbol=sym,
+        trade_type=sig,
+        price=px,
+        amount=qty,
+        commission=float(commission or 0.0),
+        commission_ccy=str(commission_ccy or ""),
+        commission_quote=commission_quote,
+        profit=profit,
+        close_reason=str(reason or sig),
+        matched_entry_price=matched_entry,
+        leg=leg,
+        fee_status=str(fee_status or "pending"),
+        fee_source=str(fee_source or "rest"),
+    )
+    if trade_id and (exchange_order_id or client_order_id):
+        try:
+            from app.services.execution_streams.repository import ExecutionEventRepository
 
-                ExecutionEventRepository().register_binding(
-                    credential_id=int(leg.credential_id or 0),
-                    exchange_id=str(exchange_id or tc.get("exchange_id") or tc.get("exchange") or ""),
-                    market_type=leg.normalized_market_type(),
-                    owner_type="grid_market",
-                    owner_id=int(trade_id),
-                    user_id=int(user_id or 1),
-                    strategy_id=int(strategy_id),
-                    symbol=sym,
-                    signal_type=sig,
-                    client_order_id=str(client_order_id or ""),
-                    exchange_order_id=str(exchange_order_id or ""),
-                    observed_filled=qty,
-                )
-            except Exception:
-                logger.debug(
-                    "grid market execution binding failed trade_id=%s",
-                    trade_id,
-                    exc_info=True,
-                )
-        return int(trade_id or 0)
-    except Exception as e:
-        logger.warning("record_grid_market_fill sid=%s: %s", strategy_id, e)
-        return 0
+            ExecutionEventRepository().register_binding(
+                credential_id=int(leg.credential_id or 0),
+                exchange_id=str(
+                    exchange_id or tc.get("exchange_id") or tc.get("exchange") or ""
+                ),
+                market_type=leg.normalized_market_type(),
+                owner_type="grid_market",
+                owner_id=int(trade_id),
+                user_id=int(user_id or 1),
+                strategy_id=int(strategy_id),
+                symbol=sym,
+                signal_type=sig,
+                client_order_id=str(client_order_id or ""),
+                exchange_order_id=str(exchange_order_id or ""),
+                observed_filled=qty,
+            )
+        except Exception:
+            logger.debug(
+                "grid market execution binding failed trade_id=%s",
+                trade_id,
+                exc_info=True,
+            )
+    return int(trade_id or 0)

backend_api_python/tests/test_grid_engine.py

@@ -357,6 +357,15 @@ def test_initial_market_target_qty_100u_20pct_20x():
     assert make_grid_initial_client_order_id(42, leg="long") != make_grid_initial_client_order_id(42, leg="short")
 
 
+def test_grid_resting_client_order_ids_do_not_collide_within_same_second():
+    from app.services.grid.exchange_orders import make_grid_client_order_id
+
+    order_ids = {make_grid_client_order_id(42, 58, "long_exit") for _ in range(100)}
+
+    assert len(order_ids) == 100
+    assert all(len(order_id) <= 32 and order_id.isalnum() for order_id in order_ids)
+
+
 def test_grid_line_qty_uses_quote_amount_times_leverage():
     from app.services.grid.engine import GridEngine
 
@@ -1578,3 +1587,58 @@ def list_cells(self, strategy_id, symbol=None):
     assert captured["profit"] == pytest.approx(expected)
     assert captured["grid_matched_profit"] == pytest.approx(expected)
     assert captured["matched_entry_price"] == pytest.approx(669.3)
+
+
+def test_grid_fill_ledger_failure_is_not_silently_marked_processed(monkeypatch):
+    from app.services.grid import fill_handler
+    from app.services.grid.resting_orders_repo import GridRestingOrder
+
+    monkeypatch.setattr(fill_handler, "resolve_leg_context", lambda **kwargs: None)
+    monkeypatch.setattr(
+        fill_handler,
+        "apply_fill_to_local_position",
+        lambda **kwargs: (_ for _ in ()).throw(RuntimeError("ledger unavailable")),
+    )
+
+    order = GridRestingOrder(
+        id=99,
+        strategy_id=1,
+        symbol="BNB/USDT",
+        cell_index=11,
+        purpose="long_entry",
+        side="buy",
+        pos_side="long",
+        price=676.7,
+        quantity=0.05,
+    )
+
+    with pytest.raises(RuntimeError, match="ledger unavailable"):
+        fill_handler.apply_grid_fill_to_local_state(
+            1,
+            "BNB/USDT",
+            order,
+            0.05,
+            676.7,
+            {"market_type": "swap"},
+        )
+
+
+def test_grid_market_fill_ledger_failure_is_not_silently_accepted(monkeypatch):
+    from app.services.grid import fill_handler
+
+    monkeypatch.setattr(fill_handler, "resolve_leg_context", lambda **kwargs: None)
+    monkeypatch.setattr(
+        fill_handler,
+        "apply_fill_to_local_position",
+        lambda **kwargs: (_ for _ in ()).throw(RuntimeError("ledger unavailable")),
+    )
+
+    with pytest.raises(RuntimeError, match="ledger unavailable"):
+        fill_handler.record_grid_market_fill(
+            1,
+            "BNB/USDT",
+            "open_long",
+            0.05,
+            676.7,
+            {"market_type": "swap"},
+        )

backend_api_python/tests/test_grid_poller.py

@@ -81,6 +81,34 @@ def test_poller_dispatches_partial_fill_immediately():
     assert update.call_args.kwargs["processed_fill_qty"] == pytest.approx(0.4)
 
 
+def test_poller_retries_fill_when_local_ledger_posting_fails():
+    poller = GridFillPoller()
+    runner = MagicMock()
+    runner.engine.on_order_filled.side_effect = RuntimeError("ledger unavailable")
+    order = GridRestingOrder(
+        id=14,
+        strategy_id=1,
+        symbol="BTC/USDT",
+        price=100.0,
+        quantity=1.0,
+        processed_fill_qty=0.0,
+        filled_quantity=0.0,
+        status="open",
+    )
+
+    with patch(
+        "app.services.grid.poller.query_grid_order_fill",
+        return_value=(0.4, 99.5, "partial"),
+    ):
+        with patch.object(poller._repo, "update_status") as update:
+            poller._poll_order(runner, MagicMock(), order, "swap")
+
+    runner.engine.on_order_filled.assert_called_once_with(order, 0.4, 99.5)
+    assert update.call_args.kwargs["status"] == "partial"
+    assert update.call_args.kwargs["filled_quantity"] == pytest.approx(0.4)
+    assert update.call_args.kwargs["processed_fill_qty"] == pytest.approx(0.0)
+
+
 def test_poller_reconciles_cancelled_partial_fill():
     poller = GridFillPoller()
     runner = MagicMock()