From d5b7ea5a76c54a0f0d74446a053f390cbb93b510 Mon Sep 17 00:00:00 2001 From: jsboige Date: Fri, 4 Sep 2026 16:45:00 +0200 Subject: [PATCH] feat(training,#14579): OOT protocol instrument -- precision DM jambe + train-end freeze MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grain: MED/training -- lane myia-po-2027:CoursIA -- prev: LIGHT/ledger #14563 Instrument du protocole out-of-time (#14579, sous-grain EPIC #1454) pour le verdict OOT reel de L4 Decision Transformer. Deux deltas de code + doc: 1. validate_xrp_dt_holdout.py — jambe de precision §C (#11010) : erreur directionnelle e_t = position_t - sign(r_{t+1}), DM mse/mae (sign-aware pour des positions +-1 ; le mse direct sur retours positionnes reste sign-blind, (-r)^2 == r^2, cf #10228). DM linear conserve comme controle de biais (role explicite). Biais signes par modele rapportes (mean return DT/momentum/BH). Verdict : conjonction edge >= 2 sigma ET dm_p_mse_median < 0.05. 2. train_dt_multiseed.py — --train-end : tronque le CSV a la borne gelée avant le walk-forward, hash recalcule sur la tranche. 3. docs/L4_decision_transformer.md — section "Protocole OOT (#14579)" décrivant l'instrument + dataset de reference (bornes 2018-01-01 -> 2026-09-04, sha256 71d8aee9d0bda18b). Validation : - smoke CPU holdout PASS (3 jambes DM rendent : mse 0.78 / mae 0.69 / linear-ctrl 0.066, biais signes rapportes, verdict NO-BEATS coherent sur data synthetique) - dry-run WF --train-end 2025-06-30 : parse + troncature + walk-forward OK (5/10 coins executes avant abandon manuel du smoke) Le verdict OOT lui-meme (run GPU label=oot, train-end 2025-06-30, seeds 0,1,7,42,99) est un grain SEPARE : il depend du run en cours et mettra a jour REGISTRY.md + la section verdict — voir PR suivante. See #14579 --- .../docs/L4_decision_transformer.md | 17 ++++ .../scripts/train_dt_multiseed.py | 15 +++- .../scripts/validate_xrp_dt_holdout.py | 89 ++++++++++++++++--- 3 files changed, 107 insertions(+), 14 deletions(-) diff --git a/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/docs/L4_decision_transformer.md b/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/docs/L4_decision_transformer.md index 2a3caadbff..489ac607b1 100644 --- a/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/docs/L4_decision_transformer.md +++ b/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/docs/L4_decision_transformer.md @@ -94,6 +94,23 @@ dans le temps. Le BEATS du ladder est donc **(panel @10bps)**, pas absolu. Résultats détaillés : `results/xrp_dt_validation/holdout_internal_20260806_093143.json` et `holdout_fresh_20260806_094210.json` (gitignored, machine d'entraînement). +## Protocole OOT (#14579) — verdict sur données réellement futures + +L'issue #14579 demande le premier verdict **out-of-time réel** : l'entraînement est gelé à +une borne, le holdout est postérieur et **absent de tous les folds et de toute sélection +d'hyperparamètres**. Instrument mis en place le 04/09 : + +| Brique | Où | Rôle | +|---|---|---| +| **Gel de borne** `--train-end` | `scripts/train_dt_multiseed.py` | Tronque le CSV à la borne inclusive AVANT le walk-forward ; le hash est recalculé sur la tranche gelée (l'absence de contamination devient vérifiable). | +| **Jambe de précision §C** | `scripts/validate_xrp_dt_holdout.py` | Erreur directionnelle `e_t = position_t − sign(r_{t+1})` — une politique alignée au marché fait `e = 0`, une politique opposée `e = ±2` : le DM mse/mae sur `e` est sign-aware pour des positions ±1 (le mse direct sur les retours positionnés reste sign-blind, `(−r)² = r²`, cf #10228). | +| **Contrôle de biais** | idem | DM `linear` sur les retours positionnés (différentiel de performance moyenne), **jamais la jambe de conjonction** (§C amendé #11010) ; biais signés par modèle (`mean(return)` DT / momentum / BH) rapportés dans le JSON. | +| **Conjonction** | idem | `edge ≥ 2σ` cross-seed **et** `dm_p_mse_median < 0.05` (mae rapportée en variante), sinon `NO-BEATS` / `INCONCLUSIVE` explicites. | + +Dataset de référence (04/09, yfinance, gitignored) : `XRP-USD.csv` — bornes +`2018-01-01 → 2026-09-04` (3169 rows), sha256 `71d8aee9d0bda18b`. Le verdict du run OOT +(label `oot`, `--train-end 2025-06-30`) est rapporté dans cette section une fois terminé. + ## Implication pour le ladder | Échelon | Paradigme | Verdict | diff --git a/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/train_dt_multiseed.py b/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/train_dt_multiseed.py index 4b0cb41519..6ac38cc25a 100644 --- a/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/train_dt_multiseed.py +++ b/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/train_dt_multiseed.py @@ -82,6 +82,7 @@ def run_single_seed_coin( nhead: int = 4, num_layers: int = 3, context_length: int = 20, + train_end: str | None = None, window: int = 20, batch_size: int = 32, lr: float = 1e-4, @@ -101,7 +102,15 @@ def run_single_seed_coin( data_hash = "synthetic-dryrun" else: raw = load_data(CRYPTO_DIR, coin) - data_hash = compute_data_hash(raw) + if train_end is not None: + # Gel de la borne d'entrainement (#14579) : tout ce qui est + # posterieur a train_end est retro-grade OOT -- aucun fold ni + # aucune selection d'hyperparametres ne doit le voir. + import pandas as pd + raw = raw.loc[raw.index <= pd.Timestamp(train_end)] + data_hash = compute_data_hash(raw) + else: + data_hash = compute_data_hash(raw) # Feature engineering indicators = [ @@ -343,6 +352,9 @@ def main(): parser.add_argument("--context-length", type=int, default=20) parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--lr", type=float, default=1e-4) + parser.add_argument("--train-end", default=None, + help="Borne de gel : tronque les donnees a cette date " + "inclusive AVANT le walk-forward (#14579 OOT).") args = parser.parse_args() device = "cuda" if torch.cuda.is_available() else "cpu" @@ -390,6 +402,7 @@ def main(): lr=args.lr, device=device, dry_run=args.dry_run, + train_end=args.train_end, ) elapsed = time.time() - t0 diff --git a/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/validate_xrp_dt_holdout.py b/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/validate_xrp_dt_holdout.py index 4c19b34bcf..1f5db587d9 100644 --- a/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/validate_xrp_dt_holdout.py +++ b/MyIA.AI.Notebooks/QuantConnect/ML-Training-Pipeline/scripts/validate_xrp_dt_holdout.py @@ -128,26 +128,59 @@ def run_one_seed_holdout(seed: int, raw: pd.DataFrame, train_end: str, dm = None dm_mom = None + dm_prec = None if len(dt_n) > 30: - # loss_fn="linear" (#10228): mse/mae are symmetric ((-r)**2 == r**2) - # and made the test sign-blind -- a winning and a losing return series - # got bit-identical dm_stat. Linear loss L(e) = e preserves the sign, - # so d = (-bh_g) - (-dt_n) = dt_n - bh_g and E[d] < 0 <=> DT beats BH. + # Controle de biais (linear, #10228/#10956) : L(e) = e sur les retours + # positionnes, d_mean = perf_moyenne_a - perf_moyenne_b. Jambe de + # DIAGNOSTIC uniquement -- jamais la jambe de conjonction §C (#11010). try: r = DM.diebold_mariano_test(-dt_n, -bh_g, loss_fn="linear", hln_correction=True) - dm = {"dm_stat": round(r.dm_statistic, 4), "p_value": round(r.p_value, 4), + dm = {"loss_fn": "linear", "role": "bias_control", + "dm_stat": round(r.dm_statistic, 4), "p_value": round(r.p_value, 4), "n_obs": int(r.n_observations)} except Exception as e: - dm = {"error": str(e)} + dm = {"loss_fn": "linear", "role": "bias_control", "error": str(e)} # Momentum is the real adversary: BH is degenerate when the asset falls # (fresh-window BH sharpe was -1.80). A DM vs naked momentum is what # makes the third conjunct of pr-review-discipline section C informative. try: r_mom = DM.diebold_mariano_test(-dt_n, -mom_n, loss_fn="linear", hln_correction=True) - dm_mom = {"dm_stat": round(r_mom.dm_statistic, 4), "p_value": round(r_mom.p_value, 4), + dm_mom = {"loss_fn": "linear", "role": "bias_control", + "dm_stat": round(r_mom.dm_statistic, 4), "p_value": round(r_mom.p_value, 4), "n_obs": int(r_mom.n_observations)} except Exception as e: - dm_mom = {"error": str(e)} + dm_mom = {"loss_fn": "linear", "role": "bias_control", "error": str(e)} + # Jambe de PRECISION §C (#11010, #14579) : erreur directionnelle + # e_t = position_t - sign(retour_{t+1}). Une politique alignee au marche + # fait e=0, une politique opposee e=+-2 : mse/mae sur e sont sign-aware + # pour des positions +-1. (mse/mae directement sur les retours + # positionnes restent sign-blind -- (-r)**2 == r**2, cf #10228 -- donc + # la precision directionnelle est la construction applicable ici.) + try: + # Alignement min_len, meme idiome que net_returns/gross_returns : + # dt_positions_on_test peut rendre plus de predictions que de steps. + L = min(len(dt_pos), len(test_returns_full)) + target_sign = np.sign(test_returns_full[:L]) + e_dt = dt_pos[:L] - target_sign + e_bh = bh_pos[:L] - target_sign + e_mom = mom_pos[:L] - target_sign + + def _prec(ea, eb, loss_fn): + rp = DM.diebold_mariano_test(ea, eb, loss_fn=loss_fn, hln_correction=True) + return {"loss_fn": loss_fn, "role": "precision_conjunction", + "dm_stat": round(rp.dm_statistic, 4), "p_value": round(rp.p_value, 4), + "mean_loss_diff": round(rp.mean_loss_diff, 6), + "n_obs": int(rp.n_observations)} + + dm_prec = { + "error_def": "e_t = position_t - sign(r_{t+1})", + "dt_vs_bh_mse": _prec(e_dt, e_bh, "mse"), + "dt_vs_bh_mae": _prec(e_dt, e_bh, "mae"), + "dt_vs_momentum_mse": _prec(e_dt, e_mom, "mse"), + "dt_vs_momentum_mae": _prec(e_dt, e_mom, "mae"), + } + except Exception as e: + dm_prec = {"error": str(e)} del model, result if torch.cuda.is_available(): @@ -164,8 +197,14 @@ def run_one_seed_holdout(seed: int, raw: pd.DataFrame, train_end: str, "dt_gross_sharpe": round(sharpe(dt_g), 4), "momentum_naked_net_sharpe": round(sharpe(mom_n), 4), "bh_sharpe": round(sharpe(bh_g), 4), + "bias_mean_returns": { + "dt": round(float(np.mean(dt_n)), 6), + "momentum": round(float(np.mean(mom_n)), 6), + "bh": round(float(np.mean(bh_g)), 6), + }, "dm_dt_vs_bh": dm, "dm_dt_vs_momentum": dm_mom, + "dm_precision": dm_prec, "dt_net": [round(float(x), 6) for x in dt_n], "bh_gross": [round(float(x), 6) for x in bh_g], "momentum_net": [round(float(x), 6) for x in mom_n], @@ -247,9 +286,12 @@ def main(): num_layers=args.num_layers, device=device, commission_bps=args.commission_bps) seed_results.append(r) + prec = r.get("dm_precision") or {} + p_mse = prec.get("dt_vs_bh_mse", {}).get("p_value") \ + if isinstance(prec, dict) else None print(f" net={r['dt_net_sharpe']} gross={r['dt_gross_sharpe']} " f"mom_net={r['momentum_naked_net_sharpe']} bh={r['bh_sharpe']} " - f"dm_p={r['dm_dt_vs_bh'].get('p_value') if r.get('dm_dt_vs_bh') else None}", + f"dm_mse_p={p_mse}", flush=True) # Agregation cross-seed. BH est deterministe (meme serie pour toutes les seeds). @@ -259,9 +301,25 @@ def main(): seeds_beat = int(np.sum(dt_nets > bh)) edge_sigma = float((dt_nets.mean() - bh) / (dt_nets.std(ddof=1) + 1e-12)) \ if len(dt_nets) > 1 else 0.0 - dm_ps = [r["dm_dt_vs_bh"]["p_value"] for r in seed_results - if r.get("dm_dt_vs_bh") and "p_value" in r["dm_dt_vs_bh"]] + # Conjonction §C (#11010, #14579) : la jambe DM est la PRECISION + # directionnelle (mse), linear reste rapporte comme controle de biais. + dm_ps = [r["dm_precision"]["dt_vs_bh_mse"]["p_value"] for r in seed_results + if r.get("dm_precision") and isinstance(r["dm_precision"], dict) + and "p_value" in r["dm_precision"].get("dt_vs_bh_mse", {})] dm_p_median = float(np.median(dm_ps)) if dm_ps else None + dm_ps_mae = [r["dm_precision"]["dt_vs_bh_mae"]["p_value"] for r in seed_results + if r.get("dm_precision") and isinstance(r["dm_precision"], dict) + and "p_value" in r["dm_precision"].get("dt_vs_bh_mae", {})] + dm_p_mae_median = float(np.median(dm_ps_mae)) if dm_ps_mae else None + dm_ps_lin = [r["dm_dt_vs_bh"]["p_value"] for r in seed_results + if r.get("dm_dt_vs_bh") and "p_value" in r["dm_dt_vs_bh"]] + dm_p_linear_median = float(np.median(dm_ps_lin)) if dm_ps_lin else None + bias_report = { + "dt_mean_net_return": round(float(np.mean( + [r["bias_mean_returns"]["dt"] for r in seed_results])), 6), + "momentum_mean_net_return": seed_results[0]["bias_mean_returns"]["momentum"], + "bh_mean_gross_return": seed_results[0]["bias_mean_returns"]["bh"], + } if dt_nets.mean() <= bh: verdict = "NO-BEATS" @@ -289,7 +347,10 @@ def main(): "momentum_naked_net_sharpe": mom_net, "seeds_beat_bh": f"{seeds_beat}/{len(dt_nets)}", "edge_sigma": round(edge_sigma, 2), - "dm_p_median": dm_p_median, + "dm_p_mse_median": dm_p_median, + "dm_p_mae_median": dm_p_mae_median, + "dm_p_linear_median_bias_control": dm_p_linear_median, + "bias_report": bias_report, }, "verdict": verdict, } @@ -301,7 +362,9 @@ def main(): print(f" DT net {dt_nets.mean():.3f} (+/- {dt_nets.std(ddof=1):.3f}) vs BH {bh:.3f} " f"| mom_naked net {mom_net:.3f}") print(f" seeds>{'BH'}: {seeds_beat}/{len(dt_nets)} | edge {edge_sigma:.2f} sigma " - f"| DM p mediane {dm_p_median}") + f"| DM mse p mediane {dm_p_median} (mae {dm_p_mae_median}, " + f"linear-ctrl {dm_p_linear_median})") + print(f" biais moyens (retour/jour): {bias_report}") print(f" -> {out_path}")