diff --git a/MyIA.AI.Notebooks/QuantConnect/projects/Research-Executor/research_long_short_harvest.ipynb b/MyIA.AI.Notebooks/QuantConnect/projects/Research-Executor/research_long_short_harvest.ipynb index 51a5119faf..2b3afe126b 100644 --- a/MyIA.AI.Notebooks/QuantConnect/projects/Research-Executor/research_long_short_harvest.ipynb +++ b/MyIA.AI.Notebooks/QuantConnect/projects/Research-Executor/research_long_short_harvest.ipynb @@ -2,7 +2,17 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "id": "80ec66bb", + "metadata": { + "papermill": { + "duration": 0.003846, + "end_time": "2026-09-28T01:41:23.413385", + "exception": false, + "start_time": "2026-09-28T01:41:23.409539", + "status": "completed" + }, + "tags": [] + }, "source": [ "# Long-Short Volatility Harvest ML - Research Notebook\n", "\n", @@ -17,19 +27,29 @@ { "cell_type": "code", "execution_count": 1, + "id": "61d2900c", "metadata": { "execution": { - "iopub.execute_input": "2026-05-12T18:47:59.269406Z", - "iopub.status.busy": "2026-05-12T18:47:59.269264Z", - "iopub.status.idle": "2026-05-12T18:48:00.089507Z", - "shell.execute_reply": "2026-05-12T18:48:00.088714Z" - } + "iopub.execute_input": "2026-09-28T01:41:23.421644Z", + "iopub.status.busy": "2026-09-28T01:41:23.420634Z", + "iopub.status.idle": "2026-09-28T01:41:24.007941Z", + "shell.execute_reply": "2026-09-28T01:41:24.007434Z" + }, + "papermill": { + "duration": 0.592247, + "end_time": "2026-09-28T01:41:24.009920", + "exception": false, + "start_time": "2026-09-28T01:41:23.417673", + "status": "completed" + }, + "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Local environment - loading SPY, GLD and ^VIX via yfinance\n", "Long book: Top 4 market cap stocks (monthly rebalance)\n", "Short book: Hurst > 0.85 + extension + momentum (weekly rebalance)\n", "ML model: RandomForestClassifier (11 features, VIX + SPY)\n" @@ -37,12 +57,24 @@ } ], "source": [ - "from AlgorithmImports import *\n", - "qb = QuantBook()\n", + "try:\n", + " from AlgorithmImports import *\n", + " qb = QuantBook()\n", + " QC_ENV = True\n", + "except (ImportError, NameError):\n", + " import pandas as pd\n", + " import numpy as np\n", + " from datetime import timedelta\n", + " QC_ENV = False\n", + " qb = None\n", "\n", - "# Define universe\n", - "qb.AddEquity(\"SPY\", Resolution.Daily)\n", - "qb.AddEquity(\"GLD\", Resolution.Daily)\n", + "if QC_ENV:\n", + " qb.AddEquity(\"SPY\", Resolution.Daily)\n", + " qb.AddEquity(\"GLD\", Resolution.Daily)\n", + "else:\n", + " # Local research path: SPY + GLD + the real ^VIX index via yfinance --\n", + " # the three inputs the strategy's ML core is built on.\n", + " print(\"Local environment - loading SPY, GLD and ^VIX via yfinance\")\n", "\n", "print(f\"Long book: Top 4 market cap stocks (monthly rebalance)\")\n", "print(f\"Short book: Hurst > 0.85 + extension + momentum (weekly rebalance)\")\n", @@ -51,7 +83,17 @@ }, { "cell_type": "markdown", - "metadata": {}, + "id": "fa81149b", + "metadata": { + "papermill": { + "duration": 0.002889, + "end_time": "2026-09-28T01:41:24.015590", + "exception": false, + "start_time": "2026-09-28T01:41:24.012701", + "status": "completed" + }, + "tags": [] + }, "source": [ "## Strategy Logic\n", "\n", @@ -77,34 +119,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, + "id": "9dea8ad3", "metadata": { "execution": { - "iopub.execute_input": "2026-05-12T18:48:00.109578Z", - "iopub.status.busy": "2026-05-12T18:48:00.109431Z", - "iopub.status.idle": "2026-05-12T18:48:00.255742Z", - "shell.execute_reply": "2026-05-12T18:48:00.255081Z" - } + "iopub.execute_input": "2026-09-28T01:41:24.023394Z", + "iopub.status.busy": "2026-09-28T01:41:24.023394Z", + "iopub.status.idle": "2026-09-28T01:41:25.511827Z", + "shell.execute_reply": "2026-09-28T01:41:25.510667Z" + }, + "papermill": { + "duration": 1.494265, + "end_time": "2026-09-28T01:41:25.513165", + "exception": false, + "start_time": "2026-09-28T01:41:24.018900", + "status": "completed" + }, + "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SPY data: 3017 days\n", + "GLD data: 3017 days\n", + "VIX data: 3020 days (real CBOE index)\n", + "\n", + "SPY above 200-SMA: 76.7% of the time\n", + " Bull regime avg daily return: 0.0010\n", + " Bear regime avg daily return: -0.0007\n" + ] + } + ], "source": [ - "# Fetch historical data for analysis\n", - "spy_sym = qb.AddEquity(\"SPY\", Resolution.Daily).Symbol\n", - "gld_sym = qb.AddEquity(\"GLD\", Resolution.Daily).Symbol\n", - "\n", - "spy_hist = qb.History(spy_sym, timedelta(252 * 12), Resolution.Daily)\n", - "gld_hist = qb.History(gld_sym, timedelta(252 * 12), Resolution.Daily)\n", - "\n", - "# Convert to pandas DataFrame if needed (QC Cloud returns DataFrame, local may return enumerable)\n", - "if not hasattr(spy_hist, 'shape'):\n", - " import pandas as pd\n", - " spy_hist = pd.DataFrame([{'time': b.Time, 'close': float(b.Close)} for b in spy_hist])\n", - " spy_hist = spy_hist.set_index('time')\n", - " gld_hist = pd.DataFrame([{'time': b.Time, 'close': float(b.Close)} for b in gld_hist])\n", - " gld_hist = gld_hist.set_index('time')\n", + "# Fetch historical data for analysis (QC Cloud or local yfinance)\n", + "if QC_ENV:\n", + " spy_hist = qb.History(qb.AddEquity(\"SPY\", Resolution.Daily).Symbol, timedelta(252 * 12), Resolution.Daily)\n", + " gld_hist = qb.History(qb.AddEquity(\"GLD\", Resolution.Daily).Symbol, timedelta(252 * 12), Resolution.Daily)\n", + " vix_close = None # CBOE custom data handled by the strategy on QC Cloud\n", + "else:\n", + " import yfinance as yf\n", + " spy_hist = yf.Ticker(\"SPY\").history(period=\"12y\", auto_adjust=True)[[\"Close\"]].rename(columns={\"Close\": \"close\"})\n", + " gld_hist = yf.Ticker(\"GLD\").history(period=\"12y\", auto_adjust=True)[[\"Close\"]].rename(columns={\"Close\": \"close\"})\n", + " for h in (spy_hist, gld_hist):\n", + " h.index = h.index.tz_localize(None) # normalize mixed tz (see env-gotchas)\n", + " vix_close = yf.Ticker(\"^VIX\").history(period=\"12y\", auto_adjust=True)[\"Close\"]\n", + " vix_close.index = vix_close.index.tz_localize(None)\n", "\n", "print(f\"SPY data: {spy_hist.shape[0]} days\")\n", "print(f\"GLD data: {gld_hist.shape[0]} days\")\n", + "if vix_close is not None:\n", + " print(f\"VIX data: {vix_close.shape[0]} days (real CBOE index)\")\n", "\n", "# Regime analysis: SPY vs 200-SMA\n", "spy_close = spy_hist[\"close\"]\n", @@ -118,18 +184,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, + "id": "2bf8bfd0", "metadata": { "execution": { - "iopub.execute_input": "2026-05-12T18:48:00.257407Z", - "iopub.status.busy": "2026-05-12T18:48:00.257285Z", - "iopub.status.idle": "2026-05-12T18:48:00.270315Z", - "shell.execute_reply": "2026-05-12T18:48:00.269571Z" - } + "iopub.execute_input": "2026-09-28T01:41:25.518353Z", + "iopub.status.busy": "2026-09-28T01:41:25.518353Z", + "iopub.status.idle": "2026-09-28T01:41:25.557013Z", + "shell.execute_reply": "2026-09-28T01:41:25.557013Z" + }, + "papermill": { + "duration": 0.044016, + "end_time": "2026-09-28T01:41:25.559544", + "exception": false, + "start_time": "2026-09-28T01:41:25.515528", + "status": "completed" + }, + "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SPY return statistics:\n", + " Annualized return: 15.67%\n", + " Annualized vol: 17.47%\n", + " Sharpe (rf=0): 0.83\n", + " Max drawdown: -33.72%\n", + "\n", + "Hurst exponent (aggregated variance, measured):\n", + " SPY: 0.009 (mean-reverting)\n", + " GLD: 0.020 (mean-reverting)\n", + " Short-screen threshold H > 0.85 is an extreme tail: both benchmark\n", + " series sit well below it on 12y daily data.\n", + "\n", + "VIX quintile -> next-21d SPY return (real index, measured):\n", + " Q1: +0.82% mean (n=605)\n", + " Q2: +0.40% mean (n=592)\n", + " Q3: +0.90% mean (n=593)\n", + " Q4: +1.04% mean (n=602)\n", + " Q5: +2.87% mean (n=604)\n" + ] + } + ], "source": [ - "# VIX regime analysis\n", + "# VIX regime analysis -- measured on the real index\n", "import numpy as np\n", "\n", "spy_returns = spy_close.pct_change().dropna()\n", @@ -138,12 +238,49 @@ "print(f\" Annualized return: {(1 + spy_returns.mean())**252 - 1:.2%}\")\n", "print(f\" Annualized vol: {spy_returns.std() * (252**0.5):.2%}\")\n", "print(f\" Sharpe (rf=0): {(spy_returns.mean() / spy_returns.std()) * (252**0.5):.2f}\")\n", - "print(f\" Max drawdown: {(spy_close / spy_close.cummax() - 1).min():.2%}\")" + "print(f\" Max drawdown: {(spy_close / spy_close.cummax() - 1).min():.2%}\")\n", + "\n", + "# Hurst exponent (aggregated-variance method) -- the strategy's short-selection\n", + "# concept, measured instead of cited\n", + "def hurst_aggvar(returns, lags=(2, 5, 10, 20, 40)):\n", + " x = np.log(returns[returns > 0]) # drop zero/negative returns (log undefined)\n", + " vars_ = {k: x.diff(k).dropna().std() for k in lags}\n", + " logs = np.array([[np.log(k), np.log(v)] for k, v in vars_.items()\n", + " if np.isfinite(v) and v > 0])\n", + " if logs.shape[0] < 2:\n", + " return float(\"nan\")\n", + " return np.polyfit(logs[:, 0], logs[:, 1], 1)[0]\n", + "\n", + "h_spy = hurst_aggvar(spy_returns)\n", + "h_gld = hurst_aggvar(gld_hist[\"close\"].pct_change().dropna())\n", + "print(f\"\\nHurst exponent (aggregated variance, measured):\")\n", + "print(f\" SPY: {h_spy:.3f} ({'trending' if h_spy > 0.5 else 'mean-reverting'})\")\n", + "print(f\" GLD: {h_gld:.3f} ({'trending' if h_gld > 0.5 else 'mean-reverting'})\")\n", + "print(f\" Short-screen threshold H > 0.85 is an extreme tail: both benchmark\")\n", + "print(f\" series sit well below it on 12y daily data.\")\n", + "\n", + "if vix_close is not None:\n", + " fwd21 = spy_close.pct_change(21).shift(-21)\n", + " q = pd.qcut(vix_close.reindex(spy_close.index).ffill(), 5, labels=False)\n", + " tbl = pd.DataFrame({\"vix_q\": q, \"fwd\": fwd21}).dropna().groupby(\"vix_q\")[\"fwd\"].agg([\"mean\", \"count\"])\n", + " print(f\"\\nVIX quintile -> next-21d SPY return (real index, measured):\")\n", + " for qi, row in tbl.iterrows():\n", + " print(f\" Q{qi + 1}: {row['mean']:+.2%} mean (n={int(row['count'])})\")" ] }, { "cell_type": "markdown", - "metadata": {}, + "id": "f040dce1", + "metadata": { + "papermill": { + "duration": 0.002435, + "end_time": "2026-09-28T01:41:25.564947", + "exception": false, + "start_time": "2026-09-28T01:41:25.562512", + "status": "completed" + }, + "tags": [] + }, "source": [ "## Key Observations\n", "\n", @@ -171,54 +308,159 @@ { "cell_type": "code", "execution_count": 4, + "id": "dc190dde", "metadata": { "execution": { - "iopub.execute_input": "2026-05-12T18:48:00.271620Z", - "iopub.status.busy": "2026-05-12T18:48:00.271506Z", - "iopub.status.idle": "2026-05-12T18:48:00.274723Z", - "shell.execute_reply": "2026-05-12T18:48:00.274017Z" - } + "iopub.execute_input": "2026-09-28T01:41:25.571004Z", + "iopub.status.busy": "2026-09-28T01:41:25.570004Z", + "iopub.status.idle": "2026-09-28T01:41:29.202892Z", + "shell.execute_reply": "2026-09-28T01:41:29.201768Z" + }, + "papermill": { + "duration": 3.637089, + "end_time": "2026-09-28T01:41:29.204514", + "exception": false, + "start_time": "2026-09-28T01:41:25.567425", + "status": "completed" + }, + "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "==================================================\n", - "BACKTEST RESULTS (12-year)\n", - "==================================================\n", - "Run backtest via QC MCP to populate metrics:\n", - " create_compile -> create_backtest -> read_backtest\n", + "ML regime core, measured out-of-sample (temporal 70/30 split):\n", + " AUC: 0.386 | accuracy: 61.20% vs majority baseline 69.76%\n", + " Honest verdict on this window: NO EDGE -- the ML tilt's input signal is weak;\n", + " the production engine adds Hurst screens and stops on top.\n", "\n", - "Expected characteristics:\n", - " - CAGR: ~12-20% (long-short with leverage)\n", - " - Max Drawdown: ~15-25%\n", - " - Sharpe: ~0.8-1.3\n", - " - Rebalance: Daily signal, weekly shorts, monthly ML retrain\n", - " - Universe: 2000 coarse -> 150 fine -> 4 long + 1 short\n", - " - Margin account required for short selling\n" + "============================================================\n", + "NOT MEASURED HERE: the full long-short engine -- top-4 market-cap\n", + "longs with 3-stage trailing stops, weekly Hurst short book, margin.\n", + "These need live universe selection and brokerage simulation:\n", + "-> QC Cloud, companion strategy project.\n", + "============================================================\n" ] } ], "source": [ - "# Backtest results placeholder\n", - "print(\"=\" * 50)\n", - "print(\"BACKTEST RESULTS (12-year)\")\n", - "print(\"=\" * 50)\n", - "print(\"Run backtest via QC MCP to populate metrics:\")\n", - "print(\" create_compile -> create_backtest -> read_backtest\")\n", + "# The promised RandomForestClassifier on VIX/SPY features -- trained and measured\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import roc_auc_score\n", + "\n", + "if vix_close is not None:\n", + " vix = vix_close.reindex(spy_close.index).ffill()\n", + " feat = pd.DataFrame(index=spy_close.index)\n", + " feat[\"vix_level\"] = vix\n", + " feat[\"vix_zscore\"] = (vix - vix.rolling(252).mean()) / vix.rolling(252).std()\n", + " feat[\"vix_ratio_short\"] = vix / vix.rolling(10).mean()\n", + " feat[\"vix_ratio_med\"] = vix / vix.rolling(60).mean()\n", + " feat[\"spy_5d\"] = spy_close.pct_change(5)\n", + " feat[\"spy_10d\"] = spy_close.pct_change(10)\n", + " feat[\"spy_20d\"] = spy_close.pct_change(20)\n", + " feat[\"spy_vs_sma50\"] = spy_close / spy_close.rolling(50).mean()\n", + " feat[\"spy_vs_sma200\"] = spy_close / spy_close.rolling(200).mean()\n", + " feat[\"realized_vol\"] = spy_returns.rolling(21).std() * np.sqrt(252)\n", + " feat[\"cross\"] = feat[\"vix_zscore\"] * feat[\"spy_20d\"]\n", + "\n", + " target = (spy_close.pct_change(21).shift(-21) > 0).astype(int)\n", + " data = feat.join(target.rename(\"y\")).dropna()\n", + " split = int(len(data) * 0.7)\n", + " clf = RandomForestClassifier(n_estimators=300, min_samples_leaf=20, random_state=42, n_jobs=-1)\n", + " clf.fit(data.iloc[:split, :-1], data.iloc[:split, -1])\n", + " auc = roc_auc_score(data.iloc[split:, -1], clf.predict_proba(data.iloc[split:, :-1])[:, 1])\n", + " acc = clf.score(data.iloc[split:, :-1], data.iloc[split:, -1])\n", + " base = data.iloc[split:, -1].mean()\n", + " print(f\"ML regime core, measured out-of-sample (temporal 70/30 split):\")\n", + " print(f\" AUC: {auc:.3f} | accuracy: {acc:.2%} vs majority baseline {max(base, 1-base):.2%}\")\n", + " verdict = \"NO EDGE\" if auc < 0.55 else \"marginal\"\n", + " print(f\" Honest verdict on this window: {verdict} -- the ML tilt's input signal is weak;\")\n", + " print(f\" the production engine adds Hurst screens and stops on top.\")\n", + "else:\n", + " print(\"ML core: VIX custom data belongs to the QC Cloud engine (not available locally)\")\n", + "\n", "print()\n", - "print(\"Expected characteristics:\")\n", - "print(\" - CAGR: ~12-20% (long-short with leverage)\")\n", - "print(\" - Max Drawdown: ~15-25%\")\n", - "print(\" - Sharpe: ~0.8-1.3\")\n", - "print(\" - Rebalance: Daily signal, weekly shorts, monthly ML retrain\")\n", - "print(\" - Universe: 2000 coarse -> 150 fine -> 4 long + 1 short\")\n", - "print(\" - Margin account required for short selling\")" + "print(\"=\" * 60)\n", + "print(\"NOT MEASURED HERE: the full long-short engine -- top-4 market-cap\")\n", + "print(\"longs with 3-stage trailing stops, weekly Hurst short book, margin.\")\n", + "print(\"These need live universe selection and brokerage simulation:\")\n", + "print(\"-> QC Cloud, companion strategy project.\")\n", + "print(\"=\" * 60)" + ] + }, + { + "cell_type": "markdown", + "id": "7425d8f0", + "metadata": { + "papermill": { + "duration": 0.001832, + "end_time": "2026-09-28T01:41:29.210173", + "exception": false, + "start_time": "2026-09-28T01:41:29.208341", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Reading the measurements\n", + "\n", + "- **VIX quintiles vs 21-day forward SPY returns** are the notebook's core\n", + " empirical claim, now measured on the real index instead of asserted.\n", + "- **The Hurst exponent of SPY and GLD** (aggregated-variance estimator) sits\n", + " far below the strategy's H > 0.85 short-screen threshold on 12y daily data --\n", + " informative about how selective that screen really is.\n", + "- **The ML core is trained for real** with an honest out-of-sample verdict; an\n", + " AUC near or below 0.5 means the 21-day direction signal is weak on this\n", + " window, and the notebook says so instead of hedging in prose.\n", + "- The full long-short machinery (market-cap universe, weekly shorts, stops,\n", + " margin) is engine territory -- QC Cloud, not a research notebook." + ] + }, + { + "cell_type": "markdown", + "id": "46243125", + "metadata": { + "papermill": { + "duration": 0.002162, + "end_time": "2026-09-28T01:41:29.214914", + "exception": false, + "start_time": "2026-09-28T01:41:29.212752", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Reading the measurements\n", + "\n", + "- **VIX quintiles vs 21-day forward SPY returns** are the notebook's core\n", + " empirical claim, now measured on the real index instead of asserted.\n", + "- **The Hurst exponent of SPY and GLD** (aggregated-variance estimator) sits\n", + " far below the strategy's H > 0.85 short-screen threshold on 12y daily data --\n", + " informative about how selective that screen really is.\n", + "- **The ML core is trained for real** with an honest out-of-sample verdict; an\n", + " AUC near or below 0.5 means the 21-day direction signal is weak on this\n", + " window, and the notebook says so instead of hedging in prose.\n", + "- The full long-short machinery (market-cap universe, weekly shorts, stops,\n", + " margin) is engine territory -- QC Cloud, not a research notebook." ] } ], "metadata": { + "cost": { + "api_provider": "none", + "api_usd_est": 0.0, + "cpu_min": 0, + "external_account": "quantconnect-organization", + "free_alternative": null, + "gpu_required": false, + "metadata_written": "2026-07-26", + "network": true, + "qcc_tokens_est": 400, + "reduced_pedagogical": null, + "reproducibility": "MED", + "validator": "qc_cloud" + }, "kernelspec": { "display_name": "Python 3", "language": "python", @@ -234,23 +476,21 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.14" + "version": "3.11.9" }, - "cost": { - "api_usd_est": 0.0, - "api_provider": "none", - "qcc_tokens_est": 400, - "cpu_min": 0, - "gpu_required": false, - "network": true, - "external_account": "quantconnect-organization", - "free_alternative": null, - "reduced_pedagogical": null, - "reproducibility": "MED", - "metadata_written": "2026-07-26", - "validator": "qc_cloud" + "papermill": { + "default_parameters": {}, + "duration": 8.843037, + "end_time": "2026-09-28T01:41:29.786464", + "environment_variables": {}, + "exception": null, + "input_path": "MyIA.AI.Notebooks/QuantConnect/projects/Research-Executor/research_long_short_harvest.ipynb", + "output_path": "research_long_short_harvest.ipynb", + "parameters": {}, + "start_time": "2026-09-28T01:41:20.943427", + "version": "2.6.0" } }, "nbformat": 4, - "nbformat_minor": 2 -} + "nbformat_minor": 5 +} \ No newline at end of file