Merge pull request #48 from Kirchberg/codex/codex-desktop-harvest

Add Codex Desktop transcript harvesting

Author: carpedkm

README.md

@@ -70,7 +70,7 @@ Dreams** (offline consolidation; review-then-adopt), and the **agent sleep**
 idea (short-term experience → long-term competence). One "night":
 
 ```
-harvest session transcripts → mine recurring tasks → replay offline
+harvest Claude Code / Codex Desktop transcripts → mine recurring tasks → replay offline
    → consolidate (reflect → bounded edit → GATE on real held-out tasks)
    → stage proposal → (you) adopt
 ```
@@ -99,6 +99,11 @@ positive, and the gate blocks regressions
 
 Deterministic proof (no API key): `python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves`.
 
+For local sleep cycles, transcript source and replay backend are separate knobs:
+use `--source claude` for Claude Code transcripts, `--source codex` for Codex
+Desktop archived sessions under `~/.codex/archived_sessions`, and
+`--backend codex` only when you want the replay/optimizer to spend Codex budget.
+
 ---
 
 ## Extensibility & WebUI

docs/sleep/CONTROLLABLE_DREAMING.md

@@ -4,6 +4,23 @@ The sleep engine is no longer a single fixed pipeline. It is a controllable
 offline "dream / imagination" loop the user steers. This documents the knobs
 added in the four-stage refactor and how they map to the user's design.
 
+## Transcript sources
+
+Sleep separates the source of past sessions from the backend used to replay and
+optimize tasks:
+
+```bash
+python -m skillopt_sleep dry-run --project "$(pwd)" --source claude --backend mock
+python -m skillopt_sleep dry-run --project "$(pwd)" --source codex --backend mock
+python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex
+```
+
+`--source claude` reads Claude Code transcripts from `~/.claude/projects`.
+`--source codex` reads Codex Desktop archives from
+`~/.codex/archived_sessions`. `--source auto` tries Codex archives first, then
+falls back to Claude Code transcripts. Use `--codex-home /path/to/.codex` or
+`--claude-home /path/to/.claude` to point at non-default homes.
+
 ## The mental model
 
 > Sleep = an offline imagination rollout. Re-run the user's real

plugins/codex/README.md

@@ -48,13 +48,18 @@ Use the skillopt-sleep skill to adopt the latest staged proposal.
 Or call the engine directly:
 
 ```bash
-python -m skillopt_sleep run --project "$(pwd)" --backend codex
+python -m skillopt_sleep dry-run --project "$(pwd)" --source codex --backend mock
+python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex
 ```
 
-Default backend is `mock` (no API spend). `--backend codex` uses your Codex
-budget for real improvement. All the controllable knobs (`--gate on|off`,
-`--rollouts-k`, `--budget-tokens`, `--preferences`, optimizer/target split) work
-identically — see [`../../docs/sleep/CONTROLLABLE_DREAMING.md`](../../docs/sleep/CONTROLLABLE_DREAMING.md).
+`--source codex` reads Codex Desktop archived sessions from
+`~/.codex/archived_sessions`. Use `--codex-home /path/to/.codex` to point at a
+different Codex home, or `--source auto` to try Codex archives first and fall
+back to Claude Code transcripts. Default backend is `mock` (no API spend).
+`--backend codex` uses your Codex budget for real improvement. All the
+controllable knobs (`--gate on|off`, `--rollouts-k`, `--budget-tokens`,
+`--preferences`, optimizer/target split) work identically — see
+[`../../docs/sleep/CONTROLLABLE_DREAMING.md`](../../docs/sleep/CONTROLLABLE_DREAMING.md).
 
 ## Notes / status
 

plugins/codex/skills/skillopt-sleep/SKILL.md

@@ -44,18 +44,20 @@ finds the engine and a Python >= 3.10 automatically.
 ```bash
 # point at the repo if it isn't auto-detected from CWD:
 export SKILLOPT_SLEEP_REPO=/path/to/SkillOpt-Sleep
-
 bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" status --project "$(pwd)"
 bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" harvest --project "$(pwd)"
 bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" dry-run --project "$(pwd)" --backend mock
 bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run --project "$(pwd)" --backend codex
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run --project "$(pwd)" --source codex  # harvest from Codex Desktop
 bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" adopt --project "$(pwd)"
 ```
 
 Actions are `status`, `harvest`, `dry-run`, `run`, and `adopt`.
 
 - Default backend is `mock`, which is deterministic and spends no API budget.
 - `--backend codex` uses the user's Codex budget for real improvement.
+- `--source codex` reads Codex Desktop archived sessions from `~/.codex/archived_sessions`;
+  use `--codex-home /path/to/.codex` if the archive lives elsewhere.
 - Keep `dry-run --backend mock` as the first smoke check unless the user
   explicitly asked for a real optimization run.
 

skillopt_sleep/__main__.py

@@ -9,7 +9,8 @@
 Common flags:
     --project PATH      project to evolve (default: cwd)
     --scope all|invoked harvest scope (default: invoked)
-    --backend mock|anthropic
+    --backend mock|claude|codex
+    --source claude|codex|auto
     --model NAME
     --lookback-hours N
     --auto-adopt
@@ -25,10 +26,11 @@
 
 from skillopt_sleep.config import load_config
 from skillopt_sleep.cycle import run_sleep_cycle
-from skillopt_sleep.harvest import harvest
+from skillopt_sleep.harvest_sources import harvest_for_config
 from skillopt_sleep.mine import mine
+from skillopt_sleep.staging import adopt as adopt_staging
+from skillopt_sleep.staging import latest_staging
 from skillopt_sleep.state import SleepState
-from skillopt_sleep.staging import latest_staging, adopt as adopt_staging
 
 
 def _add_common(p: argparse.ArgumentParser) -> None:
@@ -38,6 +40,9 @@ def _add_common(p: argparse.ArgumentParser) -> None:
     p.add_argument("--model", default="")
     p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
     p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)")
+    p.add_argument("--codex-home", default="", help="override ~/.codex for archived session harvest")
+    p.add_argument("--source", default="", choices=["", "claude", "codex", "auto"],
+                   help="session transcript source")
     p.add_argument("--lookback-hours", type=int, default=0)
     p.add_argument("--edit-budget", type=int, default=0)
     p.add_argument("--auto-adopt", action="store_true")
@@ -59,6 +64,10 @@ def _cfg_from_args(args) -> Any:
         overrides["codex_path"] = os.path.abspath(args.codex_path)
     if getattr(args, "claude_home", ""):
         overrides["claude_home"] = os.path.abspath(args.claude_home)
+    if getattr(args, "codex_home", ""):
+        overrides["codex_home"] = os.path.abspath(args.codex_home)
+    if getattr(args, "source", ""):
+        overrides["transcript_source"] = args.source
     if getattr(args, "lookback_hours", 0):
         overrides["lookback_hours"] = args.lookback_hours
     if getattr(args, "edit_budget", 0):
@@ -143,12 +152,7 @@ def cmd_adopt(args) -> int:
 
 def cmd_harvest(args) -> int:
     cfg = _cfg_from_args(args)
-    digests = harvest(
-        cfg.transcripts_dir,
-        scope=cfg.get("projects", "invoked"),
-        invoked_project=cfg.get("invoked_project", ""),
-        limit=cfg.get("max_tasks_per_night", 40) * 3,
-    )
+    digests = harvest_for_config(cfg, limit=cfg.get("max_tasks_per_night", 40) * 3)
     tasks = mine(digests, max_tasks=cfg.get("max_tasks_per_night", 40),
                  holdout_fraction=cfg.get("holdout_fraction", 0.34), seed=cfg.get("seed", 42))
     if args.json:

skillopt_sleep/config.py

@@ -13,17 +13,19 @@
 
 import json
 import os
-from dataclasses import dataclass, field, asdict
-from typing import Any, Dict, List, Optional
-
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
 
 HOME_STATE_DIR = os.path.expanduser("~/.skillopt-sleep")
 CLAUDE_HOME = os.path.expanduser("~/.claude")
+CODEX_HOME = os.path.expanduser("~/.codex")
 
 
 DEFAULTS: Dict[str, Any] = {
     # ── scope ──────────────────────────────────────────────────────────────
     "claude_home": CLAUDE_HOME,
+    "codex_home": CODEX_HOME,
+    "transcript_source": "claude",  # "claude" | "codex" | "auto"
     "projects": "invoked",        # "invoked" | "all" | [list of abs paths]
     "invoked_project": "",        # filled at runtime (cwd) when projects == "invoked"
     "lookback_hours": 72,         # harvest window when no prior sleep recorded
@@ -94,6 +96,10 @@ def state_path(self) -> str:
     def transcripts_dir(self) -> str:
         return os.path.join(self.data["claude_home"], "projects")
 
+    @property
+    def codex_archived_sessions_dir(self) -> str:
+        return os.path.join(self.data["codex_home"], "archived_sessions")
+
     @property
     def history_path(self) -> str:
         return os.path.join(self.data["claude_home"], "history.jsonl")

skillopt_sleep/cycle.py

@@ -10,18 +10,18 @@
 from __future__ import annotations
 
 import os
-import time
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from typing import List, Optional
 
 from skillopt_sleep.backend import get_backend
 from skillopt_sleep.config import SleepConfig, load_config
 from skillopt_sleep.consolidate import consolidate
-from skillopt_sleep.harvest import harvest
+from skillopt_sleep.harvest_sources import harvest_for_config
 from skillopt_sleep.memory import ensure_skill_scaffold
 from skillopt_sleep.mine import mine
+from skillopt_sleep.staging import adopt as adopt_staging
+from skillopt_sleep.staging import write_staging
 from skillopt_sleep.state import SleepState, _now_iso
-from skillopt_sleep.staging import write_staging, adopt as adopt_staging
 from skillopt_sleep.types import SessionDigest, SleepReport, TaskRecord
 
 
@@ -117,10 +117,8 @@ def run_sleep_cycle(
         n_sessions = 0
     else:
         since = state.last_harvest_for(project)
-        digests = harvest(
-            cfg.transcripts_dir,
-            scope=cfg.get("projects", "invoked"),
-            invoked_project=cfg.get("invoked_project", ""),
+        digests = harvest_for_config(
+            cfg,
             since_iso=since,
             limit=cfg.get("max_tasks_per_night", 40) * 3,
         )
@@ -151,7 +149,7 @@ def run_sleep_cycle(
     if not skill:
         skill = ensure_skill_scaffold(
             "", name=cfg.get("managed_skill_name", "skillopt-sleep-learned"),
-            description="Preferences and procedures learned from past Claude Code sessions.",
+            description="Preferences and procedures learned from past local agent sessions.",
         )
 
     report = SleepReport(