fix(audiobook,#1028): suspension auto du monitor d'idle pendant une chaine LLM->TTS

MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/idle_monitor.py

@@ -0,0 +1,92 @@
+"""Suspension du sidecar ``tts-fishaudio-idle-monitor`` pendant une fenetre de chaine.
+
+Epic #1028, residu 2 : sur le livre complet, les phases LLM amont (p3 + p4)
+durent ~36 min sans aucun appel TTS -- l'idle monitor (IDLE_TIMEOUT=1200 s)
+tue alors le service ``tts-fishaudio`` AVANT que p5 n'arrive, et la chaine
+echoue sur port refuse (385/385 echecs au premier run du livre entier, #14059).
+Le monitor est une economie GPU legitime : on le SUSPEND pendant la fenetre
+de chaine LLM->TTS puis on le redemarre, au lieu d'un geste manuel a ne pas
+oublier.
+
+Fail-safe par construction : docker absent, daemon muet, inspect/stop en
+echec -- la chaine continue TOUJOURS, au pire avec un avertissement (le
+comportement d'avant ce module etait l'echec silencieux en chaine longue).
+"""
+
+from __future__ import annotations
+
+import subprocess
+from contextlib import contextmanager
+from collections.abc import Iterator
+
+MONITOR_CONTAINER = "tts-fishaudio-idle-monitor"
+
+# Phases dont la duree est dominee par le LLM, sans appel TTS.
+_LLM_PHASES_BEFORE_TTS = {"p0", "p1", "p1_5", "p2", "p3", "p4"}
+
+_DOCKER_TIMEOUT_S = 30
+
+
+def _docker(args: list[str]) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["docker", *args], capture_output=True, text=True, timeout=_DOCKER_TIMEOUT_S
+    )
+
+
+def chain_needs_suspension(phases: list[str]) -> bool:
+    """True si la chaine contient p5 precede d'au moins une phase LLM.
+
+    ``["p5"]`` seul n'a pas besoin de suspension (p5 appelle le TTS
+    immediatement, le monitor ne le voit jamais inactif) ; ce sont les
+    chaines LLM->TTS qui tuent le service pendant les phases amont.
+    """
+    seen: set[str] = set()
+    for phase in phases:
+        if phase == "p5":
+            return bool(seen & _LLM_PHASES_BEFORE_TTS)
+        seen.add(phase)
+    return False
+
+
+def monitor_is_running() -> bool:
+    """True si le conteneur monitor existe ET tourne. Docker absent => False."""
+    try:
+        result = _docker(["inspect", "-f", "{{.State.Running}}", MONITOR_CONTAINER])
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+    return result.returncode == 0 and result.stdout.strip().lower() == "true"
+
+
+@contextmanager
+def suspended_idle_monitor(phases: list[str]) -> Iterator[bool]:
+    """Suspend le monitor pour la duree de la fenetre ; yield True si suspendu.
+
+    Le monitor n'est redemarre que si CETTE fenetre l'a arrete (jamais au
+    hasard d'un etat anterieur).
+    """
+    if not chain_needs_suspension(phases):
+        yield False
+        return
+    if not monitor_is_running():
+        print(f"[monitor] {MONITOR_CONTAINER} absent ou arrete -- chaine sans suspension")
+        yield False
+        return
+    stop = _docker(["stop", MONITOR_CONTAINER])
+    if stop.returncode != 0:
+        print(
+            f"[monitor] WARN: stop echoue (rc={stop.returncode}) -- la chaine "
+            "continue, le monitor peut tuer tts-fishaudio en cours de route"
+        )
+        yield False
+        return
+    print(f"[monitor] {MONITOR_CONTAINER} suspendu pour la fenetre de chaine")
+    try:
+        yield True
+    finally:
+        restart = _docker(["start", MONITOR_CONTAINER])
+        state = (
+            "redemarre"
+            if restart.returncode == 0
+            else f"ECHEC restart (rc={restart.returncode}) -- relancer docker start {MONITOR_CONTAINER}"
+        )
+        print(f"[monitor] {MONITOR_CONTAINER} {state}")

MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/run_pipeline.py

@@ -95,23 +95,30 @@ def main():
 
     total_start = time.time()
 
-    for phase in phases:
-        print(f"\n{'─' * 60}")
-        print(f"  {phase.upper()} — {PHASE_NAMES[phase]}")
-        print(f"{'─' * 60}")
-
-        start = time.time()
-        try:
-            output = run_phase(phase, force=args.force)
-            elapsed = time.time() - start
-            print(f"\n  {phase.upper()} completed in {elapsed:.1f}s")
-            print(f"  Output: {output}")
-        except Exception as e:
-            elapsed = time.time() - start
-            print(f"\n  {phase.upper()} FAILED after {elapsed:.1f}s: {e}")
-            if "--force" not in sys.argv:
-                print("  Use --force to retry, or fix the issue and re-run.")
-            sys.exit(1)
+    # Epic #1028 residu 2 : les chaines LLM->TTS durent plus que l'IDLE_TIMEOUT
+    # du monitor (1200 s) -- on suspend tts-fishaudio-idle-monitor le temps de
+    # la fenetre, redemarrage garanti meme si une phase echoue (sys.exit passe
+    # par le finally du context manager).
+    from .idle_monitor import suspended_idle_monitor
+
+    with suspended_idle_monitor(phases):
+        for phase in phases:
+            print(f"\n{'─' * 60}")
+            print(f"  {phase.upper()} — {PHASE_NAMES[phase]}")
+            print(f"{'─' * 60}")
+
+            start = time.time()
+            try:
+                output = run_phase(phase, force=args.force)
+                elapsed = time.time() - start
+                print(f"\n  {phase.upper()} completed in {elapsed:.1f}s")
+                print(f"  Output: {output}")
+            except Exception as e:
+                elapsed = time.time() - start
+                print(f"\n  {phase.upper()} FAILED after {elapsed:.1f}s: {e}")
+                if "--force" not in sys.argv:
+                    print("  Use --force to retry, or fix the issue and re-run.")
+                sys.exit(1)
 
     total_elapsed = time.time() - total_start
     print(f"\n{'═' * 60}")

MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_idle_monitor.py

@@ -0,0 +1,139 @@
+"""
+test_idle_monitor.py — coverage de v4/idle_monitor.py (suspension du sidecar
+tts-fishaudio-idle-monitor pendant une fenetre de chaine LLM->TTS).
+
+Epic #1028 residu 2. Avant ce module, la seule parade etait un geste manuel
+(``docker stop tts-fishaudio-idle-monitor``) a ne pas oublier -- sur le livre
+complet les phases p3+p4 (~36 min sans appel TTS) laissaient le monitor tuer
+``tts-fishaudio`` avant p5 (385/385 echecs, #14059).
+
+Hermetique CPU-only : ``_docker`` est monkeypatche, aucun appel docker reel.
+Pattern d'import de test_canonical.py (sys.path parent + ``v4.<module>``).
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+_v4_parent = Path(__file__).resolve().parent.parent.parent  # 04-Applications/
+if str(_v4_parent) not in sys.path:
+    sys.path.insert(0, str(_v4_parent))
+
+import v4.idle_monitor as im  # noqa: E402
+
+
+# ---------------------------------------------------------------------------
+# chain_needs_suspension — la matrice des formes de chaine
+# ---------------------------------------------------------------------------
+def test_chain_needs_suspension_matrix():
+    """p5 seul : non. LLM avant p5 : oui. p5 avant les LLM : non. Sans p5 : non."""
+    assert im.chain_needs_suspension(["p5"]) is False
+    assert im.chain_needs_suspension(["p3", "p4", "p5"]) is True
+    assert im.chain_needs_suspension(["p5", "p3"]) is False
+    assert im.chain_needs_suspension(["p3", "p4"]) is False
+    assert im.chain_needs_suspension(["p6", "p7"]) is False
+    # La chaine complete du livre (cas nominal du run live #14059).
+    assert im.chain_needs_suspension(
+        ["p0", "p1", "p1_5", "p2", "p3", "p4", "p5", "p6", "p7"]
+    ) is True
+
+
+# ---------------------------------------------------------------------------
+# suspended_idle_monitor — les 4 situations d'execution
+# ---------------------------------------------------------------------------
+class _FakeDocker:
+    """Rejoue une session docker : inspect -> stop -> start."""
+
+    def __init__(self, inspect_rc=0, inspect_out="true\n", stop_rc=0, start_rc=0):
+        self.inspect_rc, self.inspect_out = inspect_rc, inspect_out
+        self.stop_rc, self.start_rc = stop_rc, start_rc
+        self.calls: list[list[str]] = []
+
+    def __call__(self, args):
+        self.calls.append(list(args))
+        result = subprocess.CompletedProcess(
+            args=["docker", *args], returncode=0, stdout="", stderr=""
+        )
+        if args[:2] == ["inspect", "-f"]:
+            result = subprocess.CompletedProcess(
+                args=["docker", *args],
+                returncode=self.inspect_rc,
+                stdout=self.inspect_out,
+                stderr="",
+            )
+        elif args[0] == "stop":
+            result = subprocess.CompletedProcess(
+                args=["docker", *args], returncode=self.stop_rc, stdout="", stderr=""
+            )
+        elif args[0] == "start":
+            result = subprocess.CompletedProcess(
+                args=["docker", *args], returncode=self.start_rc, stdout="", stderr=""
+            )
+        return result
+
+    def commands(self):
+        return [" ".join(c[:2]) for c in self.calls]
+
+
+def test_suspends_and_restarts_when_monitor_running(monkeypatch, capsys):
+    """Monitor actif + chaine LLM->TTS : stop au debut, start a la fin."""
+    fake = _FakeDocker()
+    monkeypatch.setattr(im, "_docker", fake)
+    with im.suspended_idle_monitor(["p3", "p4", "p5"]) as suspended:
+        assert suspended is True
+        # Pendant la fenetre : le monitor est deja stoppe, rien d'autre.
+        assert fake.commands() == ["inspect -f", "stop tts-fishaudio-idle-monitor"]
+    assert "suspendu pour la fenetre" in capsys.readouterr().out
+    assert fake.commands()[-1] == "start tts-fishaudio-idle-monitor"
+
+
+def test_restart_guaranteed_on_chain_failure(monkeypatch):
+    """Une exception dans la fenetre redemarre quand meme le monitor."""
+    fake = _FakeDocker()
+    monkeypatch.setattr(im, "_docker", fake)
+    try:
+        with im.suspended_idle_monitor(["p3", "p5"]):
+            raise RuntimeError("phase p5 FAILED")
+    except RuntimeError:
+        pass
+    assert fake.commands() == [
+        "inspect -f",
+        "stop tts-fishaudio-idle-monitor",
+        "start tts-fishaudio-idle-monitor",
+    ]
+
+
+def test_noop_when_monitor_absent(monkeypatch, capsys):
+    """Monitor arrete/inexistant : chaine sans suspension, aucun start sauvage."""
+    fake = _FakeDocker(inspect_out="false\n")
+    monkeypatch.setattr(im, "_docker", fake)
+    with im.suspended_idle_monitor(["p3", "p5"]) as suspended:
+        assert suspended is False
+    assert fake.commands() == ["inspect -f"]
+    assert "sans suspension" in capsys.readouterr().out
+
+
+def test_stop_failure_does_not_block_the_chain(monkeypatch, capsys):
+    """Stop en echec : la chaine continue (WARN), aucun start (rien n'a ete arrete)."""
+    fake = _FakeDocker(stop_rc=1)
+    monkeypatch.setattr(im, "_docker", fake)
+    with im.suspended_idle_monitor(["p3", "p5"]) as suspended:
+        assert suspended is False
+    assert fake.commands() == ["inspect -f", "stop tts-fishaudio-idle-monitor"]
+    assert "WARN" in capsys.readouterr().out
+
+
+def test_docker_absent_tolerated(monkeypatch):
+    """Docker absent de la machine (OSError) : la chaine continue sans suspension."""
+
+    def _no_docker(args):
+        raise OSError("docker: command not found")
+
+    monkeypatch.setattr(im, "_docker", _no_docker)
+    # monitor_is_running doit avaler l'OSError...
+    assert im.monitor_is_running() is False
+    # ...et la fenetre reste franchissable.
+    with im.suspended_idle_monitor(["p3", "p5"]) as suspended:
+        assert suspended is False