From 0c5172daa6e15ede146218ee6f68b33a0e2ef355 Mon Sep 17 00:00:00 2001 From: jsboige Date: Thu, 3 Sep 2026 04:56:02 +0200 Subject: [PATCH] feat(audiobook,#1028): suspendre automatiquement tts-fishaudio-idle-monitor pendant une chaine Residu 2 de l'EPIC (le gesture manuel a ne pas oublier) : sur le livre complet, p3+p4 (~36 min sans appel TTS) laissaient le monitor tuer tts-fishaudio avant p5 (IDLE_TIMEOUT=1200 s ; 385/385 echecs au premier run #14059). run_pipeline suspend maintenant le monitor le temps de la fenetre LLM->TTS et le redemarre en finally (garanti meme si une phase echoue -- sys.exit traverse le context manager). Fail-safe : docker absent / monitor arrete / stop en echec => la chaine continue, au pire avec un WARN (l'etat d'avant etait l'echec silencieux). 6 tests hermetiques (120/120 au total v4) + preuve end-to-end sur le conteneur reel po-2023 : suspendu dans la fenetre (running=false), redemarre apres. Co-Authored-By: Claude-Code --- .../Audio/04-Applications/v4/idle_monitor.py | 92 ++++++++++++ .../Audio/04-Applications/v4/run_pipeline.py | 41 +++--- .../v4/tests/test_idle_monitor.py | 139 ++++++++++++++++++ 3 files changed, 255 insertions(+), 17 deletions(-) create mode 100644 MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/idle_monitor.py create mode 100644 MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_idle_monitor.py diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/idle_monitor.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/idle_monitor.py new file mode 100644 index 0000000000..7f6477ada0 --- /dev/null +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/idle_monitor.py @@ -0,0 +1,92 @@ +"""Suspension du sidecar ``tts-fishaudio-idle-monitor`` pendant une fenetre de chaine. + +Epic #1028, residu 2 : sur le livre complet, les phases LLM amont (p3 + p4) +durent ~36 min sans aucun appel TTS -- l'idle monitor (IDLE_TIMEOUT=1200 s) +tue alors le service ``tts-fishaudio`` AVANT que p5 n'arrive, et la chaine +echoue sur port refuse (385/385 echecs au premier run du livre entier, #14059). +Le monitor est une economie GPU legitime : on le SUSPEND pendant la fenetre +de chaine LLM->TTS puis on le redemarre, au lieu d'un geste manuel a ne pas +oublier. + +Fail-safe par construction : docker absent, daemon muet, inspect/stop en +echec -- la chaine continue TOUJOURS, au pire avec un avertissement (le +comportement d'avant ce module etait l'echec silencieux en chaine longue). +""" + +from __future__ import annotations + +import subprocess +from contextlib import contextmanager +from collections.abc import Iterator + +MONITOR_CONTAINER = "tts-fishaudio-idle-monitor" + +# Phases dont la duree est dominee par le LLM, sans appel TTS. +_LLM_PHASES_BEFORE_TTS = {"p0", "p1", "p1_5", "p2", "p3", "p4"} + +_DOCKER_TIMEOUT_S = 30 + + +def _docker(args: list[str]) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=_DOCKER_TIMEOUT_S + ) + + +def chain_needs_suspension(phases: list[str]) -> bool: + """True si la chaine contient p5 precede d'au moins une phase LLM. + + ``["p5"]`` seul n'a pas besoin de suspension (p5 appelle le TTS + immediatement, le monitor ne le voit jamais inactif) ; ce sont les + chaines LLM->TTS qui tuent le service pendant les phases amont. + """ + seen: set[str] = set() + for phase in phases: + if phase == "p5": + return bool(seen & _LLM_PHASES_BEFORE_TTS) + seen.add(phase) + return False + + +def monitor_is_running() -> bool: + """True si le conteneur monitor existe ET tourne. Docker absent => False.""" + try: + result = _docker(["inspect", "-f", "{{.State.Running}}", MONITOR_CONTAINER]) + except (OSError, subprocess.TimeoutExpired): + return False + return result.returncode == 0 and result.stdout.strip().lower() == "true" + + +@contextmanager +def suspended_idle_monitor(phases: list[str]) -> Iterator[bool]: + """Suspend le monitor pour la duree de la fenetre ; yield True si suspendu. + + Le monitor n'est redemarre que si CETTE fenetre l'a arrete (jamais au + hasard d'un etat anterieur). + """ + if not chain_needs_suspension(phases): + yield False + return + if not monitor_is_running(): + print(f"[monitor] {MONITOR_CONTAINER} absent ou arrete -- chaine sans suspension") + yield False + return + stop = _docker(["stop", MONITOR_CONTAINER]) + if stop.returncode != 0: + print( + f"[monitor] WARN: stop echoue (rc={stop.returncode}) -- la chaine " + "continue, le monitor peut tuer tts-fishaudio en cours de route" + ) + yield False + return + print(f"[monitor] {MONITOR_CONTAINER} suspendu pour la fenetre de chaine") + try: + yield True + finally: + restart = _docker(["start", MONITOR_CONTAINER]) + state = ( + "redemarre" + if restart.returncode == 0 + else f"ECHEC restart (rc={restart.returncode}) -- relancer docker start {MONITOR_CONTAINER}" + ) + print(f"[monitor] {MONITOR_CONTAINER} {state}") diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/run_pipeline.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/run_pipeline.py index 73d349f129..0025b5d077 100644 --- a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/run_pipeline.py +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/run_pipeline.py @@ -95,23 +95,30 @@ def main(): total_start = time.time() - for phase in phases: - print(f"\n{'─' * 60}") - print(f" {phase.upper()} — {PHASE_NAMES[phase]}") - print(f"{'─' * 60}") - - start = time.time() - try: - output = run_phase(phase, force=args.force) - elapsed = time.time() - start - print(f"\n {phase.upper()} completed in {elapsed:.1f}s") - print(f" Output: {output}") - except Exception as e: - elapsed = time.time() - start - print(f"\n {phase.upper()} FAILED after {elapsed:.1f}s: {e}") - if "--force" not in sys.argv: - print(" Use --force to retry, or fix the issue and re-run.") - sys.exit(1) + # Epic #1028 residu 2 : les chaines LLM->TTS durent plus que l'IDLE_TIMEOUT + # du monitor (1200 s) -- on suspend tts-fishaudio-idle-monitor le temps de + # la fenetre, redemarrage garanti meme si une phase echoue (sys.exit passe + # par le finally du context manager). + from .idle_monitor import suspended_idle_monitor + + with suspended_idle_monitor(phases): + for phase in phases: + print(f"\n{'─' * 60}") + print(f" {phase.upper()} — {PHASE_NAMES[phase]}") + print(f"{'─' * 60}") + + start = time.time() + try: + output = run_phase(phase, force=args.force) + elapsed = time.time() - start + print(f"\n {phase.upper()} completed in {elapsed:.1f}s") + print(f" Output: {output}") + except Exception as e: + elapsed = time.time() - start + print(f"\n {phase.upper()} FAILED after {elapsed:.1f}s: {e}") + if "--force" not in sys.argv: + print(" Use --force to retry, or fix the issue and re-run.") + sys.exit(1) total_elapsed = time.time() - total_start print(f"\n{'═' * 60}") diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_idle_monitor.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_idle_monitor.py new file mode 100644 index 0000000000..cca537dd80 --- /dev/null +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_idle_monitor.py @@ -0,0 +1,139 @@ +""" +test_idle_monitor.py — coverage de v4/idle_monitor.py (suspension du sidecar +tts-fishaudio-idle-monitor pendant une fenetre de chaine LLM->TTS). + +Epic #1028 residu 2. Avant ce module, la seule parade etait un geste manuel +(``docker stop tts-fishaudio-idle-monitor``) a ne pas oublier -- sur le livre +complet les phases p3+p4 (~36 min sans appel TTS) laissaient le monitor tuer +``tts-fishaudio`` avant p5 (385/385 echecs, #14059). + +Hermetique CPU-only : ``_docker`` est monkeypatche, aucun appel docker reel. +Pattern d'import de test_canonical.py (sys.path parent + ``v4.``). +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +_v4_parent = Path(__file__).resolve().parent.parent.parent # 04-Applications/ +if str(_v4_parent) not in sys.path: + sys.path.insert(0, str(_v4_parent)) + +import v4.idle_monitor as im # noqa: E402 + + +# --------------------------------------------------------------------------- +# chain_needs_suspension — la matrice des formes de chaine +# --------------------------------------------------------------------------- +def test_chain_needs_suspension_matrix(): + """p5 seul : non. LLM avant p5 : oui. p5 avant les LLM : non. Sans p5 : non.""" + assert im.chain_needs_suspension(["p5"]) is False + assert im.chain_needs_suspension(["p3", "p4", "p5"]) is True + assert im.chain_needs_suspension(["p5", "p3"]) is False + assert im.chain_needs_suspension(["p3", "p4"]) is False + assert im.chain_needs_suspension(["p6", "p7"]) is False + # La chaine complete du livre (cas nominal du run live #14059). + assert im.chain_needs_suspension( + ["p0", "p1", "p1_5", "p2", "p3", "p4", "p5", "p6", "p7"] + ) is True + + +# --------------------------------------------------------------------------- +# suspended_idle_monitor — les 4 situations d'execution +# --------------------------------------------------------------------------- +class _FakeDocker: + """Rejoue une session docker : inspect -> stop -> start.""" + + def __init__(self, inspect_rc=0, inspect_out="true\n", stop_rc=0, start_rc=0): + self.inspect_rc, self.inspect_out = inspect_rc, inspect_out + self.stop_rc, self.start_rc = stop_rc, start_rc + self.calls: list[list[str]] = [] + + def __call__(self, args): + self.calls.append(list(args)) + result = subprocess.CompletedProcess( + args=["docker", *args], returncode=0, stdout="", stderr="" + ) + if args[:2] == ["inspect", "-f"]: + result = subprocess.CompletedProcess( + args=["docker", *args], + returncode=self.inspect_rc, + stdout=self.inspect_out, + stderr="", + ) + elif args[0] == "stop": + result = subprocess.CompletedProcess( + args=["docker", *args], returncode=self.stop_rc, stdout="", stderr="" + ) + elif args[0] == "start": + result = subprocess.CompletedProcess( + args=["docker", *args], returncode=self.start_rc, stdout="", stderr="" + ) + return result + + def commands(self): + return [" ".join(c[:2]) for c in self.calls] + + +def test_suspends_and_restarts_when_monitor_running(monkeypatch, capsys): + """Monitor actif + chaine LLM->TTS : stop au debut, start a la fin.""" + fake = _FakeDocker() + monkeypatch.setattr(im, "_docker", fake) + with im.suspended_idle_monitor(["p3", "p4", "p5"]) as suspended: + assert suspended is True + # Pendant la fenetre : le monitor est deja stoppe, rien d'autre. + assert fake.commands() == ["inspect -f", "stop tts-fishaudio-idle-monitor"] + assert "suspendu pour la fenetre" in capsys.readouterr().out + assert fake.commands()[-1] == "start tts-fishaudio-idle-monitor" + + +def test_restart_guaranteed_on_chain_failure(monkeypatch): + """Une exception dans la fenetre redemarre quand meme le monitor.""" + fake = _FakeDocker() + monkeypatch.setattr(im, "_docker", fake) + try: + with im.suspended_idle_monitor(["p3", "p5"]): + raise RuntimeError("phase p5 FAILED") + except RuntimeError: + pass + assert fake.commands() == [ + "inspect -f", + "stop tts-fishaudio-idle-monitor", + "start tts-fishaudio-idle-monitor", + ] + + +def test_noop_when_monitor_absent(monkeypatch, capsys): + """Monitor arrete/inexistant : chaine sans suspension, aucun start sauvage.""" + fake = _FakeDocker(inspect_out="false\n") + monkeypatch.setattr(im, "_docker", fake) + with im.suspended_idle_monitor(["p3", "p5"]) as suspended: + assert suspended is False + assert fake.commands() == ["inspect -f"] + assert "sans suspension" in capsys.readouterr().out + + +def test_stop_failure_does_not_block_the_chain(monkeypatch, capsys): + """Stop en echec : la chaine continue (WARN), aucun start (rien n'a ete arrete).""" + fake = _FakeDocker(stop_rc=1) + monkeypatch.setattr(im, "_docker", fake) + with im.suspended_idle_monitor(["p3", "p5"]) as suspended: + assert suspended is False + assert fake.commands() == ["inspect -f", "stop tts-fishaudio-idle-monitor"] + assert "WARN" in capsys.readouterr().out + + +def test_docker_absent_tolerated(monkeypatch): + """Docker absent de la machine (OSError) : la chaine continue sans suspension.""" + + def _no_docker(args): + raise OSError("docker: command not found") + + monkeypatch.setattr(im, "_docker", _no_docker) + # monitor_is_running doit avaler l'OSError... + assert im.monitor_is_running() is False + # ...et la fenetre reste franchissable. + with im.suspended_idle_monitor(["p3", "p5"]) as suspended: + assert suspended is False