diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p5_tts.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p5_tts.py index 5d6c55d4d3..0e25821e9a 100644 --- a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p5_tts.py +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p5_tts.py @@ -63,6 +63,36 @@ _NARRATOR_QWEN_ROUTING: bool = os.getenv("NARRATOR_QWEN_ROUTING", "1") == "1" _QWEN_NARRATOR_REFERENCE_ID: str = "qwen-voicedesign-narrator-fr-literary" +# Phase 1 of the narrator pivot (#19692): a second candidate engine, +# CosyVoice3 (Fun-CosyVoice3-0.5B-2512, Apache-2.0), synthesizes the narrator +# chunked-by-sentence behind its own flag. The DEFAULT IS UNCHANGED — flag +# "0": Qwen VoiceDesign stays the narrator engine until the association's +# listening picks the final engine (#17586). Engines are exclusive: both +# flags ON raises a configuration error, never a silent precedence (same +# no-silent-swap contract as #15002 acceptance 6). +_NARRATOR_COSYVOICE3_ROUTING: bool = os.getenv("NARRATOR_COSYVOICE3_ROUTING", "0") == "1" +# The "-v2" suffix is not decoration: this sentinel is bumped whenever a +# change can alter the AUDIO of an already-rendered segment, so the batch +# cache cannot keep serving a stale MP3 for it (#19692). v2 = the +# degenerate-render guard. The pre-guard run had written 9 narrator MP3s of +# ~0.10 s -- one speech token, up to 560 chars/s -- while reporting them as +# generated; a cached segment never enters _synthesize_narrator_cosyvoice3, +# so no guard placed there can ever see them. v3 = the tag-strip +# fix (be7aa641d5): _strip_brackets_for_qwen now replaces a tag with a +# space instead of eating it with the following whitespace. The composed +# text (tags included) is unchanged, so its hash -- the other cache key -- +# cannot see the fix; only this sentinel re-renders the glued narrators +# (measured: 16/20 sampled segments carried omissions traced to glue +# points like "route,[short pause] quand" -> "route,quand"). +_CV3_NARRATOR_REFERENCE_ID: str = "cosyvoice3-zeroshot-narrator-fr-v3" + +# Engine -> reference_id sentinel. The narrator cache is keyed on it +# (#19692): an MP3 rendered by one engine is never served for another. +_NARRATOR_ENGINE_REFERENCE_IDS: dict[str, str] = { + "qwen_voicedesign": _QWEN_NARRATOR_REFERENCE_ID, + "cosyvoice3": _CV3_NARRATOR_REFERENCE_ID, +} + # VoiceDesign `instructions` prompt — describes the desired voice and delivery # for a 19th-century French audiobook narrator. Tuned from prosody_lab A/B # measurements (Issue #11624 / #1028, melody partition verdict EXPRESSIVE). @@ -451,7 +481,15 @@ def _compose_tts_text(seg: AnnotatedSegment) -> str: # CRITICAL: ensure space after every ] — S2-Pro ignores tags without trailing space result = re.sub(r"\](?=\S)", "] ", result) - if len(result) > _MAX_TTS_CHARS: + # The 500-char cap is an S2-Pro input limit, not a pipeline one. A + # narrator rerouted to CosyVoice3 is bounded per-chunk by + # _chunk_narration instead: truncating here would silently drop the + # tail of every long narration (#19692 phase 1, measured: seg 1, + # 981 chars of fishaudio_text -> ~440 rendered, 17.8 s of audio for + # a 950-char segment). The default (FishAudio and Qwen) is unchanged. + if len(result) > _MAX_TTS_CHARS and not _should_route_narrator_to( + "cosyvoice3", seg + ): result = result[:_MAX_TTS_CHARS].rsplit(" ", 1)[0] + "..." return result @@ -849,10 +887,11 @@ def _synthesize_segment(seg: AnnotatedSegment, fishaudio_text: str) -> TTSResult mp3_path = TTS_DIR / f"seg_{seg.seg_index:04d}_{seg.speaker}.mp3" current_hash = _text_hash(fishaudio_text) - # F6 (Issue #15002): narrator-only routing to Qwen3-TTS VoiceDesign. - # Runs BEFORE the cache check so a stale FishAudio MP3 cannot shadow a - # narrator reroute. - if _should_route_narrator_to_qwen(seg): + # F6 (#15002) / phase 1 (#19692): narrator-only routing to the selected + # alternate engine (Qwen VoiceDesign by default, CosyVoice3 behind its + # flag). Runs BEFORE the cache check so a stale FishAudio MP3 cannot + # shadow a narrator reroute. + if _should_route_narrator_to("qwen_voicedesign", seg): return _synthesize_narrator_qwen( seg=seg, fishaudio_text=fishaudio_text, @@ -860,6 +899,14 @@ def _synthesize_segment(seg: AnnotatedSegment, fishaudio_text: str) -> TTSResult seed=seed, text_hash=current_hash, ) + if _should_route_narrator_to("cosyvoice3", seg): + return _synthesize_narrator_cosyvoice3( + seg=seg, + fishaudio_text=fishaudio_text, + mp3_path=mp3_path, + seed=seed, + text_hash=current_hash, + ) return _synthesize_fishaudio_path( seg=seg, @@ -891,8 +938,18 @@ def _strip_brackets_for_qwen(text: str) -> str: natural speech. Brackets would either be ignored or, worse, vocalized (S2-Pro regression #1277/#1485 — WER explosion on bracket text). """ - # Drop content in [brackets] and any leading tag prefix block. - return re.sub(r"\[[^\]]+\]\s*", "", text).strip() + # Replace bracket tags with a SPACE, never with nothing: the old + # pattern ate the tag and its trailing whitespace, gluing the + # neighbouring words ("route,[short pause] quand" -> "route,quand", + # "neige[pause] et" -> "neigeet"). The glued punctuation corrupts the + # plain-text engines' rendering -- measured #19692: CosyVoice3 fed a + # glued run-on omits everything before the glue point (seg 57: the + # audio starts at "quand", the first clause never spoken; same + # signature on segs 19/38/76) -- and it hides sentence boundaries + # from the CV3 chunker, un-chunking text the chunking was built to + # bound. + text = re.sub(r"\[[^\]]+\]", " ", text) + return re.sub(r"\s+", " ", text).strip() def _synthesize_narrator_qwen( @@ -962,15 +1019,466 @@ def _synthesize_narrator_qwen( ) -def _should_route_narrator_to_qwen(seg: AnnotatedSegment) -> bool: - """Decide whether a segment is routed to Qwen VoiceDesign. +def _narrator_engine_flags() -> dict[str, bool]: + """Narrator engine activation flags, one entry per candidate engine.""" + return { + "qwen_voicedesign": _NARRATOR_QWEN_ROUTING, + "cosyvoice3": _NARRATOR_COSYVOICE3_ROUTING, + } + + +def _selected_narrator_engine() -> str | None: + """The single active narrator engine, or None (legacy FishAudio path). + + Two active flags is a configuration error: raising beats an undocumented + precedence that would silently change the rendered voice (#15002 + acceptance 6 — no silent engine swap). + """ + active = [engine for engine, on in _narrator_engine_flags().items() if on] + if len(active) > 1: + raise ValueError( + "narrator engine flags are exclusive: " + + ", ".join(f"{e}={on}" for e, on in _narrator_engine_flags().items()) + ) + return active[0] if active else None + + +def _should_route_narrator_to(engine: str, seg: AnnotatedSegment) -> bool: + """Decide whether a narrator segment routes to `engine`. + + Engine-independent successor of _should_route_narrator_to_qwen (#19692 + acceptance 4): routing stays explicit, testable, and limited to the + narrator (acceptance #1 of #15002). All other speakers stay on FishAudio + clone whatever the flags. + """ + return seg.speaker == "narrateur" and _selected_narrator_engine() == engine + + +# --------------------------------------------------------------------------- +# Phase 1 (#19692) — Narrator routing to CosyVoice3, chunked by sentence +# --------------------------------------------------------------------------- +# Measured on the A0C bench (#17586): the unchunked CosyVoice3 render omits +# whole sentences (absent under 3/3 ASRs), and the chunked re-render recovers +# the targeted segments but moves losses to chunk onsets — which is exactly +# why p7_verify counts missing >=3-word segments instead of trusting the mean +# WER (c.6035598289). + +class NarratorCosyVoice3Unavailable(RuntimeError): + """Raised when CosyVoice3 cannot synthesize a narrator segment (model + missing, inference failure, conversion failure). Same contract as + NarratorQwenUnavailable: hard failure, never a silent fall back to + FishAudio (#15002 acceptance 6).""" + + +# Chunker spec = the one measured on the A0C bench (#17586, suite 1 Zonos / +# suite 2 CosyVoice3): <=280 chars, sentence boundaries, clause fallback. +_CV3_MAX_CHUNK_CHARS = 280 +_CV3_PAUSE_S = 0.25 # silence between chunks + +# Degenerate-render guard (#19692). CosyVoice3 can return from +# `inference_zero_shot` having emitted a single speech token: the caller gets +# a well-formed WAV of exactly 1/25 s -- CosyVoice's own `token_hop_len` -- +# with no error at all. Slightly longer inputs crash instead, in the f0 +# predictor, because the mel is shorter than its first conv1d kernel. The two +# faces are ONE event, so a duration floor catches both and a re-roll clears +# both. +# +# Measured on the run-5 corpus (2026-10-07): the slowest 5% of the 257 +# legitimate narrator segments run at 15.4 chars/s (median 20.1), while the 9 +# silently degenerate ones ran at 108-560 chars/s -- one of them 57 chars in +# 0.10 s. A 40 chars/s floor sits 2.6x under the slowest legitimate segment +# and 2.7x over the fastest degenerate one; the absolute term keeps a +# two-character chunk from being judged by a ratio alone. +# +# The degeneracy is a function of (text, SEED), not of the text: across 3 +# failing texts x 5 alternate seeds, 17 of 18 renders came back at a +# plausible duration for the same text. Hence a re-roll rather than a text +# transformation -- and note the corollary, that no chunking rule could have +# fixed it, since the failing texts spanned 1 to 7 tokens while a 7-token +# input failed and a 4-token one succeeded. +# +# NOT transferable to the cache path: these floors are WAV durations, and an +# MP3 carries ~0.1 s of tag and final-frame padding, so the same check applied +# to a cached MP3 would re-render a legitimately short "--Non." on every run. +# +# Where 40 comes from, measured over the 270-narrator artifact and NOT derived +# from a ratio anyone liked (#19692): +# * legitimate narrator speech: p05 15.3, median 20.1, highest 38.9 chars/s +# * segments between 43.1 and 69 chars/s (eight of them transcribed by ASR) +# are ALL defective: seg 158 reads 3 words where the text has 12, seg 227 +# 79 where it has 156 -- partial degeneration, not fast speech +# * gross degenerates (one token): 99 to 560 chars/s +# The corpus has NOTHING between 38.9 and 43.1, so the floor sits in an +# empirically empty band rather than in the middle of a population. Its +# margins are therefore thin on both sides (1.03x above the fastest legitimate, +# 1.08x under the slowest defective) and that is stated rather than dressed up: +# this floor is a GROSS backstop, not the fidelity organ. Fidelity is p7's +# omission control, which votes three ASRs and is what the gate reads. +_CV3_MIN_CHARS_PER_SEC = 40.0 +_CV3_MIN_ABS_S = 0.20 +# Five, not one: an alternate seed is not a guaranteed good seed. Measured +# over 8 texts x 6 seeds, seed 7 degenerated on a text that seed 42 rendered +# fine, and seed 43 on another -- so every attempt has to be checked against +# the floor rather than the first re-roll trusted. At ~5% per-attempt +# degeneracy off the unlucky seed, five attempts leave ~3e-7 per chunk, and a +# healthy chunk still leaves the loop on its first pass. +_CV3_RENDER_ATTEMPTS = 5 +# attempt 0 reproduces the pre-guard seed exactly, so a healthy chunk renders +# bit-identically and only a degenerate one is perturbed. +_CV3_RETRY_SEED_STRIDE = 104_729 # prime, unrelated to seed + seg_index + +# CosyVoice3 AutoModel: ~3.4 GB VRAM, tens of seconds to load — one instance +# per process, not per segment. +_CV3_MODEL_CACHE: dict[str, object] = {} + +# The re-roll above treats the symptom; this repairs the cause. Upstream's +# non-vLLM decoder protects its first `min_len` steps against an early stop +# token, but masks only ONE of the three ids it breaks on (#19692): +# +# self.stop_token_ids = [speech_token_size + i for i in range(3)] +# +# def sampling_ids(self, weighted_scores, decoded_tokens, sampling, ignore_eos=True): +# if ignore_eos is True: +# weighted_scores[self.speech_token_size] = -float('inf') +# ... +# +# top_ids = self.sampling_ids(..., ignore_eos=True if i < min_len else False) +# if top_ids in self.stop_token_ids: # ANY of the three breaks the loop +# break +# +# `speech_token_size` IS `stop_token_ids[0]`, so ids 1 and 2 stay samplable +# inside the protected prefix: the floor leaks, and a render ends after a +# single speech token -- 1/25 s = 0.04 s, CosyVoice's token_hop_len. This is +# not a short-text effect and not a seed effect. `min_len` is never even 0 +# here: the decoder concatenates prompt_text and adds its length back +# (`text_len += prompt_text_len`), so `min_len == 2 x tokens(tts_text)`, i.e. +# 10 for "Il demanda:" -- a text whose own measured render stopped at exactly +# one token. The floor was armed; two ids walked through it. +# +# The vLLM branch has no such hole (`SamplingParams(min_tokens=min_len)` +# applies to every id in `stop_token_ids`), which is the shape this patch +# restores to the branch we actually run. Patching the loaded sampler rather +# than vendoring a fork keeps the reuse organ-first (#17586): the upstream +# tree stays exactly as installed, and a CosyVoice upgrade that moves these +# attributes fails loudly at load instead of silently rendering unguarded. +_CV3_STOP_FLOOR_FLAG = "_v4_stop_floor_patched" +# The decoder sits two levels down from the object `load_model()` returns: +# `AutoModel()` is a FACTORY that returns `CosyVoice3(...)`, whose `__init__` +# sets `self.model = CosyVoice3Model(...)`, whose own `__init__` sets +# `self.llm = CosyVoice3LM(...)` — and it is the LM that carries +# `sampling_ids` / `stop_token_ids`. Measured, not inferred: a first version of +# this patch addressed `model.llm` and raised on all nine probe segments, for a +# full GPU run of learning nothing (#19692). +_CV3_DECODER_PATH = "model.llm" + + +def _cv3_decoder(model): + """Return the CosyVoice3 LM, or raise naming the shape actually observed. + + The diagnosis carries the observed types and attribute names on purpose: + a vague "no decoder" costs another GPU run to localise, which is exactly + what the first attempt at this patch spent. + """ + inner = getattr(model, "model", None) + llm = getattr(inner, "llm", None) if inner is not None else None + if llm is None: + outer_attrs = sorted(vars(model)) if hasattr(model, "__dict__") else [] + inner_attrs = ( + sorted(vars(inner)) + if inner is not None and hasattr(inner, "__dict__") + else [] + ) + raise NarratorCosyVoice3Unavailable( + f"CosyVoice3 decoder not found at `{_CV3_DECODER_PATH}`: the loaded " + f"object is {type(model).__name__}{outer_attrs}, its `.model` is " + f"{type(inner).__name__ if inner is not None else None}{inner_attrs} " + "-- the stop-token floor patch cannot be applied; refusing to " + "render unguarded (#19692)" + ) + return llm + + +def _patch_cv3_stop_token_floor(model) -> str: + """Mask every stop id CosyVoice3 breaks on, for its whole min_len prefix. + + Returns "patched" or "already-patched". Raises + NarratorCosyVoice3Unavailable when the decoder lacks the attributes the + patch relies on — a silent skip would mean rendering with the leak still + open, which is the failure this patch exists to close. + """ + llm = _cv3_decoder(model) + missing = [ + name for name in ("sampling_ids", "stop_token_ids") if not hasattr(llm, name) + ] + if missing: + raise NarratorCosyVoice3Unavailable( + "CosyVoice3 decoder without " + " / ".join(missing) + ": the " + "stop-token floor patch cannot be applied; refusing to render " + "unguarded (#19692)" + ) + if getattr(llm, _CV3_STOP_FLOOR_FLAG, False): + return "already-patched" + + stop_ids = [int(i) for i in llm.stop_token_ids] + original = llm.sampling_ids + + def sampling_ids(weighted_scores, decoded_tokens, sampling, ignore_eos=True): + if ignore_eos: + for stop_id in stop_ids: + weighted_scores[stop_id] = -float("inf") + return original(weighted_scores, decoded_tokens, sampling, ignore_eos=False) + + llm.sampling_ids = sampling_ids + setattr(llm, _CV3_STOP_FLOOR_FLAG, True) + return "patched" + + +def _get_cv3_client(): + """Return the bakeoff CosyVoice3 client module, loaded once. + + The module is loaded DIRECTLY from its file: bakeoff_large/__init__.py + is markdown prose (documentation), not valid Python — a package import + of prosody_lab.bakeoff_large crashes on it with a SyntaxError. Same + organ-first reuse as the A0C bench (#17586), which dodged the same file + by importing clients.cosyvoice3 off sys.path. + """ + if "client" not in _CV3_MODEL_CACHE: + import importlib.util + + cv3_path = ( + Path(__file__).parent + / "prosody_lab" + / "bakeoff_large" + / "clients" + / "cosyvoice3.py" + ) + spec = importlib.util.spec_from_file_location("cv3_bakeoff_client", cv3_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + _CV3_MODEL_CACHE["client"] = module + return _CV3_MODEL_CACHE["client"] + + +def _load_cv3_model(): + """Load the CosyVoice3 AutoModel once per process (organ-first reuse of + the prosody_lab bakeoff client, measured on #17586). + + The decoder is patched on the way out — `_patch_cv3_stop_token_floor` + closes the leaky `min_len` floor at its source, so a segment renders at + the length its text implies instead of stopping after one speech token + (#19692). The status line is logged once, which is what makes the patch + verifiable in the measured run's log rather than assumed. + """ + if "stop_floor_error" in _CV3_MODEL_CACHE: + # The refusal is cached too. Without this the model is reloaded (~16 s) + # for every segment only to fail identically -- measured: a failing + # patch cost six reloads in a nine-segment probe, so a 269-segment run + # would have burned most of an hour before reporting anything. + raise NarratorCosyVoice3Unavailable(_CV3_MODEL_CACHE["stop_floor_error"]) + if "model" not in _CV3_MODEL_CACHE: + model = _get_cv3_client().load_model() + try: + status = _patch_cv3_stop_token_floor(model) + except NarratorCosyVoice3Unavailable as exc: + _CV3_MODEL_CACHE["stop_floor_error"] = str(exc) + raise + _CV3_MODEL_CACHE["stop_floor"] = status + logger.info("[CV3] stop-token floor patch: %s", status) + _CV3_MODEL_CACHE["model"] = model + return _CV3_MODEL_CACHE["model"] + + +def _chunk_narration(text: str, max_chars: int = _CV3_MAX_CHUNK_CHARS) -> list[str]: + """Split narration into <=max_chars chunks at sentence boundaries. + + Spec = the chunker measured on the A0C bench (#17586): split after + . ! ?, clause fallback (, ; :) for over-long sentences, greedy whole-unit + grouping, never a cut mid-word. Identity is asserted — the chunks + reassemble to the source — so chunking can never itself drop text: the + omission class of #19692 lives in the model, not the chunker. + """ + sentence_re = re.compile(r"(?<=[.!?])\s+") + clause_re = re.compile(r"(?<=[,;:])\s+") + text = text.strip() + if not text: + return [] + + units: list[str] = [] + for sentence in (s.strip() for s in sentence_re.split(text) if s.strip()): + if len(sentence) <= max_chars: + units.append(sentence) + continue + clauses = [c.strip() for c in clause_re.split(sentence) if c.strip()] + if any(len(c) > max_chars for c in clauses): + raise ValueError( + f"clause > {max_chars} chars; mid-word split forbidden: {sentence[:60]}" + ) + units.extend(clauses) + + chunks: list[str] = [] + current = "" + for unit in units: + candidate = f"{current} {unit}".strip() if current else unit + if len(candidate) <= max_chars: + current = candidate + else: + if current: + chunks.append(current) + current = unit + if current: + chunks.append(current) + + def _norm_ws(s: str) -> str: + return re.sub(r"\s+", " ", s).strip() + + assert _norm_ws(" ".join(chunks)) == _norm_ws(text), "chunking lost text" + return chunks + + +def _cv3_min_expected_s(text: str) -> float: + """Shortest audio that could plausibly hold `text` (#19692). + + A render below this floor is a degenerate CosyVoice3 output, not fast + speech: see _CV3_MIN_CHARS_PER_SEC for the measurement (legitimate speech + tops out at 38.9 chars/s, the slowest ASR-confirmed defect at 43.1) and for + why this is a gross backstop rather than the fidelity organ. + """ + return max(_CV3_MIN_ABS_S, len(text) / _CV3_MIN_CHARS_PER_SEC) + - Acceptance #1 of #15002: routing is explicit, testable, and limited to - the narrator. All other speakers stay on FishAudio clone. +def _synthesize_narrator_cosyvoice3( + seg: AnnotatedSegment, + fishaudio_text: str, + mp3_path: Path, + seed: int, + text_hash: str, +) -> TTSResult: + """Synthesize a narrator segment via CosyVoice3, chunked by sentence. + + Organ-first: model loading and the zero-shot prompt constants come from + the prosody_lab bakeoff client (clients.cosyvoice3, #17586); the + per-chunk orchestration (seed, pause, concat) is pipeline logic and lives + here. Chunking bounds each inference to one sentence group so p7's + omission control can attribute a dropped passage to a chunk onset + instead of losing it in the mean WER. """ - return ( - _NARRATOR_QWEN_ROUTING - and seg.speaker == "narrateur" + plain_text = _strip_brackets_for_qwen(fishaudio_text) + if not plain_text: + raise NarratorCosyVoice3Unavailable( + f"seg {seg.seg_index}: empty text after stripping brackets" + ) + chunks = _chunk_narration(plain_text) + if not chunks: + raise NarratorCosyVoice3Unavailable( + f"seg {seg.seg_index}: chunker returned nothing" + ) + + # Imported here (not at module top) so hermetic tests need neither torch + # nor the CosyVoice runtime — the empty-input guard above runs torch-free. + import time + + cv3_client = _get_cv3_client() + + import torch + + + _, _, asset_wav = cv3_client._bootstrap_paths() + model = _load_cv3_model() + prompt_text = cv3_client.PROMPT_TEXT_ZH + cv3_client.ENDOFPROMPT + sr = int(model.sample_rate) + + t0 = time.time() + wavs = [] + attempts = 0 + for i, chunk in enumerate(chunks): + floor_s = _cv3_min_expected_s(chunk) + wav = None + last_dur = 0.0 + last_exc: Exception | None = None + for attempt in range(_CV3_RENDER_ATTEMPTS): + attempts += 1 + # attempt 0 keeps the pre-guard seed, so a healthy chunk is + # unchanged and only a degenerate one is perturbed. + torch.manual_seed(seed + i + attempt * _CV3_RETRY_SEED_STRIDE) + try: + gen = model.inference_zero_shot( + chunk, prompt_text, str(asset_wav), stream=False + ) + parts = [j["tts_speech"] for j in gen] + candidate = torch.cat(parts, dim=-1) if len(parts) > 1 else parts[0] + except Exception as exc: # noqa: BLE001 + # The crash and the silent 0.04 s are the same event: too few + # speech tokens. Here the mel was short enough to break the f0 + # predictor's conv1d before anything was yielded, so it is the + # same re-roll that answers it. + last_exc, last_dur = exc, 0.0 + else: + last_dur = candidate.shape[-1] / sr + if last_dur >= floor_s: + wav = candidate + break + logger.warning( + "[CV3] seg %s chunk %d/%d attempt %d/%d: degenerate render " + "(%.2fs for %d chars, floor %.2fs%s) - re-rolling seed", + seg.seg_index, i + 1, len(chunks), attempt + 1, + _CV3_RENDER_ATTEMPTS, last_dur, len(chunk), floor_s, + f", {type(last_exc).__name__}" if last_exc else "", + ) + if wav is None: + # Hard failure, never a silent short MP3: the run counts it in + # Failed, which is honest, where a 0.10 s file is not. + raise NarratorCosyVoice3Unavailable( + f"seg {seg.seg_index}: chunk {i + 1}/{len(chunks)} still " + f"degenerate after {_CV3_RENDER_ATTEMPTS} attempts (last " + f"{last_dur:.2f}s for {len(chunk)} chars, floor " + f"{floor_s:.2f}s): {chunk[:60]!r}" + ) from last_exc + wavs.append(wav) + elapsed = time.time() - t0 + + pause = torch.zeros(1, int(_CV3_PAUSE_S * sr)) + parts_audio: list[torch.Tensor] = [] + for i, wav in enumerate(wavs): + if i: + parts_audio.append(pause) + parts_audio.append(wav) + full = torch.cat(parts_audio, dim=-1) + + import torchaudio + + wav_buf = io.BytesIO() + torchaudio.save(wav_buf, full, sr, format="wav") + wav_buf.seek(0) + try: + from pydub import AudioSegment + + audio = AudioSegment.from_file(wav_buf, format="wav") + buf = io.BytesIO() + audio.export(buf, format="mp3", bitrate="192k") + mp3_bytes = buf.getvalue() + except Exception as exc: # pragma: no cover — pydub/ffmpeg missing + raise NarratorCosyVoice3Unavailable( + f"seg {seg.seg_index}: WAV->MP3 conversion failed: {exc}" + ) from exc + + mp3_path.write_bytes(mp3_bytes) + duration = audio_duration_mp3(mp3_bytes) + rtf = elapsed / duration if duration > 0 else float("inf") + logger.info( + "[CV3] seg %s: %d chunks, %.1fs audio, RTF %.2f", + seg.seg_index, len(chunks), duration, rtf, + ) + + return TTSResult( + seg_index=seg.seg_index, + speaker=seg.speaker, + reference_id=_CV3_NARRATOR_REFERENCE_ID, + mp3_path=str(mp3_path), + duration_s=duration, + seed=seed, + status="generated", + attempts=attempts, + text_hash=text_hash, ) @@ -986,9 +1494,28 @@ def _load_cached_hashes() -> dict[int, str]: return {} +def _load_cached_reference_ids() -> dict[int, str]: + """Load the engine/voice sentinel (reference_id) per cached segment. + + The narrator cache is engine-aware (#19692 phase 1): a text hash alone + cannot tell which engine rendered the MP3, so flipping + NARRATOR_COSYVOICE3_ROUTING would silently serve the previous engine's + audio. The reference_id sentinel can. + """ + results_path = BASE_DIR / "outputs" / "tts_results.json" + if not results_path.exists(): + return {} + try: + data = json.loads(results_path.read_text(encoding="utf-8")) + return {r["seg_index"]: r.get("reference_id", "") for r in data} + except (json.JSONDecodeError, KeyError): + return {} + + def _synthesize_batch( segments: list[tuple[AnnotatedSegment, str]], cached_hashes: dict[int, str] | None = None, + cached_refs: dict[int, str] | None = None, ) -> list[TTSResult]: """Process a batch of segments concurrently. @@ -997,6 +1524,8 @@ def _synthesize_batch( """ if cached_hashes is None: cached_hashes = {} + if cached_refs is None: + cached_refs = {} results: dict[int, TTSResult] = {} # Quick first pass: check which are truly cached (hash matches) @@ -1007,7 +1536,24 @@ def _synthesize_batch( mp3_path = TTS_DIR / f"seg_{seg.seg_index:04d}_{seg.speaker}.mp3" current_hash = _text_hash(text) - if mp3_path.exists() and cached_hashes.get(seg.seg_index) == current_hash: + cache_ok = ( + mp3_path.exists() and cached_hashes.get(seg.seg_index) == current_hash + ) + if cache_ok and seg.speaker == "narrateur": + # Engine-aware narrator cache (#19692): the MP3 is reusable only + # if the engine that produced it is the selected one — a stale + # MP3 from the previous engine must not shadow the reroute + # (same contract as the segment-level dispatch, #15002). + engine = _selected_narrator_engine() + expected_ref = ( + _NARRATOR_ENGINE_REFERENCE_IDS.get(engine) + if engine + else _resolve_voice(seg) + ) + if cached_refs.get(seg.seg_index) != expected_ref: + cache_ok = False + + if cache_ok: size = mp3_path.stat().st_size duration = audio_duration_mp3(mp3_path.read_bytes()) if size > 0 else 0.0 results[seg.seg_index] = TTSResult( @@ -1035,6 +1581,7 @@ def _gen(item: tuple[int, AnnotatedSegment, str]) -> TTSResult: _, seg, text = item return _synthesize_segment(seg, text) + seg_by_idx = {idx: seg for idx, seg, _ in to_generate} with ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: futures = {pool.submit(_gen, item): item[0] for item in to_generate} for future in as_completed(futures): @@ -1042,9 +1589,19 @@ def _gen(item: tuple[int, AnnotatedSegment, str]) -> TTSResult: try: results[seg_idx] = future.result() except Exception as exc: + # Surface the cause FIRST: a failure record that itself + # crashes on schema validation (speaker="" against a + # Literal) masks the original exception and kills the whole + # run — measured while routing the narrator to CosyVoice3 + # (#19692), where the first synthesis failure took the + # pipeline down without a usable traceback. + logger.error( + "seg %s synthesis failed: %s", seg_idx, exc, exc_info=True + ) + print(f" [P5] seg {seg_idx} FAILED: {type(exc).__name__}: {exc}") results[seg_idx] = TTSResult( seg_index=seg_idx, - speaker="", + speaker=seg_by_idx[seg_idx].speaker, reference_id="", mp3_path="", duration_s=0.0, @@ -1096,6 +1653,7 @@ def run(force: bool = False) -> Path: # Preload cached hashes once (O(N) instead of O(N²)) cached_hashes = _load_cached_hashes() + cached_refs = _load_cached_reference_ids() print(f" Cache: {len(cached_hashes)} entries preloaded") # Process in batches for progress reporting @@ -1107,7 +1665,7 @@ def run(force: bool = False) -> Path: for i in range(batch_start, batch_end) ] - batch_results = _synthesize_batch(batch_segs, cached_hashes) + batch_results = _synthesize_batch(batch_segs, cached_hashes, cached_refs) results.extend(batch_results) for r in batch_results: diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p7_verify.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p7_verify.py index 204de7edd2..9d7bacada9 100644 --- a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p7_verify.py +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/p7_verify.py @@ -54,6 +54,106 @@ def normalize(text: str) -> list[str]: return dp[n][m] / n +def _normalize_words(text: str) -> list[str]: + """Normalize text to comparison words: NFC, lowercase, non-alnum to + spaces, collapsed whitespace. Same convention as the A0C bench (#17586) + so p7 counts what the bench measured.""" + import re + import unicodedata + + text = unicodedata.normalize("NFC", text).lower() + text = "".join(c if c.isalnum() else " " for c in text) + return re.sub(r"\s+", " ", text).strip().split() + + +def _missing_word_spans( + reference: str, hypothesis: str, min_words: int = 3 +) -> list[dict]: + """Passages of >=min_words consecutive words missing from the hypothesis. + + Omission control of #19692 phase 1: the mean WER hides a render that + drops whole sentences (measured on CosyVoice3 A0C, #17586 c.6035598289 — + two segments absent under 3/3 ASRs while WER stayed ~0.12). Method = + the bench's trigram alignment: normalized 3-consecutive-word shingles of + the reference absent from the hypothesis, grouped into maximal spans; a + span of >=3 words means the passage is gone, not mispronounced. + + Known false positives, accepted: an ASR inflection drift ("Tout semblait + accablé" vs "Tous semblaient accablés") breaks every trigram it touches + and can flag a passage the render actually spoke. The per-segment detail + (omission_report.json) exists to be read before acting on a span. + """ + ref_words = _normalize_words(reference) + hyp_words = _normalize_words(hypothesis) + if len(ref_words) < min_words: + return [] + + hyp_trigrams = { + " ".join(hyp_words[i : i + 3]) for i in range(len(hyp_words) - 2) + } + missing_idx = [ + i + for i in range(len(ref_words) - 2) + if " ".join(ref_words[i : i + 3]) not in hyp_trigrams + ] + + spans: list[tuple[int, int]] = [] + for i in missing_idx: + if spans and i == spans[-1][1] + 1: + spans[-1] = (spans[-1][0], i) + else: + spans.append((i, i)) + # An interval (i..j) makes words i..j+2 suspect (trigram window). + return [ + {"start": a, "end": b + 3, "words": " ".join(ref_words[a : b + 3])} + for a, b in spans + if b + 3 - a >= min_words + ] + + +def _voted_missing_spans( + spans_by_model: dict[str, list[dict]], min_models: int = 2 +) -> list[dict]: + """Keep the spans missing under >=min_models distinct ASRs. + + The single-ASR primitive over-flags: a substituted word breaks every + trigram it touches, producing a halo span of ~5 words around one bad + word. On the A0C bench (#17586) the 2-of-3 vote is what separated real + omissions from ASR noise — a passage the render dropped is missing under + every model, a passage the model spoke but one ASR misheard is not. Two + spans from different models support the same omission when they overlap + by >=3 words (halos shift by a word or two between models); the reported + span is the covering one, with its supporting models recorded. + """ + clusters: list[dict] = [] + for model, spans in spans_by_model.items(): + for span in spans: + placed = False + for cluster in clusters: + for other in cluster["spans"]: + overlap = min(span["end"], other["end"]) - max( + span["start"], other["start"] + ) + if overlap >= 3: + cluster["spans"].append(span) + cluster["models"].add(model) + placed = True + break + if placed: + break + if not placed: + clusters.append({"spans": [span], "models": {model}}) + + voted = [] + for cluster in clusters: + models = cluster["models"] + if len(models) < min_models: + continue + covering = max(cluster["spans"], key=lambda s: s["end"] - s["start"]) + voted.append({**covering, "absent_sous": sorted(models)}) + return sorted(voted, key=lambda s: s["start"]) + + def run(force: bool = False) -> Path: """Run P7 — quality verification. Returns path to quality_report.json.""" output_path = BASE_DIR / "outputs" / "quality_report.json" @@ -102,6 +202,7 @@ def run(force: bool = False) -> Path: tts_by_idx = {r["seg_index"]: r for r in tts_data} wers: list[float] = [] + omission_records: list[dict] = [] for idx in sample_indices: if idx >= len(segments): continue @@ -114,18 +215,58 @@ def run(force: bool = False) -> Path: try: mp3_bytes = Path(mp3_path).read_bytes() - resp = req.post( - "http://localhost:8190/v1/audio/transcriptions", - files={"file": (Path(mp3_path).name, mp3_bytes, "audio/mpeg")}, - data={"language": "fr", "model": "large-v3-turbo"}, - headers={"Authorization": f"Bearer {os.getenv('WHISPER_API_KEY', '')}"}, - timeout=30, - ) - if resp.status_code == 200: - hypothesis = resp.json().get("text", "") - if hypothesis: - wer = _word_error_rate(seg.text, hypothesis) - wers.append(wer) + # Omission control (#19692 phase 1): transcribe each + # sampled segment under three models and vote — the + # 2-of-3 vote is what separated real omissions from ASR + # noise on the A0C bench (#17586). WER keeps its single + # large-v3-turbo convention (unchanged metric). + hypotheses: dict[str, str] = {} + for model in ("tiny", "large-v3", "large-v3-turbo"): + resp = req.post( + "http://localhost:8190/v1/audio/transcriptions", + files={ + "file": (Path(mp3_path).name, mp3_bytes, "audio/mpeg") + }, + data={"language": "fr", "model": model}, + headers={ + "Authorization": f"Bearer {os.getenv('WHISPER_API_KEY', '')}" + }, + timeout=60, + ) + if resp.status_code == 200: + text = resp.json().get("text", "") + if text: + hypotheses[model] = text + else: + print( + f" seg {idx} {model}: HTTP {resp.status_code}" + ) + + hypothesis = hypotheses.get("large-v3-turbo", "") + if hypothesis: + wer = _word_error_rate(seg.text, hypothesis) + wers.append(wer) + if hypotheses: + spans_by_model = { + model: _missing_word_spans(seg.text, hyp) + for model, hyp in hypotheses.items() + } + voted = _voted_missing_spans(spans_by_model) + if voted: + omission_records.append({ + "seg_index": seg.seg_index, + "speaker": seg.speaker, + "models_present": sorted(hypotheses), + "n_spans": len(voted), + "n_words": sum( + s["end"] - s["start"] for s in voted + ), + "spans": voted, + "per_model_spans": { + m: [s["words"] for s in spans] + for m, spans in spans_by_model.items() + }, + }) except Exception as e: print(f" seg {idx} STT error: {e}") @@ -141,6 +282,55 @@ def run(force: bool = False) -> Path: print(f" Mean WER: {wer_results['mean']}") print(f" P95 WER: {wer_results['p95']}") + # Omission control (#19692 phase 1): counts go into the quality + # report (numeric schema), the per-segment detail goes to its own + # artifact so a span can be read before acting on it (known + # false positives: ASR inflection drifts, cf _missing_word_spans). + wer_results["omission_segments"] = float(len(omission_records)) + wer_results["omission_spans_3plus"] = float( + sum(r["n_spans"] for r in omission_records) + ) + wer_results["omission_words_3plus"] = float( + sum(r["n_words"] for r in omission_records) + ) + omission_path = BASE_DIR / "outputs" / "omission_report.json" + omission_path.write_text( + json.dumps( + { + "method": ( + "trigram alignment + 2-of-3 ASR vote (#19692 " + "phase 1, method of the A0C bench #17586): " + "normalized 3-consecutive-word shingles of the " + "source absent from each ASR hypothesis, grouped " + "into maximal spans; a span counts as an omission " + "when missing under >=2 of {tiny, large-v3, " + "large-v3-turbo} (halos of single-word ASR errors " + "do not survive the vote)" + ), + "sampled_segments": len(sample_indices), + "segments_with_omissions": len(omission_records), + "records": omission_records, + }, + indent=2, + ensure_ascii=False, + ), + encoding="utf-8", + ) + n_spans = int(wer_results["omission_spans_3plus"]) + if omission_records: + print( + f" OMISSIONS: {n_spans} span(s) >=3 mots sur " + f"{len(omission_records)} segment(s) -> {omission_path.name}" + ) + for rec in omission_records[:5]: + first = rec["spans"][0]["words"] + print( + f" seg {rec['seg_index']} ({rec['speaker']}): " + f"{rec['n_spans']} span(s), ex: {first[:70]}" + ) + else: + print(" OMISSIONS: none >=3 mots") + # Step 2: Diarization on full audiobook via Whisper WebUI Gradio API print("[P7] Step 2: Speaker diarization...") print(" This may take 10-30 minutes...") diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_cv3_narrator.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_cv3_narrator.py new file mode 100644 index 0000000000..6aa574d5ef --- /dev/null +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_cv3_narrator.py @@ -0,0 +1,567 @@ +"""CosyVoice3 narrator path, phase 1 (#19692) — offline tests. + +Covers the sentence chunker (identity, max_chars, clause fallback, no +mid-word cut) and the torch-free failure guards of the CV3 synthesis +branch. The live model is exercised by the measured run in the PR body — +these tests guard the chunking and dispatch logic without torch. + +Run from the v4 directory: + python -m pytest tests/test_p5_cv3_narrator.py -v +""" +from __future__ import annotations + +import sys +import types +from pathlib import Path + +_V4 = Path(__file__).resolve().parent.parent +if str(_V4.parent) not in sys.path: + sys.path.insert(0, str(_V4.parent)) + + +# Extract C of the A0C bench (#17586): real narration long enough to need +# clause fallback and multi-chunk grouping. +_EXTRAIT_C = ( + "Pendant plusieurs jours, des fugitifs discrets arrivaient par tous les " + "chemins, gens du monde, des femmes, des enfants. Des artilleurs " + "sombres alignés avec des fantassins divers, sans drapeau, sans " + "régiment, venaient se ranger sous la croix de guerre. Tous semblaient " + "accablés, éreintés, incapables de comprendre ou de se décider, prêts à " + "la fuite puis à la bataille. La maison du notaire était pleine, " + "jour et nuit." +) + + +def _build_seg(*, speaker: str, text: str = "x"): + from v4.schemas import AnnotatedSegment + + return AnnotatedSegment( + seg_index=1, + speaker=speaker, + type="narration" if speaker == "narrateur" else "dialogue", + text=text, + annotated_text=text, + ) + + +def test_chunker_preserves_every_word(): + """Identity: chunks reassemble to the source — the chunker can never be + the source of a dropped passage (#19692's omission class lives in the + model, the chunker must not add its own).""" + from v4.p5_tts import _chunk_narration + + for text in (_EXTRAIT_C, "Une seule phrase.", "Trois mots. Quatre mots ici. Et voilà."): + chunks = _chunk_narration(text) + assert " ".join(chunks).replace(" ", " ").strip() == " ".join(text.split()) + + +def test_chunker_respects_max_chars(): + """Every chunk stays <= max_chars — the bound the A0C bench measured + (#17586 spec: 280).""" + from v4.p5_tts import _chunk_narration + + chunks = _chunk_narration(_EXTRAIT_C) + assert chunks, "chunker returned nothing for a long narration" + assert all(len(c) <= 280 for c in chunks), [len(c) for c in chunks] + + +def test_chunker_splits_at_sentence_boundaries(): + """Chunks break after . ! ? — never mid-word, never mid-sentence when + the sentence fits (small max_chars forces one sentence per chunk).""" + from v4.p5_tts import _chunk_narration + + chunks = _chunk_narration("Phrase un. Phrase deux ! Phrase trois ?", max_chars=14) + assert chunks == ["Phrase un.", "Phrase deux !", "Phrase trois ?"] + # With the default 280 the greedy packer merges short sentences into one + # chunk — whole units only, identity preserved. + merged = _chunk_narration("Phrase un. Phrase deux ! Phrase trois ?") + assert merged == ["Phrase un. Phrase deux ! Phrase trois ?"] + + +def test_chunker_clause_fallback_for_long_sentence(): + """A sentence over max_chars falls back to clause boundaries (, ; :) + rather than truncating mid-word.""" + from v4.p5_tts import _chunk_narration + + long_sentence = ( + "Une phrase très longue, avec des propositions, des virgules, des " + "points-virgules ; et encore des mots, beaucoup de mots, toujours " + "plus de mots pour dépasser largement la limite de deux cent " + "quatre-vingts caractères, encore quelques mots supplémentaires " + "ici, puis d'autres encore, et enfin une fin de phrase." + ) + assert len(long_sentence) > 280 + chunks = _chunk_narration(long_sentence) + assert all(len(c) <= 280 for c in chunks) + # Identity still holds through the fallback. + rejoined = " ".join(chunks) + assert rejoined == " ".join(long_sentence.split()) + + +def test_chunker_refuses_undivisible_clause(): + """A clause longer than max_chars with no , ; : boundary is a hard + error: a mid-word cut would corrupt the narration instead of failing + loudly.""" + from v4.p5_tts import _chunk_narration + + monolith = "mot " * 100 # 400 chars, no punctuation + try: + _chunk_narration(monolith.strip()) + except ValueError: + pass + else: + raise AssertionError("undivisible clause must raise ValueError, not truncate") + + +def test_chunker_empty_input_returns_empty(): + from v4.p5_tts import _chunk_narration + + assert _chunk_narration("") == [] + assert _chunk_narration(" \n ") == [] + + +def test_cv3_unavailable_is_runtime_error(): + """NarratorCosyVoice3Unavailable shares the NarratorQwenUnavailable + contract: a single RuntimeError class for any CV3-side failure, hard + failure, never a silent FishAudio fallback (#15002 acceptance 6).""" + from v4.p5_tts import NarratorCosyVoice3Unavailable, NarratorQwenUnavailable + + assert issubclass(NarratorCosyVoice3Unavailable, RuntimeError) + assert NarratorCosyVoice3Unavailable is not NarratorQwenUnavailable + + +def test_cv3_reference_id_is_distinct_sentinel(): + """The CV3 branch stamps its own reference_id so tts_results.json and p7 + can tell which engine rendered a segment.""" + from v4.p5_tts import _CV3_NARRATOR_REFERENCE_ID, _QWEN_NARRATOR_REFERENCE_ID + + assert _CV3_NARRATOR_REFERENCE_ID != _QWEN_NARRATOR_REFERENCE_ID + assert not _CV3_NARRATOR_REFERENCE_ID.startswith("v4_") + + +def test_synthesize_narrator_cv3_raises_on_empty_input(): + """Empty text after bracket stripping raises BEFORE any torch import — + hermetic: the guard must not require the CosyVoice runtime.""" + from v4.p5_tts import NarratorCosyVoice3Unavailable, _synthesize_narrator_cosyvoice3 + + seg = _build_seg(speaker="narrateur", text="") + try: + _synthesize_narrator_cosyvoice3( + seg=seg, + fishaudio_text="[whispering]", # empty after strip + mp3_path=Path("should_never_be_written.mp3"), + seed=42, + text_hash="deadbeef", + ) + except NarratorCosyVoice3Unavailable: + return # success: raised as designed + raise AssertionError("NarratorCosyVoice3Unavailable not raised on empty input") + + +# Durations measured 2026-10-07 by bisect 2/3 (WAV, not MP3) on a free GPU. +# The same text renders 0.04 s at seed 42 -- one speech token, CosyVoice's +# token_hop_len -- and a plausible duration at other seeds; the degeneracy is +# a function of (text, seed), reproducible bit-for-bit within a process. +_MEASURED_DEGENERATE = [ + ("Oui.", 0.04), + ("Il marcha.", 0.04), + ("C'était un Bordelais.", 0.04), + ("C'était un Bordelais", 0.04), # trailing period is not the trigger + ("Il marchait lentement sur le quai désert.", 0.04), + ("Il marchait lentement sur le quai désert.", 0.20), # seed 43 + ("Le vieux marin regardait.", 0.16), # seed 7 +] +_MEASURED_LEGITIMATE = [ + ("Oui.", 0.36), + ("Oui.", 0.64), + ("Il marcha.", 0.76), + ("Il marcha.", 1.64), + ("C'était un Bordelais.", 1.16), + ("C'était un Bordelais.", 1.88), + ("Il marchait lentement sur le quai désert.", 2.24), + ("Le vieux marin regardait.", 1.28), + ("Le vieux marin regardait.", 2.12), + ("Le vieux marin regardait la mer sans rien dire.", 2.56), +] + + +def test_cv3_floor_separates_measured_degenerate_from_legitimate(): + """The floor must accept every measured legitimate render and reject every + measured degenerate one -- both columns come from the same sweep, so the + threshold cannot be tuned without contradicting the data.""" + from v4.p5_tts import _cv3_min_expected_s + + for text, dur in _MEASURED_DEGENERATE: + floor = _cv3_min_expected_s(text) + assert dur < floor, f"{text!r} at {dur}s must be rejected (floor {floor:.2f}s)" + for text, dur in _MEASURED_LEGITIMATE: + floor = _cv3_min_expected_s(text) + assert dur >= floor, f"{text!r} at {dur}s must be accepted (floor {floor:.2f}s)" + + +def test_cv3_floor_sits_in_the_measured_empty_band(): + """Corpus-level calibration (#19692), separate from the bisect table above. + + Over the 270-narrator artifact the fastest LEGITIMATE segment runs 38.9 + chars/s and the slowest ASR-confirmed DEFECT runs 43.1 (seg 227: 156 + reference words, 79 transcribed). Nothing in the corpus lies between, so + the floor is placed in an empty band -- and its two margins are therefore + thin, which the constant documents rather than hides. Restated as a rate + on a 100-char chunk: 2.57 s must pass, 2.32 s must not. + """ + from v4.p5_tts import _cv3_min_expected_s + + text = "x" * 100 # 2.57 s -> 38.9 chars/s; 2.32 s -> 43.1 chars/s + assert 2.57 >= _cv3_min_expected_s(text), "38.9 chars/s is legitimate" + assert 2.32 < _cv3_min_expected_s(text), "43.1 chars/s is a defect" + + +def test_cv3_floor_keeps_an_absolute_term(): + """A ratio alone would let a tiny chunk through on 0.04 s of audio.""" + from v4.p5_tts import _CV3_MIN_ABS_S, _cv3_min_expected_s + + assert _cv3_min_expected_s("Oui.") == _CV3_MIN_ABS_S + assert _cv3_min_expected_s(".") == _CV3_MIN_ABS_S + assert _CV3_MIN_ABS_S > 0.04, "the absolute floor must reject one speech token" + + +class _FakeTensor: + def __init__(self, n: int): + self.n = n + self.shape = (1, n) + + +class _FakeTorch: + """Just enough torch for the CV3 branch: the guard, the pause, the concat.""" + + def __init__(self): + self.seeds: list[int] = [] + + def manual_seed(self, value: int): + self.seeds.append(value) + + def cat(self, tensors, dim=-1): + return _FakeTensor(sum(t.n for t in tensors)) + + def zeros(self, *shape): + return _FakeTensor(shape[-1]) + + +class _FakeModel: + """Yields a WAV of a scripted duration per call; "CRASH" raises the + RuntimeError the f0 predictor produces when the mel is too short.""" + + sample_rate = 24000 + + def __init__(self, script): + self.script = list(script) + self.calls: list[str] = [] + + def inference_zero_shot(self, text, prompt_text, wav_path, stream=False): + dur = self.script[len(self.calls)] + self.calls.append(text) + if dur == "CRASH": + raise RuntimeError( + "Calculated padded input size per channel: (3). Kernel size: (4)." + ) + yield {"tts_speech": _FakeTensor(int(dur * self.sample_rate))} + + +def _install_fake_runtime(monkeypatch, model): + """Wire the fake torch/torchaudio/pydub and the stub client, so the retry + loop is exercised hermetically -- no GPU, no CosyVoice install.""" + import sys + import types + + fake_torch = _FakeTorch() + monkeypatch.setitem(sys.modules, "torch", fake_torch) + + torchaudio = types.ModuleType("torchaudio") + torchaudio.save = lambda buf, wav, sr, format=None: buf.write(b"WAV") + monkeypatch.setitem(sys.modules, "torchaudio", torchaudio) + + pydub = types.ModuleType("pydub") + + class _FakeAudio: + @staticmethod + def from_file(buf, format=None): + return _FakeAudio() + + def export(self, buf, format=None, bitrate=None): + buf.write(b"MP3") + + pydub.AudioSegment = _FakeAudio + monkeypatch.setitem(sys.modules, "pydub", pydub) + + client = types.SimpleNamespace( + _bootstrap_paths=lambda: (None, None, "asset.wav"), + PROMPT_TEXT_ZH="prompt", + ENDOFPROMPT="<|endofprompt|>", + ) + monkeypatch.setattr("v4.p5_tts._get_cv3_client", lambda: client) + monkeypatch.setattr("v4.p5_tts._load_cv3_model", lambda: model) + monkeypatch.setattr("v4.p5_tts.audio_duration_mp3", lambda b: 1.5) + return fake_torch + + +def test_cv3_re_rolls_a_degenerate_render(tmp_path, monkeypatch): + """A 0.04 s render is retried with a different seed instead of being + returned as generated -- and the retry is the same text, same engine.""" + from v4.p5_tts import _synthesize_narrator_cosyvoice3 + + model = _FakeModel([0.04, 0.04, 1.5]) # two degenerate, then a real one + fake_torch = _install_fake_runtime(monkeypatch, model) + seg = _build_seg(speaker="narrateur", text="Il marcha.") + out = _synthesize_narrator_cosyvoice3( + seg=seg, fishaudio_text="Il marcha.", + mp3_path=tmp_path / "seg_0001_narrateur.mp3", seed=42, text_hash="h", + ) + assert out.status == "generated" + assert out.attempts == 3, "attempts must report the real re-roll count" + assert len(model.calls) == 3 + assert len(set(model.calls)) == 1, "a re-roll must not rewrite the text" + # attempt 0 keeps the pre-guard seed; each retry moves to a fresh one. + assert fake_torch.seeds == [42, 42 + 104_729, 42 + 2 * 104_729] + + +def test_cv3_re_rolls_the_crash_face_of_the_same_event(tmp_path, monkeypatch): + """The conv1d RuntimeError and the silent 0.04 s are one event with two + faces, so the re-roll answers the crash too instead of failing the run.""" + from v4.p5_tts import _synthesize_narrator_cosyvoice3 + + model = _FakeModel(["CRASH", 1.8]) + _install_fake_runtime(monkeypatch, model) + seg = _build_seg(speaker="narrateur", text="Il marcha.") + out = _synthesize_narrator_cosyvoice3( + seg=seg, fishaudio_text="Il marcha.", + mp3_path=tmp_path / "seg_0001_narrateur.mp3", seed=42, text_hash="h", + ) + assert out.status == "generated" + assert out.attempts == 2 + + +def test_cv3_fails_loudly_when_every_attempt_degenerates(tmp_path, monkeypatch): + """Never a silent short MP3: an unresolvable chunk raises, so the run + counts it in Failed rather than shipping 0.10 s of nothing.""" + from v4.p5_tts import ( + _CV3_RENDER_ATTEMPTS, NarratorCosyVoice3Unavailable, + _synthesize_narrator_cosyvoice3, + ) + + model = _FakeModel([0.04] * _CV3_RENDER_ATTEMPTS) + _install_fake_runtime(monkeypatch, model) + mp3 = tmp_path / "seg_0001_narrateur.mp3" + seg = _build_seg(speaker="narrateur", text="Il marcha.") + try: + _synthesize_narrator_cosyvoice3( + seg=seg, fishaudio_text="Il marcha.", + mp3_path=mp3, seed=42, text_hash="h", + ) + except NarratorCosyVoice3Unavailable as exc: + assert "degenerate" in str(exc) + assert len(model.calls) == _CV3_RENDER_ATTEMPTS + assert not mp3.exists(), "a failed render must not leave a partial MP3" + return + raise AssertionError("an all-degenerate chunk must raise, not return") + + +class _StubLLM: + """Minimal stand-in for CosyVoice3LM: records what the sampler hides.""" + + def __init__(self, stop_token_ids=(100, 101, 102)): + self.stop_token_ids = list(stop_token_ids) + self.calls: list[tuple[list[int], bool]] = [] + + def sampling_ids(self, weighted_scores, decoded_tokens, sampling, ignore_eos=True): + masked = sorted( + k for k, v in weighted_scores.items() if v == float("-inf") + ) + self.calls.append((masked, ignore_eos)) + return 0 + + +def _cv3_model(llm): + """The shape the real loader returns: `AutoModel()` is a factory giving + `CosyVoice3`, whose `.model` is the `CosyVoice3Model` carrying `.llm`.""" + return types.SimpleNamespace(model=types.SimpleNamespace(llm=llm)) + + +# Upstream's non-vLLM decoder breaks on ANY of three stop ids but masks only +# `speech_token_size` -- which is `stop_token_ids[0]`. The other two walk +# through the min_len floor, and that is the whole degeneracy: a render ends +# after one speech token, 1/25 s = 0.04 s (#19692). +def test_cv3_stop_floor_patch_masks_every_stop_id(): + from v4.p5_tts import _patch_cv3_stop_token_floor + + llm = _StubLLM((100, 101, 102)) + assert _patch_cv3_stop_token_floor(_cv3_model(llm)) == "patched" + + scores = {100: 0.5, 101: 0.5, 102: 0.5, 7: 9.0} + llm.sampling_ids(scores, [], 25, ignore_eos=True) + assert scores[100] == float("-inf"), "stop_token_ids[0] was already masked" + assert scores[101] == float("-inf"), "the leaking id 1 must be masked too" + assert scores[102] == float("-inf"), "the leaking id 2 must be masked too" + assert scores[7] == 9.0, "a speech token must stay samplable" + # The inner call runs with ignore_eos=False: masking twice would make the + # protected prefix and the free tail indistinguishable. + assert llm.calls[-1][1] is False + + +def test_cv3_stop_floor_patch_is_idempotent(): + """Two prompt builds in one process must not stack wrappers -- a wrapper + over a wrapper would mask on the tail too, and generation would never + end.""" + from v4.p5_tts import _patch_cv3_stop_token_floor + + llm = _StubLLM() + model = _cv3_model(llm) + assert _patch_cv3_stop_token_floor(model) == "patched" + first = llm.sampling_ids + assert _patch_cv3_stop_token_floor(model) == "already-patched" + assert llm.sampling_ids is first + assert llm.calls == [], "patching alone must not sample" + + +def test_cv3_stop_floor_leaves_the_tail_able_to_stop(): + """Past min_len the decoder MUST be able to end: with ignore_eos=False + nothing is masked, so a stop id can win and break the loop.""" + from v4.p5_tts import _patch_cv3_stop_token_floor + + llm = _StubLLM((100, 101, 102)) + _patch_cv3_stop_token_floor(_cv3_model(llm)) + scores = {100: 0.5, 101: -2.0, 102: -3.0} + llm.sampling_ids(scores, [], 25, ignore_eos=False) + assert scores == {100: 0.5, 101: -2.0, 102: -3.0} + + +def test_cv3_stop_floor_patch_fails_loudly_when_the_decoder_moved(): + """A loader that returns another shape must stop the render rather than + quietly restore the leak -- and must SAY which shape it saw, so localising + it costs a read instead of a GPU run.""" + from v4.p5_tts import NarratorCosyVoice3Unavailable, _patch_cv3_stop_token_floor + + for model, why in ( + (types.SimpleNamespace(), "no `.model` at all"), + (types.SimpleNamespace(model=types.SimpleNamespace()), "`.model` with no llm"), + # The measured wrong path: a first version of the patch addressed + # `model.llm` and raised on all nine probe segments. + (types.SimpleNamespace(llm=_StubLLM()), "the decoder at the top level"), + ): + try: + _patch_cv3_stop_token_floor(model) + except NarratorCosyVoice3Unavailable as exc: + assert "#19692" in str(exc), why + assert "model.llm" in str(exc), why + else: + raise AssertionError(f"must raise: {why}") + + +def test_cv3_load_caches_the_refusal_instead_of_reloading_per_segment(monkeypatch): + """A failing patch must not reload the model for every segment: measured, + nine probe segments cost six loads (~16 s each) before the run gave up.""" + from v4 import p5_tts + from v4.p5_tts import NarratorCosyVoice3Unavailable + + loads = [] + + def _load(): + loads.append(1) + return types.SimpleNamespace() # a shape the patch must reject + + monkeypatch.setattr( + p5_tts, + "_get_cv3_client", + lambda: types.SimpleNamespace(load_model=_load), + ) + monkeypatch.setattr(p5_tts, "_CV3_MODEL_CACHE", {}) + + for _ in range(3): + try: + p5_tts._load_cv3_model() + except NarratorCosyVoice3Unavailable: + pass + else: + raise AssertionError("an unpatched loader must not hand back a model") + assert len(loads) == 1, f"the refusal must be cached, got {len(loads)} loads" + + +def test_load_cv3_model_patches_the_decoder_on_the_way_out(monkeypatch): + """The patch is not optional plumbing: loading the model without it would + leave every narrator render on the leaky path.""" + from v4 import p5_tts + + llm = _StubLLM() + model = _cv3_model(llm) + monkeypatch.setattr( + p5_tts, + "_get_cv3_client", + lambda: types.SimpleNamespace(load_model=lambda: model), + ) + monkeypatch.setattr(p5_tts, "_CV3_MODEL_CACHE", {}) + + assert p5_tts._load_cv3_model() is model + assert getattr(llm, "_v4_stop_floor_patched", False) is True + # Second call serves the cached model: the patch is not re-applied. + assert p5_tts._load_cv3_model() is model + assert llm.calls == [] + + +def test_narrator_cache_is_engine_aware(tmp_path, monkeypatch): + """Flipping NARRATOR_COSYVOICE3_ROUTING must not serve the previous + engine's MP3 from the batch cache: the narrator cache keys on the + engine sentinel (#19692) — a text hash alone cannot tell which engine + rendered the file. Dialogue segments stay cached regardless (they never + change engine).""" + from v4 import p5_tts + from v4.schemas import TTSResult + + (tmp_path / "seg_0001_narrateur.mp3").write_bytes(b"fake") + (tmp_path / "seg_0001_loiseau.mp3").write_bytes(b"fake") + monkeypatch.setattr(p5_tts, "TTS_DIR", tmp_path) + monkeypatch.setattr(p5_tts, "audio_duration_mp3", lambda b: 1.0) + monkeypatch.setattr(p5_tts, "thermal_wait", lambda *a, **k: 0) + + def _stub(seg, text): + return TTSResult( + seg_index=seg.seg_index, speaker=seg.speaker, reference_id="stub", + mp3_path="stub", duration_s=1.0, seed=42, status="generated", + attempts=1, text_hash="stub", + ) + + monkeypatch.setattr(p5_tts, "_synthesize_segment", _stub) + + narr = _build_seg(speaker="narrateur", text="Il marcha.") + dlg = _build_seg(speaker="loiseau", text="Bonjour.") + h_narr = p5_tts._text_hash("Il marcha.") + h_dlg = p5_tts._text_hash("Bonjour.") + + orig_qwen = p5_tts._NARRATOR_QWEN_ROUTING + orig_cv3 = p5_tts._NARRATOR_COSYVOICE3_ROUTING + try: + p5_tts._NARRATOR_QWEN_ROUTING = False + p5_tts._NARRATOR_COSYVOICE3_ROUTING = True + + # Cache says Qwen rendered it; CV3 is selected -> regenerate. + out = p5_tts._synthesize_batch( + [(narr, "Il marcha.")], + {1: h_narr}, + {1: p5_tts._QWEN_NARRATOR_REFERENCE_ID}, + ) + assert out[0].status == "generated" + + # Cache says CV3 rendered it; CV3 is selected -> served cached. + out = p5_tts._synthesize_batch( + [(narr, "Il marcha.")], + {1: h_narr}, + {1: p5_tts._CV3_NARRATOR_REFERENCE_ID}, + ) + assert out[0].status == "cached" + + # Dialogue segments never change engine: hash match is enough. + out = p5_tts._synthesize_batch( + [(dlg, "Bonjour.")], {1: h_dlg}, {1: "v4_loiseau_voice"} + ) + assert out[0].status == "cached" + finally: + p5_tts._NARRATOR_QWEN_ROUTING = orig_qwen + p5_tts._NARRATOR_COSYVOICE3_ROUTING = orig_cv3 diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_narrator_routing.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_narrator_routing.py index 3b61e4b9f0..770a6d87f0 100644 --- a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_narrator_routing.py +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p5_narrator_routing.py @@ -31,46 +31,114 @@ def _build_seg(*, speaker: str, text: str = "x"): def test_routes_narrator_to_qwen_when_flag_on(): - """When NARRATOR_QWEN_ROUTING is ON (default), narrator segments route.""" - from v4.p5_tts import _should_route_narrator_to_qwen + """When NARRATOR_QWEN_ROUTING is ON (default), narrator segments route. + Default behavior is UNCHANGED by phase 1 (#19692): Qwen VoiceDesign + stays the narrator engine.""" + from v4.p5_tts import _should_route_narrator_to seg = _build_seg(speaker="narrateur") - assert _should_route_narrator_to_qwen(seg) is True + assert _should_route_narrator_to("qwen_voicedesign", seg) is True def test_does_not_route_non_narrator_speakers(): """Acceptance #1: routing is limited to narrateur; other speakers stay - on FishAudio clone even when the flag is ON.""" - from v4.p5_tts import _should_route_narrator_to_qwen - - for speaker in ( - "elisabeth_rousset", - "loiseau", - "comte", - "comtesse", - "cornudet", - "officier", - "figurant", - ): - seg = _build_seg(speaker=speaker) - assert _should_route_narrator_to_qwen(seg) is False, ( - f"non-narrator speaker '{speaker}' must NOT route to Qwen" - ) + on FishAudio clone even when an engine flag is ON.""" + from v4.p5_tts import _should_route_narrator_to + + for engine in ("qwen_voicedesign", "cosyvoice3"): + for speaker in ( + "elisabeth_rousset", + "loiseau", + "comte", + "comtesse", + "cornudet", + "officier", + "figurant", + ): + seg = _build_seg(speaker=speaker) + assert _should_route_narrator_to(engine, seg) is False, ( + f"non-narrator speaker '{speaker}' must NOT route to {engine}" + ) def test_routes_off_when_flag_disabled(): - """Env flag NARRATOR_QWEN_ROUTING=0 disables routing (legacy path).""" + """Env flag NARRATOR_QWEN_ROUTING=0 with no other engine selected + disables routing (legacy FishAudio path).""" from v4 import p5_tts original = p5_tts._NARRATOR_QWEN_ROUTING try: p5_tts._NARRATOR_QWEN_ROUTING = False seg = _build_seg(speaker="narrateur") - assert p5_tts._should_route_narrator_to_qwen(seg) is False + assert p5_tts._should_route_narrator_to("qwen_voicedesign", seg) is False finally: p5_tts._NARRATOR_QWEN_ROUTING = original +def test_cosyvoice3_off_by_default(): + """Phase 1 (#19692): CosyVoice3 routing is OFF by default — the default + narrator engine does not change until the association's listening picks + the winner (#17586).""" + from v4 import p5_tts + + assert p5_tts._NARRATOR_COSYVOICE3_ROUTING is False + + +def test_cosyvoice3_routes_when_selected(): + """With Qwen off and CosyVoice3 on, the narrator routes to CosyVoice3 + and NOT to Qwen — engine selection is exclusive by construction.""" + from v4 import p5_tts + + orig_qwen = p5_tts._NARRATOR_QWEN_ROUTING + orig_cv3 = p5_tts._NARRATOR_COSYVOICE3_ROUTING + try: + p5_tts._NARRATOR_QWEN_ROUTING = False + p5_tts._NARRATOR_COSYVOICE3_ROUTING = True + seg = _build_seg(speaker="narrateur") + assert p5_tts._should_route_narrator_to("cosyvoice3", seg) is True + assert p5_tts._should_route_narrator_to("qwen_voicedesign", seg) is False + finally: + p5_tts._NARRATOR_QWEN_ROUTING = orig_qwen + p5_tts._NARRATOR_COSYVOICE3_ROUTING = orig_cv3 + + +def test_both_engine_flags_active_raises(): + """Two active engine flags are a configuration error, never a silent + precedence — same no-silent-swap contract as #15002 acceptance 6.""" + from v4 import p5_tts + + orig_qwen = p5_tts._NARRATOR_QWEN_ROUTING + orig_cv3 = p5_tts._NARRATOR_COSYVOICE3_ROUTING + try: + p5_tts._NARRATOR_QWEN_ROUTING = True + p5_tts._NARRATOR_COSYVOICE3_ROUTING = True + seg = _build_seg(speaker="narrateur") + try: + p5_tts._should_route_narrator_to("cosyvoice3", seg) + except ValueError: + pass + else: + raise AssertionError("two active engine flags must raise ValueError") + finally: + p5_tts._NARRATOR_QWEN_ROUTING = orig_qwen + p5_tts._NARRATOR_COSYVOICE3_ROUTING = orig_cv3 + + +def test_selected_narrator_engine_none_when_all_off(): + """No flag on -> None (legacy FishAudio narrator), not an error.""" + from v4 import p5_tts + + orig_qwen = p5_tts._NARRATOR_QWEN_ROUTING + orig_cv3 = p5_tts._NARRATOR_COSYVOICE3_ROUTING + try: + p5_tts._NARRATOR_QWEN_ROUTING = False + p5_tts._NARRATOR_COSYVOICE3_ROUTING = False + assert p5_tts._selected_narrator_engine() is None + finally: + p5_tts._NARRATOR_QWEN_ROUTING = orig_qwen + p5_tts._NARRATOR_COSYVOICE3_ROUTING = orig_cv3 + + def test_strip_brackets_for_qwen_removes_inline_tags(): """VoiceDesign receives the bare text; brackets would be ignored or, worse, vocalized (WER regression #1277/#1485 on FishAudio bracket text). @@ -153,3 +221,71 @@ def test_synthesize_narrator_qwen_raises_on_empty_input(): except NarratorQwenUnavailable: return # success: raised as designed raise AssertionError("NarratorQwenUnavailable not raised on empty input") + + +def test_compose_tts_text_keeps_full_text_for_cosyvoice3_narrator(): + """Phase 1 (#19692): the 500-char cap in _compose_tts_text is an S2-Pro + input limit. A narrator rerouted to CosyVoice3 is bounded per-chunk by + _chunk_narration instead — truncating here silently drops the tail of + every long narration (measured: seg 1, 981 chars -> ~440 rendered, + 17.8 s of audio for a 950-char segment = an omission p7's control + would attribute to the engine). Default (Qwen) and non-narrator + speakers keep the cap: their engines really do have it.""" + from v4 import p5_tts + + long_text = ( + "Pendant plusieurs jours de suite des lambeaux d'armee en deroute " + "avaient traverse la ville. " * 8 + ).strip() + assert len(long_text) > p5_tts._MAX_TTS_CHARS + + orig_qwen = p5_tts._NARRATOR_QWEN_ROUTING + orig_cv3 = p5_tts._NARRATOR_COSYVOICE3_ROUTING + try: + # CosyVoice3 selected: narrator keeps the full text. + p5_tts._NARRATOR_QWEN_ROUTING = False + p5_tts._NARRATOR_COSYVOICE3_ROUTING = True + narrator = _build_seg(speaker="narrateur", text=long_text) + composed = p5_tts._compose_tts_text(narrator) + assert len(composed) > p5_tts._MAX_TTS_CHARS + assert not composed.endswith("...") + assert long_text[: p5_tts._MAX_TTS_CHARS] in composed + + # Same flags, non-narrator: S2-Pro path keeps its cap. + speaker = _build_seg(speaker="loiseau", text=long_text) + capped = p5_tts._compose_tts_text(speaker) + assert len(capped) <= p5_tts._MAX_TTS_CHARS + assert capped.endswith("...") + finally: + p5_tts._NARRATOR_QWEN_ROUTING = orig_qwen + p5_tts._NARRATOR_COSYVOICE3_ROUTING = orig_cv3 + + # Default (Qwen narrator): behavior unchanged — cap still applies. + narrator = _build_seg(speaker="narrateur", text=long_text) + composed = p5_tts._compose_tts_text(narrator) + assert len(composed) <= p5_tts._MAX_TTS_CHARS + assert composed.endswith("...") + + +def test_strip_brackets_never_glues_neighbouring_words(): + """A mid-text tag must become a space, never nothing: the old pattern + ate the tag plus its trailing whitespace and glued the neighbours + ("route,[short pause] quand" -> "route,quand"). Measured #19692: + CosyVoice3 fed such a glued run-on omits everything before the glue + point (seg 57: audio starts at "quand"), and the gluing also hides + sentence boundaries from the CV3 chunker.""" + from v4.p5_tts import _strip_brackets_for_qwen + + stripped = _strip_brackets_for_qwen( + "[breathing]Chacun guettait pour apercevoir un cabaret sur la route," + "[short pause] quand la diligence sombra dans un amoncellement " + "de neige[pause] et il fallut deux heures pour la dégager." + ) + assert "route, quand la diligence" in stripped + assert "neige et il fallut" in stripped + assert "route,quand" not in stripped + assert "neigeet" not in stripped + # A tag between two sentences restores the boundary instead of eating it. + assert _strip_brackets_for_qwen( + "Ils entrèrent dans la ville.[pause] On voyait des soldats." + ) == "Ils entrèrent dans la ville. On voyait des soldats." diff --git a/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p7_omission_control.py b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p7_omission_control.py new file mode 100644 index 0000000000..7fe7f38e4c --- /dev/null +++ b/MyIA.AI.Notebooks/GenAI/Audio/04-Applications/v4/tests/test_p7_omission_control.py @@ -0,0 +1,168 @@ +"""Omission control of p7, phase 1 (#19692) — offline tests. + +Covers _missing_word_spans: the trigram alignment that counts >=3-word +passages missing from an ASR hypothesis. Method measured on the A0C bench +(#17586 c.6035598289): the unchunked CosyVoice3 render dropped whole +sentences that the mean WER (~0.12) could not see. + +Run from the v4 directory: + python -m pytest tests/test_p7_omission_control.py -v +""" +from __future__ import annotations + +import sys +from pathlib import Path + +_V4 = Path(__file__).resolve().parent.parent +if str(_V4.parent) not in sys.path: + sys.path.insert(0, str(_V4.parent)) + + +def _spans(source: str, hyp: str) -> list[dict]: + from v4.p7_verify import _missing_word_spans + + return _missing_word_spans(source, hyp) + + +def test_no_span_when_hypothesis_matches(): + """A faithful hypothesis yields zero spans — the control must not cry + wolf on a clean render.""" + from v4.p7_verify import _missing_word_spans + + source = "Les voyageurs se regardaient avec une certaine honte." + assert _missing_word_spans(source, source) == [] + assert _missing_word_spans(source, source.upper()) == [] + assert _missing_word_spans(source, source.replace("!", ".") + "!") == [] + + +def test_detects_dropped_sentence(): + """The founding case: a whole sentence absent from the hypothesis must + be reported as one span covering its words.""" + from v4.p7_verify import _missing_word_spans + + source = ( + "Des artilleurs sombres alignés avec des fantassins divers. " + "Tous semblaient accablés et éreintés." + ) + hyp = "Tous semblaient accablés et éreintés." + spans = _missing_word_spans(source, hyp) + assert len(spans) == 1 + assert "artilleurs" in spans[0]["words"] + assert spans[0]["end"] - spans[0]["start"] >= 6 # the 7-word sentence + + +def test_single_word_substitution_produces_halo_span(): + """Known primitive behavior: a substituted word breaks every trigram it + touches, so ONE bad word yields a ~5-word halo span. This is why p7 + votes across three ASRs (_voted_missing_spans) instead of trusting one + hypothesis — the test pins the halo so the vote's job stays visible.""" + from v4.p7_verify import _missing_word_spans + + source = "Il marchait doucement vers la grande maison blanche." + hyp = "Il marchait vite vers la grande maison blanche." + spans = _missing_word_spans(source, hyp) + assert len(spans) == 1 + assert "doucement" in spans[0]["words"] + + +def test_vote_filters_single_asr_noise(): + """A halo present under ONE model only is not an omission: the passage + is spoken, one ASR misheard it. The 2-of-3 vote drops it (the measured + behavior of the A0C bench, #17586).""" + from v4.p7_verify import _voted_missing_spans + + source = "Il marchait doucement vers la grande maison blanche du notaire." + spoken = source # the render spoke it all + misheard = "Il marchait vite vers la grande maison blanche du notaire." + spans_by_model = { + "tiny": _spans(source, misheard), + "large-v3": _spans(source, spoken), + "large-v3-turbo": _spans(source, spoken), + } + assert _voted_missing_spans(spans_by_model) == [] + + +def test_vote_keeps_real_omission(): + """A passage missing under two models is a real omission, with its + supporting models recorded.""" + from v4.p7_verify import _voted_missing_spans + + source = ( + "Des artilleurs sombres alignés avec des fantassins divers. " + "Tous semblaient accablés et éreintés." + ) + full = source + dropped = "Tous semblaient accablés et éreintés." + spans_by_model = { + "tiny": _spans(source, dropped), + "large-v3": _spans(source, dropped), + "large-v3-turbo": _spans(source, full), + } + voted = _voted_missing_spans(spans_by_model) + assert len(voted) == 1 + assert "artilleurs" in voted[0]["words"] + assert voted[0]["absent_sous"] == ["large-v3", "tiny"] + + +def test_vote_clusters_shifted_halos(): + """Halos shift by a word or two between models: spans overlapping by + >=3 words support the same omission even with different boundaries.""" + from v4.p7_verify import _voted_missing_spans + + spans_by_model = { + "tiny": [{"start": 10, "end": 17, "words": "a b c d e f g"}], + "large-v3": [{"start": 11, "end": 18, "words": "b c d e f g h"}], + } + voted = _voted_missing_spans(spans_by_model) + assert len(voted) == 1 + assert voted[0]["absent_sous"] == ["large-v3", "tiny"] + + +def test_normalization_ignores_case_and_punctuation(): + """The alignment normalizes case/punctuation: cosmetic ASR differences + are not omissions.""" + from v4.p7_verify import _missing_word_spans + + source = "Elle répondit: « Non, jamais. »" + hyp = "elle répondit non jamais" + assert _missing_word_spans(source, hyp) == [] + + +def test_inflection_drift_is_known_false_positive(): + """Documented limitation: an inflection drift ('Tout semblait accablé' + vs 'Tous semblaient accablés') breaks every trigram it touches and can + flag a spoken passage. The test pins the behavior so a future change to + the method (stemming?) is a deliberate one.""" + from v4.p7_verify import _missing_word_spans + + source = "Tous semblaient accablés, éreintés, incapables." + hyp = "Tout semblait accablé, éreinté, incapable." + spans = _missing_word_spans(source, hyp) + # Flagged today (inflection breaks the trigrams) — known false positive. + assert spans, "inflection drift is flagged by the current method" + + +def test_empty_or_tiny_reference(): + """References under min_words cannot produce a >=3-word span.""" + from v4.p7_verify import _missing_word_spans + + assert _missing_word_spans("Deux mots", "autre chose") == [] + assert _missing_word_spans("", "hypothèse") == [] + + +def test_multiple_spans_reported_separately(): + """Two disjoint dropped passages yield two spans, not one merged.""" + from v4.p7_verify import _missing_word_spans + + source = ( + "Première phrase complète à garder. Deuxième phrase qui disparaît " + "entièrement du rendu. Troisième phrase conservée aussi. Quatrième " + "passage perdu dans l'audio final." + ) + hyp = "Première phrase complète à garder. Troisième phrase conservée aussi." + spans = _missing_word_spans(source, hyp) + assert len(spans) == 2 + # Words are normalized (lowercase) — the halo pulls 2 words before each + # dropped passage, so assert on containment, not boundaries. + assert "deuxième" in spans[0]["words"] + assert "quatrième" in spans[1]["words"]