Qwen3-TTS PR 5: 1.7B CustomVoice, VoiceDesign and Base (voice clone)

benchmark/base.py

@@ -233,15 +233,40 @@ def get_supported_modalities(self):
 
 
 class Qwen3TTS(Model):
-    """Qwen3-TTS CustomVoice benchmark metadata for native M* requests."""
+    """Qwen3-TTS CustomVoice benchmark metadata (0.6B by default).
+
+    ``/v1/audio/speech`` requests carry the same ``voice`` and ``language``
+    for every engine so the Talker prefill is identical across systems.
+    """
+
+    HF_URL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
 
     def get_hf_url(self):
-        return "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
+        return self.HF_URL
+
+    def get_model_kwargs(self, request_type: RequestType):
+        return {"voice": "vivian", "language": "English"}
 
     def get_supported_modalities(self):
         return {RequestType.T2S}
 
 
+class Qwen3TTS1p7B(Qwen3TTS):
+    HF_URL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
+
+
+class Qwen3TTSVoiceDesign(Qwen3TTS):
+    """VoiceDesign has no built-in speakers; the voice is the instruction."""
+
+    HF_URL = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
+
+    def get_model_kwargs(self, request_type: RequestType):
+        return {
+            "language": "English",
+            "instructions": "A clear, friendly adult female voice with a neutral accent.",
+        }
+
+
 class Pi05(Model):
     """Physical Intelligence Pi0.5 VLA model.
 
@@ -328,6 +353,8 @@ class ModelType(Enum):
     ORPHEUS = "orpheus"
     QWEN3OMNI = "qwen3omni"
     QWEN3TTS = "qwen3_tts"
+    QWEN3TTS_1P7B = "qwen3_tts_1p7b"
+    QWEN3TTS_VOICEDESIGN = "qwen3_tts_voicedesign"
     PI05 = "pi05"
     VJEPA2AC = "vjepa2ac"
     WHISPER_LARGE = "whisper_large"
@@ -342,6 +369,10 @@ def inst(self, **kwargs) -> Model:
             return Qwen3Omni(**kwargs)
         if self == ModelType.QWEN3TTS:
             return Qwen3TTS(**kwargs)
+        if self == ModelType.QWEN3TTS_1P7B:
+            return Qwen3TTS1p7B(**kwargs)
+        if self == ModelType.QWEN3TTS_VOICEDESIGN:
+            return Qwen3TTSVoiceDesign(**kwargs)
         if self == ModelType.PI05:
             return Pi05(**kwargs)
         if self == ModelType.VJEPA2AC:

benchmark/request.py

@@ -14,7 +14,7 @@
 import aiohttp
 import numpy as np
 
-from benchmark.base import Bagel, Model, Orpheus, RequestType, Status
+from benchmark.base import Bagel, Model, Orpheus, Qwen3TTS, RequestType, Status
 from benchmark.utils import _write_wav
 
 
@@ -1494,7 +1494,7 @@ async def send_request(
                 metrics=metrics,
                 additional_model_kwargs=additional_model_kwargs,
             )
-        if req_type.get_output_modalities() == "audio" and isinstance(model, Orpheus):
+        if req_type.get_output_modalities() == "audio" and isinstance(model, (Orpheus, Qwen3TTS)):
             metrics = RequestMetrics(
                 request_id=request_id,
                 type=req_type,

benchmark/tts_report.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Assemble the BENCHMARK_PROTOCOL.md TTS table from ``tts_speech_bench`` / ``tts_wer`` / parity JSONs.
+
+    python -m benchmark.tts_report --results results/2026-09-18 --out results/2026-09-18/REPORT.md
+
+Every ``*_c<N>.json`` written by ``benchmark/tts_speech_bench.py`` becomes one row
+(label, concurrency, TTFA p50/p95, RTF, audio-seconds per second, errors); a
+sibling ``*_c<N>_wer.json`` from ``benchmark/tts_wer.py`` fills the WER column
+and ``parity_*.json`` files from ``test/qwen3-tts/parity_qwen3_tts.py`` become
+the parity table. The markdown is printed and optionally written to ``--out``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+def _load(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def benchmark_rows(results: Path) -> list[str]:
+    reports = [
+        _load(path) for path in sorted(results.glob("*_c[0-9]*.json"))
+        if not path.name.endswith("_wer.json")
+    ]
+    rows = []
+    # one system's rows in concurrency order, systems alphabetically
+    for report in sorted(reports, key=lambda r: (r.get("label") or r["engine"], int(r["concurrency"]))):
+        path = results / f"{report.get('label') or report['engine']}_c{report['concurrency']}.json"
+        med = report["median_over_repeats"]
+        wer_path = path.with_name(path.stem + "_wer.json")
+        wer = f"{_load(wer_path)['wer_percent']:.2f}" if wer_path.is_file() else "n/a"
+        version = report.get("engine_version") or ""
+        label = report.get("label") or report["engine"]
+        rows.append(
+            f"| {label} {version} | {report['concurrency']} | "
+            f"{med['ttfa_p50_ms']:.0f} / {med['ttfa_p95_ms']:.0f} | {med['rtf_mean']:.3f} | "
+            f"{med['audio_s_per_wall_s']:.1f} | {wer} | {report['errors_total']} | "
+            f"{report['num_sentences']} sentences x {report['repeats']} repeats |"
+        )
+    return rows
+
+
+def parity_rows(results: Path) -> list[str]:
+    rows = []
+    for path in sorted(results.glob("parity_*.json")):
+        r = _load(path)
+        clone = r.get("clone") or {}
+        clone_cell = (
+            f"cos {clone['xvector_cosine']:.4f}, codes {clone.get('ref_code_agreement', float('nan')):.3f}"
+            if clone else "-"
+        )
+        def agreement(report: dict) -> str:
+            confident = report.get("confident_agreement")
+            confident_cell = f"{confident:.4f}" if confident is not None else "n/a"
+            rel = report.get("rel_mean_diff", float("nan"))
+            return f"{report['argmax_agreement']:.4f} / {confident_cell} / {rel:.4f}"
+
+        rows.append(
+            f"| {r['repo'].split('/')[-1]} | {re.sub(r'^parity_', '', path.stem)} | {r['frames']} | "
+            f"{agreement(r['talker'])} | {agreement(r['code_predictor'])} | "
+            f"{r['greedy_codes']['identical_frames_before_divergence']}/{r['greedy_codes']['frames_compared']} | "
+            f"{r['codec']['max_abs_diff']:.2e} | {r['audio']['max_abs_diff']:.3f} | {clone_cell} |"
+        )
+    return rows
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--results", required=True, help="directory with the benchmark / parity JSON files")
+    parser.add_argument("--out", default=None)
+    args = parser.parse_args(argv)
+    results = Path(args.results)
+
+    lines = ["## Benchmarks (H100, back to back, warmup excluded, median over repeats)", "",
+             "| System (version) | concurrency | TTFA p50 / p95 ms | RTF | audio-s / s | WER % | errors | notes |",
+             "|---|---|---|---|---|---|---|---|", *benchmark_rows(results), "",
+             "## Parity vs qwen-tts (greedy, bf16 Talker, fp32 codec)", "",
+             "| checkpoint | mode | frames | Talker argmax / confident agreement / rel. logit diff | "
+             "CodePredictor argmax / confident agreement / rel. logit diff | "
+             "identical greedy frames | codec max-abs-diff | greedy audio max-abs-diff | "
+             "clone (x-vector cosine, ref-code agreement) |",
+             "|---|---|---|---|---|---|---|---|---|", *parity_rows(results)]
+    env = results / "environment.txt"
+    if env.is_file():
+        # versions, GPU and clocks; the raw nvidia-smi clock dump that follows is left out
+        summary = env.read_text(encoding="utf-8").split("==============NVSMI LOG")[0].strip()
+        lines += ["", "## Environment", "", "```", summary, "```"]
+    text = "\n".join(lines) + "\n"
+    print(text)
+    if args.out:
+        Path(args.out).write_text(text, encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()