From 6cbc7da2631451b47e5dca288903be68c8d1453b Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: Sat, 5 Sep 2026 05:15:54 +0000
Subject: [PATCH] docs: add reproducible SER benchmark contract

Signed-off-by: zhifu gao <zhifu.gzf@alibaba-inc.com>
---
 README.md                   |   2 +
 README_zh.md                |   2 +
 benchmarks/ser/README.md    |  36 +++++++++++
 benchmarks/ser/evaluate.py  | 121 ++++++++++++++++++++++++++++++++++++
 tests/test_ser_benchmark.py |  61 ++++++++++++++++++
 5 files changed, 222 insertions(+)
 create mode 100644 benchmarks/ser/README.md
 create mode 100644 benchmarks/ser/evaluate.py
 create mode 100644 tests/test_ser_benchmark.py

diff --git a/README.md b/README.md
index 502aca3..fea358f 100644
--- a/README.md
+++ b/README.md
@@ -70,6 +70,8 @@ We compared the performance of multilingual speech recognition between SenseVoic
 
 Due to the current lack of widely-used benchmarks and methods for speech emotion recognition, we conducted evaluations across various metrics on multiple test sets and performed a comprehensive comparison with numerous results from recent benchmarks. The selected test sets encompass data in both Chinese and English, and include multiple styles such as performances, films, and natural conversations. Without finetuning on the target data, SenseVoice was able to achieve and exceed the performance of the current best speech emotion recognition models.
 
+For a reproducible zero-shot CASIA or RAVDESS rerun, use the [SER evaluation contract](./benchmarks/ser/README.md). It reads the raw SenseVoice emotion tag and reports both UA and WA instead of deriving labels from formatted transcription text.
+
 <div align="center">  
 <img src="image/ser_table.png" width="1000" />
 </div>
diff --git a/README_zh.md b/README_zh.md
index a5d00e2..dab20c9 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -71,6 +71,8 @@ SenseVoice 是具有音频理解能力的音频基础模型，包括语音识别
 
 由于目前缺乏被广泛使用的情感识别测试指标和方法，我们在多个测试集的多种指标进行测试，并与近年来 Benchmark 上的多个结果进行了全面的对比。所选取的测试集同时包含中文 / 英文两种语言以及表演、影视剧、自然对话等多种风格的数据，在不进行目标数据微调的前提下，SenseVoice 能够在测试数据上达到和超过目前最佳情感识别模型的效果。
 
+需要复现零训练的 CASIA 或 RAVDESS 结果时，请使用 [SER 评测契约](./benchmarks/ser/README.md)。该脚本直接读取 SenseVoice 原始情感标签，同时输出 UA 和 WA；不要从富文本转写结果中用字符串切分推断情感标签。
+
 <div align="center">  
 <img src="image/ser_table.png" width="1000" />
 </div>
diff --git a/benchmarks/ser/README.md b/benchmarks/ser/README.md
new file mode 100644
index 0000000..d9b4a95
--- /dev/null
+++ b/benchmarks/ser/README.md
@@ -0,0 +1,36 @@
+# Reproducing SenseVoice SER Measurements
+
+The SER table in the project README reports zero-shot results. It is not a
+promise that every dataset mirror, parser, or aggregate metric produces the
+same number. In particular, the CASIA SenseVoiceSmall row is **70.0 UA / 70.0
+WA** for the six-label benchmark protocol shown in the table.
+
+Use `evaluate.py` to make a local evaluation reproducible. It accepts a JSONL
+manifest; every non-empty line must contain an audio path or URL and one of the
+following labels:
+
+```json
+{"audio": "/data/CASIA/angry/example.wav", "label": "angry"}
+```
+
+The accepted canonical labels are `angry`, `fearful`, `happy`, `neutral`,
+`sad`, and `surprised`. Dataset spellings `fear` and `surprise` are normalized
+to `fearful` and `surprised`.
+
+```bash
+python benchmarks/ser/evaluate.py casia.jsonl \
+  --model iic/SenseVoiceSmall --device cuda:0 --output casia-results.json
+```
+
+The evaluator reads the raw `<|EMOTION|>` tag returned by SenseVoice before
+calling rich-text post-processing. Do not infer the label by splitting the
+formatted transcription: tags and display text have different contracts.
+
+`wa` is accuracy over all records. `ua` is the mean recall over labels present
+in the manifest. The JSON result includes per-label recall and a confusion map;
+unknown labels, missing emotion tags, and malformed manifest records fail the
+run instead of being skipped.
+
+CASIA and RAVDESS distributions are controlled by their respective providers.
+Keep the dataset version, manifest, model revision, package versions, and this
+JSON result together when comparing a rerun with the README table.
diff --git a/benchmarks/ser/evaluate.py b/benchmarks/ser/evaluate.py
new file mode 100644
index 0000000..cc29b98
--- /dev/null
+++ b/benchmarks/ser/evaluate.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Evaluate zero-shot SenseVoice emotion predictions from a JSONL manifest."""
+
+import argparse
+import json
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+CANONICAL_LABELS = (
+    "angry",
+    "fearful",
+    "happy",
+    "neutral",
+    "sad",
+    "surprised",
+)
+LABEL_ALIASES = {
+    "angry": "angry",
+    "fear": "fearful",
+    "fearful": "fearful",
+    "happy": "happy",
+    "neutral": "neutral",
+    "sad": "sad",
+    "surprise": "surprised",
+    "surprised": "surprised",
+}
+EMOTION_TAG = re.compile(r"<\|(?P<label>[A-Z_]+)\|>")
+
+
+def normalize_label(label):
+    """Map dataset and model spellings to the six CASIA evaluation labels."""
+    normalized = label.strip().lower()
+    try:
+        return LABEL_ALIASES[normalized]
+    except KeyError as exc:
+        raise ValueError(f"unsupported emotion label: {label!r}") from exc
+
+
+def extract_emotion(raw_text):
+    """Read SenseVoice's raw emotion control tag before text post-processing."""
+    for match in EMOTION_TAG.finditer(raw_text):
+        label = match.group("label").lower()
+        if label in LABEL_ALIASES:
+            return normalize_label(label)
+    raise ValueError(f"SenseVoice output has no supported emotion tag: {raw_text!r}")
+
+
+def compute_metrics(references, predictions):
+    """Return weighted accuracy, unweighted accuracy, and a sparse confusion map."""
+    if not references:
+        raise ValueError("no evaluation records")
+    if len(references) != len(predictions):
+        raise ValueError("reference and prediction counts differ")
+
+    confusion = defaultdict(Counter)
+    totals = Counter()
+    correct = Counter()
+    for reference, prediction in zip(references, predictions):
+        reference = normalize_label(reference)
+        prediction = normalize_label(prediction)
+        totals[reference] += 1
+        confusion[reference][prediction] += 1
+        if reference == prediction:
+            correct[reference] += 1
+
+    recalls = {label: correct[label] / totals[label] for label in totals}
+    return {
+        "records": len(references),
+        "wa": sum(correct.values()) / len(references),
+        "ua": sum(recalls.values()) / len(recalls),
+        "recall_by_label": recalls,
+        "confusion": {label: dict(confusion[label]) for label in sorted(confusion)},
+    }
+
+
+def read_manifest(path):
+    records = []
+    for line_number, line in enumerate(Path(path).read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        record = json.loads(line)
+        if not isinstance(record.get("audio"), str) or not isinstance(record.get("label"), str):
+            raise ValueError(f"manifest line {line_number} needs string audio and label fields")
+        records.append(record)
+    if not records:
+        raise ValueError("manifest contains no records")
+    return records
+
+
+def evaluate(manifest, model_name, device):
+    from funasr import AutoModel
+
+    model = AutoModel(model=model_name, device=device, disable_update=True)
+    references, predictions = [], []
+    for record in read_manifest(manifest):
+        result = model.generate(input=record["audio"], language="auto", use_itn=True)
+        raw_text = result[0]["text"]
+        references.append(record["label"])
+        predictions.append(extract_emotion(raw_text))
+    return compute_metrics(references, predictions)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("manifest", help="JSONL records with audio and label fields")
+    parser.add_argument("--model", default="iic/SenseVoiceSmall")
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    result = evaluate(args.manifest, args.model, args.device)
+    rendered = json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)
+    if args.output:
+        args.output.write_text(rendered + "\n", encoding="utf-8")
+    print(rendered)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/test_ser_benchmark.py b/tests/test_ser_benchmark.py
new file mode 100644
index 0000000..78dad13
--- /dev/null
+++ b/tests/test_ser_benchmark.py
@@ -0,0 +1,61 @@
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+MODULE_PATH = (
+    Path(__file__).resolve().parents[1] / "benchmarks" / "ser" / "evaluate.py"
+)
+
+
+def load_evaluator():
+    spec = importlib.util.spec_from_file_location("ser_evaluate", MODULE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_normalize_label_accepts_casia_aliases():
+    evaluator = load_evaluator()
+
+    assert evaluator.normalize_label("fear") == "fearful"
+    assert evaluator.normalize_label("surprise") == "surprised"
+    assert evaluator.normalize_label("NEUTRAL") == "neutral"
+
+
+def test_extract_emotion_reads_raw_sensevoice_tag():
+    evaluator = load_evaluator()
+
+    assert (
+        evaluator.extract_emotion("<|zh|><|HAPPY|><|Speech|><|withitn|>hello")
+        == "happy"
+    )
+
+
+def test_compute_metrics_reports_ua_wa_and_confusion_without_dropping_classes():
+    evaluator = load_evaluator()
+
+    result = evaluator.compute_metrics(
+        ["angry", "angry", "fearful", "happy"],
+        ["angry", "happy", "fearful", "happy"],
+    )
+
+    assert result["wa"] == pytest.approx(0.75)
+    assert result["ua"] == pytest.approx((0.5 + 1.0 + 1.0) / 3)
+    assert result["confusion"]["angry"] == {"angry": 1, "happy": 1}
+
+
+def test_extract_emotion_rejects_output_without_an_emotion_tag():
+    evaluator = load_evaluator()
+
+    with pytest.raises(ValueError, match="emotion tag"):
+        evaluator.extract_emotion("plain transcript")
+
+
+def test_readmes_link_the_ser_reproduction_contract():
+    root = Path(__file__).resolve().parents[1]
+
+    for readme in (root / "README.md", root / "README_zh.md"):
+        text = readme.read_text(encoding="utf-8")
+        assert "benchmarks/ser/README.md" in text