Skip to content

Commit bcbd2a5

Browse files
HenryHenry
authored andcommitted
fix: avoid tracked mojibake literals
1 parent a3c2ef9 commit bcbd2a5

2 files changed

Lines changed: 18 additions & 6 deletions

File tree

backend_api_python/app/professional_report/narrative.py

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,29 @@
77
from typing import Any, Iterable, Mapping
88

99

10-
_MOJIBAKE = ("�", "—", "–", "’", "锟斤拷", "�")
10+
_MOJIBAKE = (
11+
"\u00ef\u00bf\u00bd",
12+
"\u00e2\u20ac\u201d",
13+
"\u00e2\u20ac\u201c",
14+
"\u00e2\u20ac\u2122",
15+
"\u951f\u65a4\u62f7",
16+
"\ufffd",
17+
)
1118

1219

1320
def normalize_report_text(value: Any) -> Any:
1421
"""Normalize Unicode and replace known mojibake without changing numbers."""
1522
if isinstance(value, str):
1623
text = unicodedata.normalize("NFKC", value).replace("\x00", "")
17-
replacements = {"—": "—", "–": "–", "’": "’", "�": ""}
24+
replacements = {
25+
"\u00e2\u20ac\u201d": "—",
26+
"\u00e2\u20ac\u201c": "–",
27+
"\u00e2\u20ac\u2122": "’",
28+
"\u00ef\u00bf\u00bd": "",
29+
}
1830
for broken, fixed in replacements.items():
1931
text = text.replace(broken, fixed)
20-
return text.replace("", "").strip()
32+
return text.replace("\ufffd", "").strip()
2133
if isinstance(value, dict):
2234
return {key: normalize_report_text(item) for key, item in value.items()}
2335
if isinstance(value, list):

backend_api_python/tests/test_professional_report_builder.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def _analysis(payload, evidence_claims=None):
7575
"language": "zh-CN",
7676
"decision": "BUY",
7777
"confidence": 88,
78-
"summary": "趋势改善—但仍需确认�",
78+
"summary": "趋势改善\u00e2\u20ac\u201d但仍需确认\ufffd",
7979
"timeframe": "medium",
8080
"detailed_analysis": {"technical": "动量改善", "fundamental": "数据可用", "sentiment": "中性"},
8181
"scores": {"technical": 68, "fundamental": 60, "sentiment": 52},
@@ -102,8 +102,8 @@ def test_professional_builder_produces_valid_contract_for_supported_markets(mark
102102
assert report["contract_validation"]["valid"] is True
103103
assert report["instrument"]["market"] == market
104104
assert report["claims"][0]["evidence_refs"] == [ref]
105-
assert "—" not in report["executive_summary"]
106-
assert "" not in report["executive_summary"]
105+
assert "\u00e2\u20ac\u201d" not in report["executive_summary"]
106+
assert "\ufffd" not in report["executive_summary"]
107107
assert report["data_quality"]["coverage_ratio"] == 1
108108

109109

0 commit comments

Comments
 (0)