diff --git a/hazm/normalizer.py b/hazm/normalizer.py index 29051b2b..d49278b4 100644 --- a/hazm/normalizer.py +++ b/hazm/normalizer.py @@ -1,6 +1,7 @@ """This module contains classes and functions for text normalization.""" import re +from typing import Final from hazm.api import NormalizerProtocol from hazm.constants import AFFIX_SPACING_PATTERNS @@ -24,6 +25,10 @@ class Normalizer(NormalizerProtocol): """This class includes functions for text normalization.""" + # Non-left-joining letters (isolated form) in Persian/Arabic + # These letters don't connect to the next letter, so ZWNJ is not needed before suffixes + _NON_LEFT_JOINING_LETTERS: Final[tuple[str]] = ("ا", "د", "ذ", "ر", "ز", "ژ", "و") + def __init__( self, correct_spacing: bool = True, @@ -362,6 +367,7 @@ def token_spacing(self, tokens: list[str]) -> list[str]: result: list[str] = [] for t, token in enumerate(tokens): joined = False + use_zwnj = True if result: token_pair = result[-1] + "‌" + token @@ -380,9 +386,16 @@ def token_spacing(self, tokens: list[str]) -> list[str]: elif self._words and token in SUFFIXES and result[-1] in self._words: joined = True + # Only use ZWNJ if the last character is left-joining + # Non-left-joining letters (ا, د, ذ, ر, ز, ژ, و) already create a visual gap + if result[-1] and result[-1][-1] in self._NON_LEFT_JOINING_LETTERS: + use_zwnj = False if joined: - result[-1] = token_pair + if use_zwnj: + result[-1] = result[-1] + "‌" + token + else: + result[-1] = result[-1] + token else: result.append(token) diff --git a/tests/test_normalizer.py b/tests/test_normalizer.py index 8aba455c..da1a47db 100644 --- a/tests/test_normalizer.py +++ b/tests/test_normalizer.py @@ -126,6 +126,8 @@ def test_seperate_mi(self:"TestNormazlier", normalizer, text, expected): (["کتاب", "ها"], ["کتاب‌ها"]), (["کتاب", "های"], ["کتاب‌های"]), (["کتاب", "هایی"], ["کتاب‌هایی"]), + # Issue #367: Non-left-joining letters should not have ZWNJ before suffixes + (["بندر", "ها"], ["بندرها"]), # د is non-left-joining (["او", "می", "رود"], ["او", "می‌رود"]), (["ماه", "می", "سال", "جدید"], ["ماه", "می", "سال", "جدید"]), (["اخلال", "گر"], ["اخلال‌گر"]),