fix/367

hazm/normalizer.py

@@ -1,6 +1,7 @@
 """This module contains classes and functions for text normalization."""
 
 import re
+from typing import Final
 
 from hazm.api import NormalizerProtocol
 from hazm.constants import AFFIX_SPACING_PATTERNS
@@ -24,6 +25,10 @@
 class Normalizer(NormalizerProtocol):
     """This class includes functions for text normalization."""
 
+    # Non-left-joining letters (isolated form) in Persian/Arabic
+    # These letters don't connect to the next letter, so ZWNJ is not needed before suffixes
+    _NON_LEFT_JOINING_LETTERS: Final[tuple[str]] = ("ا", "د", "ذ", "ر", "ز", "ژ", "و")
+
     def __init__(
         self,
         correct_spacing: bool = True,
@@ -362,6 +367,7 @@ def token_spacing(self, tokens: list[str]) -> list[str]:
         result: list[str] = []
         for t, token in enumerate(tokens):
             joined = False
+            use_zwnj = True
 
             if result:
                 token_pair = result[-1] + "‌" + token
@@ -380,9 +386,16 @@ def token_spacing(self, tokens: list[str]) -> list[str]:
 
                 elif self._words and token in SUFFIXES and result[-1] in self._words:
                     joined = True
+                    # Only use ZWNJ if the last character is left-joining
+                    # Non-left-joining letters (ا, د, ذ, ر, ز, ژ, و) already create a visual gap
+                    if result[-1] and result[-1][-1] in self._NON_LEFT_JOINING_LETTERS:
+                        use_zwnj = False
 
             if joined:
-                result[-1] = token_pair
+                if use_zwnj:
+                    result[-1] = result[-1] + "‌" + token
+                else:
+                    result[-1] = result[-1] + token
             else:
                 result.append(token)
 

tests/test_normalizer.py

@@ -126,6 +126,8 @@ def test_seperate_mi(self:"TestNormazlier", normalizer, text, expected):
         (["کتاب", "ها"], ["کتاب‌ها"]),
         (["کتاب", "های"], ["کتاب‌های"]),
         (["کتاب", "هایی"], ["کتاب‌هایی"]),
+        # Issue #367: Non-left-joining letters should not have ZWNJ before suffixes
+        (["بندر", "ها"], ["بندرها"]),  # د is non-left-joining
         (["او", "می", "رود"], ["او", "می‌رود"]),
         (["ماه", "می", "سال", "جدید"], ["ماه", "می", "سال", "جدید"]),
         (["اخلال", "گر"], ["اخلال‌گر"]),