fix/367

[SepehrRajabi]

درود
وقت بخیر
برای #367 چندتا تغییر دادم و فکر کنم مشکل حل شده باشه.

مشکل این بود که:
نرمالایزر قبل از پسوندهایی مثل «ها» کاراکتر نیم‌فاصله (ZWNJ) می‌ذاشت حتی وقتی حرف قبلی غیرپیوسته بود (مثل ا، د، ر، ز، ژ، و). مثلاً "بندر ها" میشد "بندر‌ها" در حالی که باید "بندرها" باشه.

راه‌ حل:
وقتی می‌خواستیم کلمه و پسوند رو به هم وصل کنیم، اول نگاه می‌کنیم حرف آخرِ کلمه قبلی چیه — اگه از حروف غیرپیوسته بود، بدون ZWNJ می‌چسبونیم؛ اگه پیوسته بود، نیم‌فاصله می‌ذاریم.

خروجی روی ورودی ای که قبلا نتیجه اشتباهی رو میداد:

>>> from hazm import Normalizer
>>> print(repr(Normalizer().normalize("بندر ها")))
'بندرها'

---

و نتایج تست:

pytest -v
======================================= test session starts =======================================
platform darwin -- Python 3.13.13, pytest-9.0.3, pluggy-1.6.0 -- /opt/homebrew/Caskroom/miniconda/base/bin/python
cachedir: .pytest_cache
rootdir: /Users/admin/Documents/hazm-local
configfile: pyproject.toml
plugins: anyio-4.12.1
collected 244 items                                                                               

tests/corpus_readers/test_arman_reader.py::test_sents PASSED                                [  0%]
tests/corpus_readers/test_bijankhan_reader.py::test_sents PASSED                            [  0%]
tests/corpus_readers/test_dadegan_reader.py::test_sents PASSED                              [  1%]
tests/corpus_readers/test_dadegan_reader.py::test_trees PASSED                              [  1%]
tests/corpus_readers/test_degarbayan_reader.py::test_pairs PASSED                           [  2%]
tests/corpus_readers/test_faspell_reader.py::test_main_entries PASSED                       [  2%]
tests/corpus_readers/test_faspell_reader.py::test_ocr_entries PASSED                        [  2%]
tests/corpus_readers/test_hamshahri_reader.py::test_docs PASSED                             [  3%]
tests/corpus_readers/test_mirastext_reader.py::test_sents PASSED                            [  3%]
tests/corpus_readers/test_mizan_reader.py::test_english_sentences PASSED                    [  4%]
tests/corpus_readers/test_mizan_reader.py::test_persian_sentences PASSED                    [  4%]
tests/corpus_readers/test_mizan_reader.py::test_english_persian_sentences PASSED            [  4%]
tests/corpus_readers/test_naab_reader.py::test_sents PASSED                                 [  5%]
tests/corpus_readers/test_ner_reader.py::test_sents PASSED                                  [  5%]
tests/corpus_readers/test_persica_reader.py::test_docs PASSED                               [  6%]
tests/corpus_readers/test_persica_reader.py::test_texts PASSED                              [  6%]
tests/corpus_readers/test_peykare_reader.py::test_sents PASSED                                                                         [  6%]
tests/corpus_readers/test_pn_summary_reader.py::test_docs PASSED                                                                       [  7%]
tests/corpus_readers/test_quran_reader.py::test_parts PASSED                                                                           [  7%]
tests/corpus_readers/test_quran_reader.py::test_words PASSED                                                                           [  8%]
tests/corpus_readers/test_sentipers_reader.py::test_comments PASSED                                                                    [  8%]
tests/corpus_readers/test_tnews_reader.py::test_docs PASSED                                                                            [  9%]
tests/corpus_readers/test_tnews_reader.py::test_texts PASSED                                                                           [  9%]
tests/corpus_readers/test_treebank_reader.py::test_trees PASSED                                                                        [  9%]
tests/corpus_readers/test_treebank_reader.py::test_sents PASSED                                                                        [ 10%]
tests/corpus_readers/test_treebank_reader.py::test_chunked_trees PASSED                                                                [ 10%]
tests/corpus_readers/test_universal_dadegan_reader.py::test_sents PASSED                                                               [ 11%]
tests/corpus_readers/test_universal_dadegan_reader.py::test_trees PASSED                                                               [ 11%]
tests/corpus_readers/test_wikipedia_reader.py::test_docs XFAIL (StopIteration issue on
Python 3.11)                                                                                                                           [ 11%]
tests/corpus_readers/test_wikipedia_reader.py::test_texts XFAIL (StopIteration issue on
Python 3.11)                                                                                                                           [ 12%]
tests/test_chunker.py::TestChunker::test_parse PASSED                                                                                  [ 12%]
tests/test_chunker.py::TestChunker::test_data_maker PASSED                                                                             [ 13%]
tests/test_chunker.py::TestChunker::test_evaluate PASSED                                                                               [ 13%]
tests/test_chunker.py::TestRuleBasedChunker::test_parse PASSED                                                                         [ 13%]
tests/test_dependency_parser.py::TestDependencyParser::test_parse PASSED                                                               [ 14%]
tests/test_embedding.py::TestWordEmbedding::test_doesnt_match PASSED                                                                   [ 14%]
tests/test_embedding.py::TestWordEmbedding::test_similarity PASSED                                                                     [ 15%]
tests/test_embedding.py::TestWordEmbedding::test_nearest_words FAILED                                                                  [ 15%]
tests/test_embedding.py::TestWordEmbedding::test_get_normal_vector PASSED                                                              [ 15%]
tests/test_embedding.py::TestWordEmbedding::test_get_vocabs PASSED                                                                     [ 16%]
tests/test_embedding.py::TestWordEmbedding::test_get_vocab_to_index PASSED                                                             [ 16%]
tests/test_embedding.py::TestWordEmbedding::test_get_vectors PASSED                                                                    [ 17%]
tests/test_embedding.py::TestWordEmbedding::test_get_vector_size PASSED                                                                [ 17%]
tests/test_embedding.py::TestSentEmbedding::test_get_sentence_vector PASSED                                                            [ 18%]
tests/test_embedding.py::TestSentEmbedding::test_similarity PASSED                                                                     [ 18%]
tests/test_embedding.py::TestSentEmbedding::test_get_vector_size PASSED                                                                [ 18%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize[\u06a9\u062a\u0627\u0628\u200c\u0647\u0627-\u06a9\u062a\u0627\u0628] PASSED   [ 19%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize[\u0622\u062a\u0634\u0641\u0634\u0627\u0646-\u0622\u062a\u0634\u0641\u0634\u0627\u0646] PASSED [ 19%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize[\u0645\u06cc\u200c\u0631\u0648\u0645-\u0631\u0641\u062a#\u0631\u0648] PASSED  [ 20%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize[\u06af\u0641\u062a\u0647_\u0634\u062f\u0647_\u0627\u0633\u062a-\u06af\u0641\u062a#\u06af\u0648] PASSED [ 20%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize[\u0646\u0686\u0634\u06cc\u062f\u0647_\u0627\u0633\u062a-\u0686\u0634\u06cc\u062f#\u0686\u0634] PASSED [ 20%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize_with_pos[\u0645\u0631\u062f\u0645-N-\u0645\u0631\u062f\u0645] PASSED          [ 21%]
tests/test_lemmatizer.py::TestLemmatizer::test_lemmatize_with_pos[\u0627\u062c\u062a\u0645\u0627\u0639\u06cc-AJ-\u0627\u062c\u062a\u0645\u0627\u0639\u06cc] PASSED [ 21%]
tests/test_lemmatizer.py::TestConjugation::test_perfective_past PASSED                                                                 [ 22%]
tests/test_lemmatizer.py::TestConjugation::test_negative_perfective_past PASSED                                                        [ 22%]
tests/test_lemmatizer.py::TestConjugation::test_passive_perfective_past PASSED                                                         [ 22%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_perfective_past PASSED                                                [ 23%]
tests/test_lemmatizer.py::TestConjugation::test_imperfective_past PASSED                                                               [ 23%]
tests/test_lemmatizer.py::TestConjugation::test_negative_imperfective_past PASSED                                                      [ 24%]
tests/test_lemmatizer.py::TestConjugation::test_passive_imperfective_past PASSED                                                       [ 24%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_imperfective_past PASSED                                              [ 25%]
tests/test_lemmatizer.py::TestConjugation::test_past_progresive PASSED                                                                 [ 25%]
tests/test_lemmatizer.py::TestConjugation::test_passive_past_progresive PASSED                                                         [ 25%]
tests/test_lemmatizer.py::TestConjugation::test_present_perfect PASSED                                                                 [ 26%]
tests/test_lemmatizer.py::TestConjugation::test_negative_present_perfect PASSED                                                        [ 26%]
tests/test_lemmatizer.py::TestConjugation::test_subjunctive_present_perfect PASSED                                                     [ 27%]
tests/test_lemmatizer.py::TestConjugation::test_negative_subjunctive_present_perfect PASSED                                            [ 27%]
tests/test_lemmatizer.py::TestConjugation::test_grammatical_present_perfect PASSED                                                     [ 27%]
tests/test_lemmatizer.py::TestConjugation::test_negative_grammatical_present_perfect PASSED                                            [ 28%]
tests/test_lemmatizer.py::TestConjugation::test_passive_present_perfect PASSED                                                         [ 28%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_present_perfect PASSED                                                [ 29%]
tests/test_lemmatizer.py::TestConjugation::test_passive_subjunctive_present_perfect PASSED                                             [ 29%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_subjunctive_present_perfect PASSED                                    [ 29%]
tests/test_lemmatizer.py::TestConjugation::test_passive_grammatical_present_perfect PASSED                                             [ 30%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_grammatical_present_perfect PASSED                                    [ 30%]
tests/test_lemmatizer.py::TestConjugation::test_imperfective_present_perfect PASSED                                                    [ 31%]
tests/test_lemmatizer.py::TestConjugation::test_negative_imperfective_present_perfect PASSED                                           [ 31%]
tests/test_lemmatizer.py::TestConjugation::test_subjunctive_imperfective_present_perfect PASSED                                        [ 31%]
tests/test_lemmatizer.py::TestConjugation::test_negative_subjunctive_imperfective_present_perfect PASSED                               [ 32%]
tests/test_lemmatizer.py::TestConjugation::test_passive_imperfective_present_perfect PASSED                                            [ 32%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_imperfective_present_perfect PASSED                                   [ 33%]
tests/test_lemmatizer.py::TestConjugation::test_passive_subjunctive_imperfective_present_perfect PASSED                                [ 33%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_subjunctive_imperfective_present_perfect PASSED                       [ 34%]
tests/test_lemmatizer.py::TestConjugation::test_present_perfect_progressive PASSED                                                     [ 34%]
tests/test_lemmatizer.py::TestConjugation::test_passive_present_perfect_progressive PASSED                                             [ 34%]
tests/test_lemmatizer.py::TestConjugation::test_past_precedent PASSED                                                                  [ 35%]
tests/test_lemmatizer.py::TestConjugation::test_negative_past_precedent PASSED                                                         [ 35%]
tests/test_lemmatizer.py::TestConjugation::test_passive_past_precedent PASSED                                                          [ 36%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_past_precedent PASSED                                                 [ 36%]
tests/test_lemmatizer.py::TestConjugation::test_imperfective_past_precedent PASSED                                                     [ 36%]
tests/test_lemmatizer.py::TestConjugation::test_negative_imperfective_past_precedent PASSED                                            [ 37%]
tests/test_lemmatizer.py::TestConjugation::test_passive_imperfective_past_precedent PASSED                                             [ 37%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_imperfective_past_precedent PASSED                                    [ 38%]
tests/test_lemmatizer.py::TestConjugation::test_past_precedent_progressive PASSED                                                      [ 38%]
tests/test_lemmatizer.py::TestConjugation::test_passive_past_precedent_progressive PASSED                                              [ 38%]
tests/test_lemmatizer.py::TestConjugation::test_past_precedent_perfect PASSED                                                          [ 39%]
tests/test_lemmatizer.py::TestConjugation::test_negative_past_precedent_perfect PASSED                                                 [ 39%]
tests/test_lemmatizer.py::TestConjugation::test_subjunctive_past_precedent_perfect PASSED                                              [ 40%]
tests/test_lemmatizer.py::TestConjugation::test_negative_subjunctive_past_precedent_perfect PASSED                                     [ 40%]
tests/test_lemmatizer.py::TestConjugation::test_grammatical_past_precedent_perfect PASSED                                              [ 40%]
tests/test_lemmatizer.py::TestConjugation::test_negative_grammatical_past_precedent_perfect PASSED                                     [ 41%]
tests/test_lemmatizer.py::TestConjugation::test_passive_past_precedent_perfect PASSED                                                  [ 41%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_past_precedent_perfect PASSED                                         [ 42%]
tests/test_lemmatizer.py::TestConjugation::test_passive_subjunctive_past_precedent_perfect PASSED                                      [ 42%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_subjunctive_past_precedent_perfect PASSED                             [ 43%]
tests/test_lemmatizer.py::TestConjugation::test_passive_grammatical_past_precedent_perfect PASSED                                      [ 43%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_grammatical_past_precedent_perfect PASSED                             [ 43%]
tests/test_lemmatizer.py::TestConjugation::test_imperfective_past_precedent_perfect PASSED                                             [ 44%]
tests/test_lemmatizer.py::TestConjugation::test_negative_imperfective_past_precedent_perfect PASSED                                    [ 44%]
tests/test_lemmatizer.py::TestConjugation::test_subjunctive_imperfective_past_precedent_perfect PASSED                                 [ 45%]
tests/test_lemmatizer.py::TestConjugation::test_negative_subjunctive_imperfective_past_precedent_perfect PASSED                        [ 45%]
tests/test_lemmatizer.py::TestConjugation::test_passive_imperfective_past_precedent_perfect PASSED                                     [ 45%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_imperfective_past_precedent_perfect PASSED                            [ 46%]
tests/test_lemmatizer.py::TestConjugation::test_passive_subjunctive_imperfective_past_precedent_perfect PASSED                         [ 46%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_subjunctive_imperfective_past_precedent_perfect PASSED                [ 47%]
tests/test_lemmatizer.py::TestConjugation::test_past_precedent_perfect_progressive PASSED                                              [ 47%]
tests/test_lemmatizer.py::TestConjugation::test_passive_past_precedent_perfect_progressive PASSED                                      [ 47%]
tests/test_lemmatizer.py::TestConjugation::test_perfective_present PASSED                                                              [ 48%]
tests/test_lemmatizer.py::TestConjugation::test_negative_perfective_present PASSED                                                     [ 48%]
tests/test_lemmatizer.py::TestConjugation::test_subjunctive_perfective_present PASSED                                                  [ 49%]
tests/test_lemmatizer.py::TestConjugation::test_negative_subjunctive_perfective_present PASSED                                         [ 49%]
tests/test_lemmatizer.py::TestConjugation::test_grammatical_perfective_present PASSED                                                  [ 50%]
tests/test_lemmatizer.py::TestConjugation::test_negative_grammatical_perfective_present PASSED                                         [ 50%]
tests/test_lemmatizer.py::TestConjugation::test_passive_perfective_present PASSED                                                      [ 50%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_perfective_present PASSED                                             [ 51%]
tests/test_lemmatizer.py::TestConjugation::test_passive_subjunctive_perfective_present PASSED                                          [ 51%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_subjunctive_perfective_present PASSED                                 [ 52%]
tests/test_lemmatizer.py::TestConjugation::test_passive_grammatical_perfective_present PASSED                                          [ 52%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_grammatical_perfective_present PASSED                                 [ 52%]
tests/test_lemmatizer.py::TestConjugation::test_imperfective_present PASSED                                                            [ 53%]
tests/test_lemmatizer.py::TestConjugation::test_negative_imperfective_present PASSED                                                   [ 53%]
tests/test_lemmatizer.py::TestConjugation::test_passive_imperfective_present PASSED                                                    [ 54%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_imperfective_present PASSED                                           [ 54%]
tests/test_lemmatizer.py::TestConjugation::test_present_progressive PASSED                                                             [ 54%]
tests/test_lemmatizer.py::TestConjugation::test_passive_present_progressive PASSED                                                     [ 55%]
tests/test_lemmatizer.py::TestConjugation::test_perfective_future PASSED                                                               [ 55%]
tests/test_lemmatizer.py::TestConjugation::test_negative_perfective_future PASSED                                                      [ 56%]
tests/test_lemmatizer.py::TestConjugation::test_passive_perfective_future PASSED                                                       [ 56%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_perfective_future PASSED                                              [ 56%]
tests/test_lemmatizer.py::TestConjugation::test_imperfective_future PASSED                                                             [ 57%]
tests/test_lemmatizer.py::TestConjugation::test_negative_imperfective_future PASSED                                                    [ 57%]
tests/test_lemmatizer.py::TestConjugation::test_passive_imperfective_future PASSED                                                     [ 58%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_imperfective_future PASSED                                            [ 58%]
tests/test_lemmatizer.py::TestConjugation::test_future_precedent PASSED                                                                [ 59%]
tests/test_lemmatizer.py::TestConjugation::test_negative_future_precedent PASSED                                                       [ 59%]
tests/test_lemmatizer.py::TestConjugation::test_passive_future_precedent PASSED                                                        [ 59%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_future_precedent PASSED                                               [ 60%]
tests/test_lemmatizer.py::TestConjugation::test_future_precedent_imperfective PASSED                                                   [ 60%]
tests/test_lemmatizer.py::TestConjugation::test_negative_future_precedent_imperfective PASSED                                          [ 61%]
tests/test_lemmatizer.py::TestConjugation::test_passive_future_precedent_imperfective PASSED                                           [ 61%]
tests/test_lemmatizer.py::TestConjugation::test_negative_passive_future_precedent_imperfective PASSED                                  [ 61%]
tests/test_normalizer.py::TestNormazlier::test_normalize[\u0627\u0650\u0639\u0644\u0627\u0645\u0645\u0645\u0645\u0645 \u06a9\u064e\u0631\u062f : \xab \u0632\u0645\u06cc\u0646 \u0644\u0631\u0632\u0647 \u0627\u06cc \u0628\u0647 \u0628\u064f\u0632\u0631\u06af\u06cc\u0650 6 \u062f\u0647\u0645 \u0631\u06cc\u0634\u062a\u0631 ...\xbb-\u0627\u0639\u0644\u0627\u0645 \u06a9\u0631\u062f: \xab\u0632\u0645\u06cc\u0646\u200c\u0644\u0631\u0632\u0647\u200c\u0627\u06cc \u0628\u0647 \u0628\u0632\u0631\u06af\u06cc \u06f6 \u062f\u0647\u0645 \u0631\u06cc\u0634\u062a\u0631 \u2026\xbb] PASSED [ 62%]
tests/test_normalizer.py::TestNormazlier::test_normalize[  -] PASSED                                                                   [ 62%]
tests/test_normalizer.py::TestNormazlier::test_normalize[-] PASSED                                                                     [ 63%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0633\u0644\u0627\u0645    \u062f\u0646\u06cc\u0627-\u0633\u0644\u0627\u0645 \u062f\u0646\u06cc\u0627] PASSED [ 63%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[     \u0633\u0644\u0627\u0645-\u0633\u0644\u0627\u0645] PASSED          [ 63%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0633\u0644\u0627\u0645     -\u0633\u0644\u0627\u0645] PASSED          [ 64%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0645\u0633\u0627\u0641\u062a \u06f9\u06a9\u06cc\u0644\u0648\u0645\u062a\u0631-\u0645\u0633\u0627\u0641\u062a \u06f9 \u06a9\u06cc\u0644\u0648\u0645\u062a\u0631] PASSED [ 64%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0645\u0633\u0627\u0641\u062a\u06f9 \u06a9\u06cc\u0644\u0648\u0645\u062a\u0631-\u0645\u0633\u0627\u0641\u062a \u06f9 \u06a9\u06cc\u0644\u0648\u0645\u062a\u0631] PASSED [ 65%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u06a9\u0627\u0631\u0648\u0627\u0646\u200c\u200c\u0633\u0631\u0627-\u06a9\u0627\u0631\u0648\u0627\u0646\u200c\u0633\u0631\u0627] PASSED [ 65%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0633\u0644\u0627\u0645 \u200c\u200c\u062f\u0646\u06cc\u0627-\u0633\u0644\u0627\u0645 \u062f\u0646\u06cc\u0627] PASSED [ 65%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0633\u0644\u0627\u0645\u200c\u200c \u062f\u0646\u06cc\u0627-\u0633\u0644\u0627\u0645 \u062f\u0646\u06cc\u0627] PASSED [ 66%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u200c\u200c\u06a9\u0627\u0631\u0648\u0627\u0646\u200c\u0633\u0631\u0627-\u06a9\u0627\u0631\u0648\u0627\u0646\u200c\u0633\u0631\u0627] PASSED [ 66%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u06a9\u0627\u0631\u0648\u0627\u0646\u200c\u0633\u0631\u0627\u200c\u200c-\u06a9\u0627\u0631\u0648\u0627\u0646\u200c\u0633\u0631\u0627] PASSED [ 67%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0633\u0640\u0640\u0644\u0627\u0645-\u0633\u0644\u0627\u0645] PASSED   [ 67%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u062c\u0645\u0639\u0647\u0647\u0627 \u0645\u0637\u0627\u0644\u0639\u0647 \u06a9\u0646\u06cc\u062f-\u062c\u0645\u0639\u0647\u200c\u0647\u0627 \u0645\u0637\u0627\u0644\u0639\u0647 \u06a9\u0646\u06cc\u062f] PASSED [ 68%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[\u0644\u062d\u0638\u0647 \u0647\u0627\u06cc\u06cc \u0633\u062e\u062a-\u0644\u062d\u0638\u0647\u200c\u0647\u0627\u06cc\u06cc \u0633\u062e\u062a] PASSED [ 68%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[   (\u0633\u0644\u0627\u0645)-(\u0633\u0644\u0627\u0645)] PASSED        [ 68%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[(\u0633\u0644\u0627\u0645)   -(\u0633\u0644\u0627\u0645)] PASSED        [ 69%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[  (\u0633\u0644\u0627\u0645)   -(\u0633\u0644\u0627\u0645)] PASSED      [ 69%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[   -] PASSED                                                            [ 70%]
tests/test_normalizer.py::TestNormazlier::test_correct_spacing[-] PASSED                                                               [ 70%]
tests/test_normalizer.py::TestNormazlier::test_remove_diacritics[\u062d\u064e\u0630\u0641\u0650 \u0627\u0650\u0639\u0631\u0627\u0628-\u062d\u0630\u0641 \u0627\u0639\u0631\u0627\u0628] PASSED [ 70%]
tests/test_normalizer.py::TestNormazlier::test_remove_diacritics[\u0622\u0645\u062f\u0646\u062f-\u0622\u0645\u062f\u0646\u062f] PASSED [ 71%]
tests/test_normalizer.py::TestNormazlier::test_remove_diacritics[\u0645\u062a\u0646 \u0628\u062f\u0648\u0646 \u0627\u0639\u0631\u0627\u0628-\u0645\u062a\u0646 \u0628\u062f\u0648\u0646 \u0627\u0639\u0631\u0627\u0628] PASSED [ 71%]
tests/test_normalizer.py::TestNormazlier::test_remove_diacritics[  -  ] PASSED                                                         [ 72%]
tests/test_normalizer.py::TestNormazlier::test_remove_diacritics[-] PASSED                                                             [ 72%]
tests/test_normalizer.py::TestNormazlier::test_remove_specials_chars[\u067e\u06cc\u0627\u0645\u0628\u0631 \u0627\u06a9\u0631\u0645 \ufdfa-\u067e\u06cc\u0627\u0645\u0628\u0631 \u0627\u06a9\u0631\u0645 ] PASSED [ 72%]
tests/test_normalizer.py::TestNormazlier::test_remove_specials_chars[\u0633\u0644\u0627\u0645-\u0633\u0644\u0627\u0645] PASSED         [ 73%]
tests/test_normalizer.py::TestNormazlier::test_remove_specials_chars[-] PASSED                                                         [ 73%]
tests/test_normalizer.py::TestNormazlier::test_decrease_repeated_chars[\u0633\u0644\u0627\u0645\u0645\u0645 \u062f\u0648\u0633\u062a\u0627\u0646-\u0633\u0644\u0627\u0645 \u062f\u0648\u0633\u062a\u0627\u0646] PASSED [ 74%]
tests/test_normalizer.py::TestNormazlier::test_decrease_repeated_chars[\u0633\u0644\u0627\u0645\u0645 \u0631\u0627 \u0628\u0631\u0633\u0627\u0646-\u0633\u0644\u0627\u0645\u0645 \u0631\u0627 \u0628\u0631\u0633\u0627\u0646] PASSED [ 74%]
tests/test_normalizer.py::TestNormazlier::test_decrease_repeated_chars[\u0633\u0644\u0627\u0645\u0645\u0645 \u0631\u0627 \u0628\u0631\u0633\u0627\u0646-\u0633\u0644\u0627\u0645\u0645 \u0631\u0627 \u0628\u0631\u0633\u0627\u0646] XFAIL [ 75%]
tests/test_normalizer.py::TestNormazlier::test_decrease_repeated_chars[\u0633\u0644\u0627\u0645\u0645 \u062f\u0648\u0633\u062a\u0627\u0646-\u0633\u0644\u0627\u0645 \u062f\u0648\u0633\u062a\u0627\u0646] XFAIL [ 75%]
tests/test_normalizer.py::TestNormazlier::test_persian_style["\u0646\u0642\u0644\u200c\u0642\u0648\u0644"-\xab\u0646\u0642\u0644\u200c\u0642\u0648\u0644\xbb] PASSED [ 75%]
tests/test_normalizer.py::TestNormazlier::test_persian_style[\u0648...-\u0648 \u2026] PASSED                                           [ 76%]
tests/test_normalizer.py::TestNormazlier::test_persian_style[10.450-10\u066b450] PASSED                                                [ 76%]
tests/test_normalizer.py::TestNormazlier::test_persian_style[\u0633\u0644\u0627\u0645-\u0633\u0644\u0627\u0645] PASSED                 [ 77%]
tests/test_normalizer.py::TestNormazlier::test_persian_style[  -  ] PASSED                                                             [ 77%]
tests/test_normalizer.py::TestNormazlier::test_persian_style[-] PASSED                                                                 [ 77%]
tests/test_normalizer.py::TestNormazlier::test_persian_number[\u0633\u0627\u0639\u062a 18-\u0633\u0627\u0639\u062a \u06f1\u06f8] PASSED [ 78%]
tests/test_normalizer.py::TestNormazlier::test_persian_number[\u0633\u0627\u0639\u062a \u06f1\u06f8-\u0633\u0627\u0639\u062a \u06f1\u06f8] PASSED [ 78%]
tests/test_normalizer.py::TestNormazlier::test_persian_number[  -  ] PASSED                                                            [ 79%]
tests/test_normalizer.py::TestNormazlier::test_persian_number[-] PASSED                                                                [ 79%]
tests/test_normalizer.py::TestNormazlier::test_unicodes_replacement[\ufdfd-\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u0627\u0644\u0631\u062d\u0645\u0646 \u0627\u0644\u0631\u062d\u06cc\u0645] PASSED [ 79%]
tests/test_normalizer.py::TestNormazlier::test_unicodes_replacement[  -  ] PASSED                                                      [ 80%]
tests/test_normalizer.py::TestNormazlier::test_unicodes_replacement[-] PASSED                                                          [ 80%]
tests/test_normalizer.py::TestNormazlier::test_seperate_mi[\u0646\u0645\u06cc\u062f\u0627\u0646\u0645 \u0686\u0647 \u0645\u06cc\u06af\u0641\u062a-\u0646\u0645\u06cc\u200c\u062f\u0627\u0646\u0645 \u0686\u0647 \u0645\u06cc\u200c\u06af\u0641\u062a] PASSED [ 81%]
tests/test_normalizer.py::TestNormazlier::test_seperate_mi[\u0645\u06cc\u0632-\u0645\u06cc\u0632] PASSED                               [ 81%]
tests/test_normalizer.py::TestNormazlier::test_seperate_mi[\u0645\u06cc\u0627\u0646\u0647-\u0645\u06cc\u0627\u0646\u0647] PASSED       [ 81%]
tests/test_normalizer.py::TestNormazlier::test_seperate_mi[\u0645\u0627\u0647\u0627\u0646-\u0645\u0627\u0647\u0627\u0646] PASSED       [ 82%]
tests/test_normalizer.py::TestNormazlier::test_seperate_mi[  -  ] PASSED                                                               [ 82%]
tests/test_normalizer.py::TestNormazlier::test_seperate_mi[-] PASSED                                                                   [ 83%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text0-expected0] PASSED                                                   [ 83%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text1-expected1] PASSED                                                   [ 84%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text2-expected2] PASSED                                                   [ 84%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text3-expected3] PASSED                                                   [ 84%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text4-expected4] PASSED                                                   [ 85%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text5-expected5] PASSED                                                   [ 85%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text6-expected6] PASSED                                                   [ 86%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text7-expected7] PASSED                                                   [ 86%]
tests/test_normalizer.py::TestNormazlier::test_token_spacing[text8-expected8] PASSED                                                   [ 86%]
tests/test_pos_tagger.py::TestPOSTagger::test_data_maker PASSED                                                                        [ 87%]
tests/test_pos_tagger.py::TestPOSTagger::test_data_maker_universal PASSED                                                              [ 87%]
tests/test_pos_tagger.py::TestPOSTagger::test_tag_sents PASSED                                                                         [ 88%]
tests/test_pos_tagger.py::TestPOSTagger::test_tag_sents_universal PASSED                                                               [ 88%]
tests/test_sentence_tokenizer.py::TestSentenceTokenizer::test_sentence_tokenizer PASSED                                                [ 88%]
tests/test_stemmer.py::TestStemmer::test_stem[\u06a9\u062a\u0627\u0628\u06cc-\u06a9\u062a\u0627\u0628] PASSED                          [ 89%]
tests/test_stemmer.py::TestStemmer::test_stem[\u06a9\u062a\u0627\u0628\u200c\u0647\u0627-\u06a9\u062a\u0627\u0628] PASSED              [ 89%]
tests/test_stemmer.py::TestStemmer::test_stem[\u06a9\u062a\u0627\u0628\u200c\u0647\u0627\u06cc\u06cc-\u06a9\u062a\u0627\u0628] PASSED  [ 90%]
tests/test_stemmer.py::TestStemmer::test_stem[\u06a9\u062a\u0627\u0628\u0647\u0627\u06cc\u0634\u0627\u0646-\u06a9\u062a\u0627\u0628] PASSED [ 90%]
tests/test_stemmer.py::TestStemmer::test_stem[\u0627\u0646\u062f\u06cc\u0634\u0647\u200c\u0627\u0634-\u0627\u0646\u062f\u06cc\u0634\u0647] PASSED [ 90%]
tests/test_stemmer.py::TestStemmer::test_stem[\u062e\u0627\u0646\u0647\u0654-\u062e\u0627\u0646\u0647] PASSED                          [ 91%]
tests/test_stemmer.py::TestStemmer::test_stem[\u0642\u062f\u0645\u0634-\u0642\u062f\u0645] PASSED                                      [ 91%]
tests/test_stemmer.py::TestStemmer::test_stem[  -  ] PASSED                                                                            [ 92%]
tests/test_stemmer.py::TestStemmer::test_stem[-] PASSED                                                                                [ 92%]
tests/test_token_splitter.py::TestTokenSplitter::test_split_token_words[\u0635\u062f\u0627\u0648\u0633\u06cc\u0645\u0627\u062c\u0645\u0647\u0648\u0631\u06cc-expected0] PASSED [ 93%]
tests/test_token_splitter.py::TestTokenSplitter::test_split_token_words[\u0635\u062f\u0627\u0648-expected1] PASSED                     [ 93%]
tests/test_token_splitter.py::TestTokenSplitter::test_split_token_words[\u062f\u0627\u0633\u062a\u0627\u0646\u200c\u0633\u0631\u0627-expected2] PASSED [ 93%]
tests/test_token_splitter.py::TestTokenSplitter::test_split_token_words[\u062f\u0633\u062a\u0627\u0646\u200c\u0633\u0631\u0627-expected3] PASSED [ 94%]
tests/test_token_splitter.py::TestTokenSplitter::test_split_token_words[-expected4] PASSED                                             [ 94%]
tests/test_utils.py::test_words_list PASSED                                                                                            [ 95%]
tests/test_utils.py::test_stopwords_list PASSED                                                                                        [ 95%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_tokenize_simple_sentence PASSED                                                  [ 95%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_tokenize_when_join_verb_parts_is_false PASSED                                    [ 96%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_tokenize_when_join_verb_parts_is_false_and_replace_links_is_true PASSED          [ 96%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_tokenize_when_join_verb_parts_is_false_and_replace_ids_and_replace_numbers_is_true PASSED [ 97%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_tokenize_when_join_verb_parts_is_false_and_separate_emoji_is_true PASSED         [ 97%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_join_verb_parts[words0-expected0] PASSED                                         [ 97%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_join_verb_parts[words1-expected1] PASSED                                         [ 98%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_join_verb_parts[words2-expected2] PASSED                                         [ 98%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_join_verb_parts[words3-expected3] PASSED                                         [ 99%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_join_verb_parts[words4-expected4] PASSED                                         [ 99%]
tests/test_word_tokenizer.py::TestWordTokenizer::test_join_verb_parts[words5-expected5] PASSED                                         [100%]

================================================================== FAILURES ==================================================================
____________________________________________________ TestWordEmbedding.test_nearest_words ____________________________________________________

self = <tests.test_embedding.TestWordEmbedding object at 0x15e092520>, word_embedding = <hazm.embedding.WordEmbedding object at 0x170bf8590>

    def test_nearest_words(self: "TestWordEmbedding", word_embedding):
>       assert word_embedding.nearest_words("ایران", topn = 5) == [("تاریخی.", 0.9997861385345459), ("نام\u200cگذاری", 0.9993206858634949), ("پناه\u200cجو", 0.9992603063583374), ("قرارگرفت", 0.9990856647491455), ("امدادگری", 0.998709499835968)]
E       AssertionError: assert [('تاریخی.', ...094402313232)] == [('تاریخی.', ...709499835968)]
E         
E         At index 1 diff: ('نام\u200cگذاری', 0.9993207454681396) != ('نام\u200cگذاری', 0.9993206858634949)
E         
E         Full diff:
E           [
E               (
E                   'تاریخی.',...
E         
E         ...Full output truncated (22 lines hidden), use '-vv' to show

tests/test_embedding.py:17: AssertionError
========================================================== short test summary info ===========================================================
FAILED tests/test_embedding.py::TestWordEmbedding::test_nearest_words - AssertionError: assert [('تاریخی.', ...094402313232)] == [('تاریخی.', ...709499835968)]
================================================= 1 failed, 239 passed, 4 xfailed in 13.67s ==================================================