Improve search by ignoring accents and use primary language for terms

lib/decidim/term_customizer/translation_directory.rb

@@ -17,12 +17,23 @@ def translations
         @translations ||= TranslationStore.new(backend_translations)
       end
 
+      # Additional languages may be incomplete, so searches also include the
+      # canonical English source translations as a fallback to improve coverage.
+      # In Decidim, English is the upstream source locale and the only locale
+      # guaranteed to contain the full translation key set.
+      def canonical_source_terms
+        @canonical_source_terms ||= TranslationStore.new(all_translations[:en])
+      end
+
       def translations_search(search)
-        translations_by_key(search).merge(translations_by_term(search))
+        merge_search_results(
+          translations.by_key(search).merge(translations.by_term(search)),
+          canonical_source_terms.by_key(search).merge(canonical_source_terms.by_term(search))
+        )
       end
 
-      def translations_by_key(search) # rubocop:disable Rails/Delegate
-        translations.by_key(search)
+      def translations_by_key(search)
+        merge_search_results(translations.by_key(search), canonical_source_terms.by_key(search))
       end
 
       def translations_by_term(search, case_sensitive: false)
@@ -42,8 +53,15 @@ def original_backend
       end
 
       def backend_translations
-        list = backend.translations(do_init: true)
-        list[locale]
+        all_translations[locale]
+      end
+
+      def all_translations
+        @all_translations ||= backend.translations(do_init: true)
+      end
+
+      def merge_search_results(locale_results, primary_results)
+        primary_results.merge(locale_results)
       end
     end
   end

lib/decidim/term_customizer/translation_store.rb

@@ -18,17 +18,23 @@ def by_key(search)
       end
 
       def by_term(search, case_sensitive: false)
+        normalized_search = case_sensitive ? search : normalize(search).downcase
+
         @values.select do |_key, term|
-          includes_string?(term, search, case_sensitive:)
+          includes_string?(term, normalized_search, case_sensitive:)
         end
       end
 
       private
 
+      def normalize(str)
+        str.unicode_normalize(:nfd).gsub(/\p{M}/, "")
+      end
+
       def includes_string?(source, search, case_sensitive: false)
         return source.include?(search) if case_sensitive
 
-        source.downcase.include?(search.downcase)
+        normalize(source).downcase.include?(search)
       end
 
       def flat_hash(hash)

spec/lib/decidim/term_customizer/translation_directory_spec.rb

@@ -49,4 +49,132 @@
       )
     end
   end
+
+  context "when using accented characters in the search" do
+    it "returns correct translations when the search is case insensitive" do
+      expect(subject.translations_search("térm custômizer")).to eq(
+        "decidim.term_customizer.menu.term_customizer" => "Term customizer"
+      )
+    end
+
+    context "when the term contains accents" do
+      let(:locale) { :ca }
+
+      # rubocop:disable RSpec/SubjectStub
+      before do
+        allow(subject).to receive(:all_translations).and_return({
+                                                                  en: {
+                                                                    decidim: {
+                                                                      term_customizer: {
+                                                                        menu: {
+                                                                          term_customizer: "Term custômizer"
+                                                                        }
+                                                                      }
+                                                                    }
+                                                                  }
+                                                                })
+      end
+
+      it "returns correct translations when the search is case insensitive" do
+        expect(subject.translations_search("térm customizer")).to eq(
+          "decidim.term_customizer.menu.term_customizer" => "Term custômizer"
+        )
+      end
+    end
+  end
+
+  context "when the locale is not present in the translations" do
+    let(:locale) { :ca }
+
+    before do
+      allow(subject).to receive(:all_translations).and_return({
+                                                                en: {
+                                                                  decidim: {
+                                                                    term_customizer: {
+                                                                      menu: {
+                                                                        term_customizer: "Term customizer"
+                                                                      }
+                                                                    }
+                                                                  }
+                                                                }
+                                                              })
+    end
+    # rubocop:enable RSpec/SubjectStub
+
+    it "does not return any translations by key when using the secondary language backend" do
+      expect(subject.translations.by_key("term_customizer")).to eq({})
+    end
+
+    it "returns translations by key when using the English source fallback" do
+      expect(subject.canonical_source_terms.by_key("term_customizer")).to eq(
+        "decidim.term_customizer.menu.term_customizer" => "Term customizer"
+      )
+    end
+
+    it "still returns the correct translations by key globally" do
+      expect(subject.translations_by_key("term_customizer")).to eq(
+        "decidim.term_customizer.menu.term_customizer" => "Term customizer"
+      )
+    end
+
+    it "does not return the correct translation by term" do
+      expect(subject.translations_by_term("term customizer")).to eq({})
+    end
+
+    it "returns the correct translations by term globally with merged search" do
+      expect(subject.translations_search("term customizer")).to eq(
+        "decidim.term_customizer.menu.term_customizer" => "Term customizer"
+      )
+    end
+  end
+
+  context "when the locale has translations for the same keys as the primary language" do
+    let(:locale) { :ca }
+
+    # rubocop:disable RSpec/SubjectStub
+    before do
+      allow(subject).to receive(:all_translations).and_return({
+                                                                en: {
+                                                                  decidim: {
+                                                                    term_customizer: {
+                                                                      menu: {
+                                                                        term_customizer: "Term customizer",
+                                                                        secondary_term: "Secondary term"
+                                                                      }
+                                                                    }
+                                                                  }
+                                                                },
+                                                                ca: {
+                                                                  decidim: {
+                                                                    term_customizer: {
+                                                                      menu: {
+                                                                        term_customizer: "Personalitzador de termes"
+                                                                      }
+                                                                    }
+                                                                  }
+                                                                }
+                                                              })
+    end
+    # rubocop:enable RSpec/SubjectStub
+
+    it "returns the localized value for overlapping keys and falls back to English for missing ones in key searches" do
+      expect(subject.translations_by_key("menu.term_customizer")).to eq(
+        "decidim.term_customizer.menu.term_customizer" => "Personalitzador de termes"
+      )
+
+      expect(subject.translations_by_key("menu.secondary_term")).to eq(
+        "decidim.term_customizer.menu.secondary_term" => "Secondary term"
+      )
+    end
+
+    it "prefers the localized value and keeps English fallback when merging global search results" do
+      expect(subject.translations_search("menu.term_customizer")).to eq(
+        "decidim.term_customizer.menu.term_customizer" => "Personalitzador de termes"
+      )
+
+      expect(subject.translations_search("menu.secondary_term")).to eq(
+        "decidim.term_customizer.menu.secondary_term" => "Secondary term"
+      )
+    end
+  end
 end

spec/lib/decidim/term_customizer/translation_store_spec.rb

@@ -17,6 +17,10 @@
         second_level: {
           third_level: "First second third"
         }
+      },
+      unicode_marks: {
+        spacing_mark: "a\u0903b",
+        enclosing_mark: "a\u20DDb"
       }
     }
   end
@@ -52,6 +56,24 @@
   end
 
   describe "#by_term" do
+    context "when searching with accented characters" do
+      it "matches terms ignoring accents" do
+        expect(subject.by_term("öne twô thrèe1").length).to eq(1)
+        expect(subject.by_term("fïrst sécond thïrd").length).to eq(1)
+      end
+
+      it "matches terms when the source contains other Unicode mark categories" do
+        expect(subject.by_term("ab")).to include(
+          "unicode_marks.spacing_mark" => "a\u0903b",
+          "unicode_marks.enclosing_mark" => "a\u20DDb"
+        )
+      end
+
+      it "does not match unrelated terms" do
+        expect(subject.by_term("öne thrèe").length).to eq(0)
+      end
+    end
+
     context "when case insensitive" do
       it "returns a specific transation for specific term" do
         expect(subject.by_term("One two three1").length).to eq(1)