Fix missing label retry state leaking across ontology processing

.gitignore

@@ -1,4 +1,5 @@
 .bundle/
+vendor/
 *.swp
 *.gemtags
 .bundle

lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb

@@ -10,12 +10,13 @@ def process(logger, options = {})
       private
 
       def handle_missing_labels(file_path, logger)
+        status = LinkedData::Models::SubmissionStatus.find('RDF_LABELS').first
         callbacks = {
           include_languages: true,
           missing_labels: {
             op_name: 'Missing Labels Generation',
             required: true,
-            status: LinkedData::Models::SubmissionStatus.find('RDF_LABELS').first,
+            status: status,
             artifacts: {
               file_path: file_path
             },
@@ -29,6 +30,12 @@ def handle_missing_labels(file_path, logger)
 
         raw_paging = LinkedData::Models::Class.in(@submission).include(:prefLabel, :synonym, :label)
         loop_classes(logger, raw_paging, @submission, callbacks)
+      rescue StandardError => e
+        logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}")
+        logger.flush
+        @submission.add_submission_status(status.get_error_status)
+        @submission.save
+        raise e
       end
 
       def process_callbacks(logger, callbacks, action_name)
@@ -39,7 +46,7 @@ def process_callbacks(logger, callbacks, action_name)
               yield(callable, callback)
             end
             false
-          rescue Exception => e
+          rescue StandardError => e
             logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}")
             logger.flush
 
@@ -96,9 +103,15 @@ def loop_classes(logger, raw_paging, submission, callbacks)
 
           begin
             t0 = Time.now
-            page_classes = paging.page(page, size).all
-            total_pages = page_classes.total_pages
-            page_len = page_classes.length
+            begin
+              page_classes = paging.page(page, size).all
+              total_pages = page_classes.total_pages
+              page_len = page_classes.length
+            rescue StandardError => e
+              logger.error("Error retrieving page #{page} for #{acr}: #{e.class}: #{e.message}")
+              logger.flush
+              raise e
+            end
 
             # nothing retrieved even though we're expecting more records
             if total_pages > 0 && page_classes.empty? && (prev_page_len == -1 || prev_page_len == size)
@@ -109,10 +122,18 @@ def loop_classes(logger, raw_paging, submission, callbacks)
                 j += 1
                 logger.error("Empty page encountered. Retrying #{j} times...")
                 sleep(2)
-                page_classes = paging.page(page, size).all
-                unless page_classes.empty?
-                  logger.info("Success retrieving a page of #{page_classes.length} classes after retrying #{j} times...")
+                begin
+                  page_classes = paging.page(page, size).all
+                  total_pages = page_classes.total_pages
+                  page_len = page_classes.length
+                  unless page_classes.empty?
+                    logger.info("Success retrieving a page of #{page_classes.length} classes after retrying #{j} times...")
+                  end
+                rescue StandardError => e
+                  page_classes = []
+                  logger.error("Retry #{j} failed retrieving page #{page} for #{acr}: #{e.class}: #{e.message}")
                 end
+                logger.flush
               end
 
               if page_classes.empty?
@@ -166,6 +187,7 @@ def loop_classes(logger, raw_paging, submission, callbacks)
             @submission.save
           end
         end
+      ensure
         RequestStore.store[:requested_lang] = nil if incl_lang
       end
 
@@ -317,4 +339,3 @@ def generate_missing_labels_post(artifacts = {}, logger, pagging)
 
   end
 end
-

test/data/ontology_files/no_concepts.skos.rdf

@@ -0,0 +1,11 @@
+<?xml version="1.0"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+         xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+         xmlns:skos="http://www.w3.org/2004/02/skos/core#"
+         xmlns:dcterms="http://purl.org/dc/terms/">
+  <skos:ConceptScheme rdf:about="http://example.org/no-concepts">
+    <dcterms:title>No Concepts SKOS Scheme</dcterms:title>
+    <skos:prefLabel xml:lang="en">No Concepts SKOS Scheme</skos:prefLabel>
+    <rdfs:comment xml:lang="en">A SKOS-like fixture with no skos:Concept resources.</rdfs:comment>
+  </skos:ConceptScheme>
+</rdf:RDF>

test/models/test_ontology_submission.rb

@@ -1,6 +1,7 @@
 require_relative "./test_ontology_common"
 require "logger"
 require "rack"
+require "mocha/minitest"
 
 class TestOntologySubmission < LinkedData::TestOntologyCommon
 
@@ -1308,4 +1309,64 @@ def test_copy_file_repository_from_tempfile
     end
   end
 
+  def test_generate_missing_labels_sets_error_status_on_initial_page_fetch_failure
+    acronym = "BRO-ML-ERR"
+    ontology_file = "./test/data/ontology_files/BRO_v3.2.owl"
+    ont_format, ont, = submission_dependent_objects("OWL", acronym, "test_linked_models", "BRO labels error")
+
+    sub = LinkedData::Models::OntologySubmission.new(submissionId: 1)
+    sub.uri = RDF::URI.new("https://test.com")
+    sub.description = "description example"
+    sub.status = "beta"
+    sub.released = DateTime.now - 4
+    sub.hasOntologyLanguage = ont_format
+    sub.ontology = ont
+    sub.uploadFilePath = LinkedData::Models::OntologySubmission.copy_file_repository(acronym, 1, ontology_file)
+    sub.contact = [LinkedData::Models::Contact.where(name: "Peter", email: "peter@example.org").first]
+    assert sub.valid?, sub.errors.inspect
+    sub.save
+
+    logger = Logger.new(TestLogFile.new)
+
+    sub.stubs(:class_count).with(logger).returns(-1)
+
+    fake_scope = mock("missing-labels-scope")
+    fake_scope.stubs(:page).with(1, 2500).returns(fake_scope)
+    fake_scope.stubs(:all).raises(StandardError.new("simulated page fetch failure"))
+
+    class_query = mock("class-query")
+    class_query.stubs(:include).with(:prefLabel, :synonym, :label).returns(fake_scope)
+
+    LinkedData::Models::Class.stubs(:in).with(sub).returns(class_query)
+
+    RequestStore.store[:requested_lang] = nil
+
+    assert_raises(StandardError) do
+      sub.generate_missing_labels(logger)
+    end
+
+    sub.bring(:submissionStatus)
+    codes = sub.submissionStatus.map { |status| status.get_code_from_id }
+
+    assert_includes codes, "ERROR_RDF_LABELS"
+    refute_includes codes, "RDF_LABELS"
+    assert_nil RequestStore.store[:requested_lang]
+  end
+
+  def test_skos_submission_without_skos_concept_triggers_current_retry_path
+    # See GH-274: SKOS submissions with no skos:Concept currently enter the retry path and fail
+    # during missing-label generation. Keep this regression test to document the current behavior
+    # until retry-start logic is corrected.
+    error = assert_raises(RuntimeError) do
+      submission_parse("SKOS-NO-CONCEPT",
+                       "SKOS without Concept",
+                       "./test/data/ontology_files/no_concepts.skos.rdf", 1,
+                       process_rdf: true, extract_metadata: false,
+                       index_search: false, run_metrics: false, diff: false)
+    end
+
+    assert_match(/Empty page 1 of 1 persisted after retrying/, error.message)
+    assert_nil RequestStore.store[:requested_lang]
+  end
+
 end

test/test_case.rb

@@ -20,6 +20,7 @@
 require_relative '../lib/ontologies_linked_data'
 require_relative '../config/config.test'
 require 'minitest/autorun'
+require 'tmpdir'
 require 'webmock/minitest'
 WebMock.allow_net_connect!
 
@@ -51,10 +52,12 @@ def run_suite(reporter, options = {})
 module LinkedData
   module MinitestRunHooks
     def run(*args)
+      LinkedData::TestCase.setup_test_repository_folder
       LinkedData::TestCase.backend_4s_delete
       super
     ensure
       LinkedData::TestCase.backend_4s_delete
+      LinkedData::TestCase.teardown_test_repository_folder
     end
   end
 end
@@ -94,6 +97,29 @@ class TestCase < Minitest::Test
     # Ensure all threads exit on any exception
     Thread.abort_on_exception = true
 
+    def self.setup_test_repository_folder
+      return if defined?(@tmp_repository_folder) && @tmp_repository_folder
+
+      @old_repository_folder = LinkedData.settings.repository_folder
+      @tmp_repository_folder = Dir.mktmpdir("ontology-repo-")
+      puts "(LD) >> Using temp repository folder #{@tmp_repository_folder}"
+      LinkedData.settings.repository_folder = @tmp_repository_folder
+    end
+
+    def self.teardown_test_repository_folder
+      return unless defined?(@tmp_repository_folder) && @tmp_repository_folder
+
+      LinkedData.settings.repository_folder = @old_repository_folder if defined?(@old_repository_folder)
+
+      if ENV["KEEP_TEST_REPOSITORY_FOLDER"] == "true"
+        puts "(LD) >> Preserving temp repository folder #{@tmp_repository_folder}"
+      elsif File.exist?(@tmp_repository_folder)
+        FileUtils.remove_entry(@tmp_repository_folder)
+      end
+
+      @tmp_repository_folder = nil
+    end
+
     def submission_dependent_objects(format, acronym, user_name)
       # ontology format
       owl = LinkedData::Models::OntologyFormat.where(acronym: format).first

test/test_log_file.rb

@@ -1,5 +1,7 @@
+require "tmpdir"
+
 class TestLogFile < File
   def initialize
-    super(File.expand_path("../test_run.log", __FILE__), "w")
+    super(File.join(Dir.tmpdir, "ontologies_linked_data-test-#{Process.pid}.log"), "w")
   end
-end
+end