diff --git a/.gitignore b/.gitignore index ff8778f9..4937634a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .bundle/ +vendor/ *.swp *.gemtags .bundle diff --git a/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb b/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb index 176d2a7b..a102d2dc 100644 --- a/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb +++ b/lib/ontologies_linked_data/services/submission_process/operations/submission_missing_labels.rb @@ -10,12 +10,13 @@ def process(logger, options = {}) private def handle_missing_labels(file_path, logger) + status = LinkedData::Models::SubmissionStatus.find('RDF_LABELS').first callbacks = { include_languages: true, missing_labels: { op_name: 'Missing Labels Generation', required: true, - status: LinkedData::Models::SubmissionStatus.find('RDF_LABELS').first, + status: status, artifacts: { file_path: file_path }, @@ -29,6 +30,12 @@ def handle_missing_labels(file_path, logger) raw_paging = LinkedData::Models::Class.in(@submission).include(:prefLabel, :synonym, :label) loop_classes(logger, raw_paging, @submission, callbacks) + rescue StandardError => e + logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") + logger.flush + @submission.add_submission_status(status.get_error_status) + @submission.save + raise e end def process_callbacks(logger, callbacks, action_name) @@ -39,7 +46,7 @@ def process_callbacks(logger, callbacks, action_name) yield(callable, callback) end false - rescue Exception => e + rescue StandardError => e logger.error("#{e.class}: #{e.message}\n#{e.backtrace.join("\n\t")}") logger.flush @@ -96,9 +103,15 @@ def loop_classes(logger, raw_paging, submission, callbacks) begin t0 = Time.now - page_classes = paging.page(page, size).all - total_pages = page_classes.total_pages - page_len = page_classes.length + begin + page_classes = paging.page(page, size).all + total_pages = page_classes.total_pages + page_len = page_classes.length + rescue StandardError => e + logger.error("Error retrieving page #{page} for #{acr}: #{e.class}: #{e.message}") + logger.flush + raise e + end # nothing retrieved even though we're expecting more records if total_pages > 0 && page_classes.empty? && (prev_page_len == -1 || prev_page_len == size) @@ -109,10 +122,18 @@ def loop_classes(logger, raw_paging, submission, callbacks) j += 1 logger.error("Empty page encountered. Retrying #{j} times...") sleep(2) - page_classes = paging.page(page, size).all - unless page_classes.empty? - logger.info("Success retrieving a page of #{page_classes.length} classes after retrying #{j} times...") + begin + page_classes = paging.page(page, size).all + total_pages = page_classes.total_pages + page_len = page_classes.length + unless page_classes.empty? + logger.info("Success retrieving a page of #{page_classes.length} classes after retrying #{j} times...") + end + rescue StandardError => e + page_classes = [] + logger.error("Retry #{j} failed retrieving page #{page} for #{acr}: #{e.class}: #{e.message}") end + logger.flush end if page_classes.empty? @@ -166,6 +187,7 @@ def loop_classes(logger, raw_paging, submission, callbacks) @submission.save end end + ensure RequestStore.store[:requested_lang] = nil if incl_lang end @@ -317,4 +339,3 @@ def generate_missing_labels_post(artifacts = {}, logger, pagging) end end - diff --git a/test/data/ontology_files/no_concepts.skos.rdf b/test/data/ontology_files/no_concepts.skos.rdf new file mode 100644 index 00000000..72c08cff --- /dev/null +++ b/test/data/ontology_files/no_concepts.skos.rdf @@ -0,0 +1,11 @@ + + + + No Concepts SKOS Scheme + No Concepts SKOS Scheme + A SKOS-like fixture with no skos:Concept resources. + + diff --git a/test/models/test_ontology_submission.rb b/test/models/test_ontology_submission.rb index 2399694b..81ccd9e6 100644 --- a/test/models/test_ontology_submission.rb +++ b/test/models/test_ontology_submission.rb @@ -1,6 +1,7 @@ require_relative "./test_ontology_common" require "logger" require "rack" +require "mocha/minitest" class TestOntologySubmission < LinkedData::TestOntologyCommon @@ -1308,4 +1309,64 @@ def test_copy_file_repository_from_tempfile end end + def test_generate_missing_labels_sets_error_status_on_initial_page_fetch_failure + acronym = "BRO-ML-ERR" + ontology_file = "./test/data/ontology_files/BRO_v3.2.owl" + ont_format, ont, = submission_dependent_objects("OWL", acronym, "test_linked_models", "BRO labels error") + + sub = LinkedData::Models::OntologySubmission.new(submissionId: 1) + sub.uri = RDF::URI.new("https://test.com") + sub.description = "description example" + sub.status = "beta" + sub.released = DateTime.now - 4 + sub.hasOntologyLanguage = ont_format + sub.ontology = ont + sub.uploadFilePath = LinkedData::Models::OntologySubmission.copy_file_repository(acronym, 1, ontology_file) + sub.contact = [LinkedData::Models::Contact.where(name: "Peter", email: "peter@example.org").first] + assert sub.valid?, sub.errors.inspect + sub.save + + logger = Logger.new(TestLogFile.new) + + sub.stubs(:class_count).with(logger).returns(-1) + + fake_scope = mock("missing-labels-scope") + fake_scope.stubs(:page).with(1, 2500).returns(fake_scope) + fake_scope.stubs(:all).raises(StandardError.new("simulated page fetch failure")) + + class_query = mock("class-query") + class_query.stubs(:include).with(:prefLabel, :synonym, :label).returns(fake_scope) + + LinkedData::Models::Class.stubs(:in).with(sub).returns(class_query) + + RequestStore.store[:requested_lang] = nil + + assert_raises(StandardError) do + sub.generate_missing_labels(logger) + end + + sub.bring(:submissionStatus) + codes = sub.submissionStatus.map { |status| status.get_code_from_id } + + assert_includes codes, "ERROR_RDF_LABELS" + refute_includes codes, "RDF_LABELS" + assert_nil RequestStore.store[:requested_lang] + end + + def test_skos_submission_without_skos_concept_triggers_current_retry_path + # See GH-274: SKOS submissions with no skos:Concept currently enter the retry path and fail + # during missing-label generation. Keep this regression test to document the current behavior + # until retry-start logic is corrected. + error = assert_raises(RuntimeError) do + submission_parse("SKOS-NO-CONCEPT", + "SKOS without Concept", + "./test/data/ontology_files/no_concepts.skos.rdf", 1, + process_rdf: true, extract_metadata: false, + index_search: false, run_metrics: false, diff: false) + end + + assert_match(/Empty page 1 of 1 persisted after retrying/, error.message) + assert_nil RequestStore.store[:requested_lang] + end + end diff --git a/test/test_case.rb b/test/test_case.rb index a72400e5..5a93191c 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -20,6 +20,7 @@ require_relative '../lib/ontologies_linked_data' require_relative '../config/config.test' require 'minitest/autorun' +require 'tmpdir' require 'webmock/minitest' WebMock.allow_net_connect! @@ -51,10 +52,12 @@ def run_suite(reporter, options = {}) module LinkedData module MinitestRunHooks def run(*args) + LinkedData::TestCase.setup_test_repository_folder LinkedData::TestCase.backend_4s_delete super ensure LinkedData::TestCase.backend_4s_delete + LinkedData::TestCase.teardown_test_repository_folder end end end @@ -94,6 +97,29 @@ class TestCase < Minitest::Test # Ensure all threads exit on any exception Thread.abort_on_exception = true + def self.setup_test_repository_folder + return if defined?(@tmp_repository_folder) && @tmp_repository_folder + + @old_repository_folder = LinkedData.settings.repository_folder + @tmp_repository_folder = Dir.mktmpdir("ontology-repo-") + puts "(LD) >> Using temp repository folder #{@tmp_repository_folder}" + LinkedData.settings.repository_folder = @tmp_repository_folder + end + + def self.teardown_test_repository_folder + return unless defined?(@tmp_repository_folder) && @tmp_repository_folder + + LinkedData.settings.repository_folder = @old_repository_folder if defined?(@old_repository_folder) + + if ENV["KEEP_TEST_REPOSITORY_FOLDER"] == "true" + puts "(LD) >> Preserving temp repository folder #{@tmp_repository_folder}" + elsif File.exist?(@tmp_repository_folder) + FileUtils.remove_entry(@tmp_repository_folder) + end + + @tmp_repository_folder = nil + end + def submission_dependent_objects(format, acronym, user_name) # ontology format owl = LinkedData::Models::OntologyFormat.where(acronym: format).first diff --git a/test/test_log_file.rb b/test/test_log_file.rb index 73ca561d..58ebc367 100644 --- a/test/test_log_file.rb +++ b/test/test_log_file.rb @@ -1,5 +1,7 @@ +require "tmpdir" + class TestLogFile < File def initialize - super(File.expand_path("../test_run.log", __FILE__), "w") + super(File.join(Dir.tmpdir, "ontologies_linked_data-test-#{Process.pid}.log"), "w") end -end \ No newline at end of file +end