From 34379a5852116f004a046ddf9e104b9ef1b525c5 Mon Sep 17 00:00:00 2001 From: Bobbi Fox Date: Mon, 6 May 2019 17:25:22 -0400 Subject: [PATCH 1/5] handle multiple exact matches for agents --- .../controllers/concerns/linked_objects.rb | 30 ++++++++++++------- frontend/locales/en.yml | 1 + frontend/models/handler.rb | 21 ++++++++++--- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/frontend/controllers/concerns/linked_objects.rb b/frontend/controllers/concerns/linked_objects.rb index d0e0162..76bc1a5 100644 --- a/frontend/controllers/concerns/linked_objects.rb +++ b/frontend/controllers/concerns/linked_objects.rb @@ -48,13 +48,23 @@ def self.get_or_create(row, type, num, resource_uri, report) end end begin - unless agent_obj || (agent_obj = get_db_agent(agent, resource_uri, num)) - agent_obj = create_agent(agent, num) - report.add_info(I18n.t('plugins.aspace-import-excel.created', :what =>"#{I18n.t('plugins.aspace-import-excel.agent')}[#{agent[:name]}]", :id => agent_obj.uri)) - end + if !agent_obj + begin + agent_obj = get_db_agent(agent, resource_uri, num) + rescue Exception => e + if e.message == 'More than one match found in the database' + agent[:name] = agent[:name] + " DISAMBIGUATE ME!" + report.add_info(I18n.t('plugins.aspace-import-excel.warn.disam', :name => agent[:name])) + else + raise e + end + end + end + if !agent_obj + agent_obj = create_agent(agent, num) + report.add_info(I18n.t('plugins.aspace-import-excel.created', :what =>"#{I18n.t('plugins.aspace-import-excel.agent')}[#{agent[:name]}]", :id => agent_obj.uri)) + end rescue Exception => e -# Pry::ColorPrinter.pp e.message -# Pry::ColorPrinter.pp e.backtrace raise ExcelImportException.new( I18n.t('plugins.aspace-import-excel.error.no_agent', :num => num, :why => e.message)) end end @@ -108,7 +118,7 @@ def self.get_db_agent(agent, resource_uri, num) if !ret_ag a_params = {"q" => "title:\"#{agent[:name]}\" AND primary_type:agent_#{agent[:type]}"} repo = resource_uri.split('/')[2] - ret_ag = search(repo, a_params, "agent_#{agent[:type]}".to_sym) + ret_ag = search(repo, a_params, "agent_#{agent[:type]}".to_sym,'', "title:#{agent[:name]}") end ret_ag end @@ -244,7 +254,7 @@ def self.get_db_tc(top_container, resource_uri) tc_params = {} tc_params["type[]"] = 'top_container' tc_params["q"] = "display_string:\"#{tc_str}\" AND collection_uri_u_sstr:\"#{resource_uri}\"" - ret_tc = search(repo_id,tc_params, :top_container) + ret_tc = search(repo_id,tc_params, :top_container,'', "display_string:#{tc_str}") end ret_tc end @@ -254,7 +264,7 @@ def self.get_db_tc_by_barcode(barcode, repo_id) if barcode tc_params = {} tc_params["type[]"] = 'top_container' - tc_params["q"] = "barcode_u_sstr:#{barcode}" + tc_params["q"] = "barcode_u_sstr:\"#{barcode}\"" ret_tc = search(repo_id,tc_params, :top_container) end ret_tc @@ -397,7 +407,7 @@ def self.get_db_subj(subject) s_params = {} s_params["q"] = "title:\"#{subject[:term]}\" AND first_term_type:#{subject[:type]}" - ret_subj = search(nil, s_params, :subject, 'subjects') + ret_subj = search(nil, s_params, :subject, 'subjects',"title:#{subject[:term]}" ) end end end diff --git a/frontend/locales/en.yml b/frontend/locales/en.yml index 82ed8c4..974c4af 100644 --- a/frontend/locales/en.yml +++ b/frontend/locales/en.yml @@ -28,6 +28,7 @@ en: ref_id_notfound: "Ref Id %{refid} not found" warn: dup: "Managed Controlled Value List %{which} has multiple instances for the Translation '%{trans}'. '%{used}' will be used as the value." + disam: "Multiple match(es) found. Creating %{name} for disabiguation." error: date_type: "Date type [%{what}] invalid. Defaulting to 'inclusive'" certainty: "Invalid 'date certainty' ignored: (%{what})" diff --git a/frontend/models/handler.rb b/frontend/models/handler.rb index 0e3d770..1a1a5de 100644 --- a/frontend/models/handler.rb +++ b/frontend/models/handler.rb @@ -20,16 +20,17 @@ def self.stored(hash, id, key) # returns nil, a hash of a jason model (if 1 found), or throws a multiples found error # if repo_id is nil, do a global search (subject and agent) # this is using archivesspace/frontend/app/models/search.rb - def self.search(repo_id,params,jmsym, *type) + def self.search(repo_id,params,jmsym, type = '', match = '') obj = nil search = nil + matches = match.split(':') if repo_id search = Search.all(repo_id, params) else begin - search = Search.global(params,type[0]) + search = Search.global(params,type) rescue Exception => e - s = JSONModel::HTTP::get_json("/search/#{type[0]}", params) + s = JSONModel::HTTP::get_json("/search/#{type}", params) raise e if !e.message.match('

Not Found

') # global search doesn't handle this gracefully :-( search = {'total_hits' => 0} end @@ -39,7 +40,19 @@ def self.search(repo_id,params,jmsym, *type) if total_hits == 1 && !search['results'].blank? # for some reason, you get a hit of '1' but still have empty results?? obj = JSONModel(jmsym).find_by_uri(search['results'][0]['id']) elsif total_hits > 1 - raise Exception.new(I18n.t('plugins.aspace-import-excel.error.too_many')) + if matches.length == 2 + search['results'].each do |result| + if result[matches[0]] == matches[1] + # if we have more than one exact match, then bail! + if obj + raise Exception.new(I18n.t('plugins.aspace-import-excel.error.too_many')) + end + obj = JSONModel(jmsym).find_by_uri(result['id']) + end + end + else + raise Exception.new(I18n.t('plugins.aspace-import-excel.error.too_many')) + end elsif total_hits == 0 # Pry::ColorPrinter.pp search end From 1fe514586fd54305c8859025f59764b47e434308 Mon Sep 17 00:00:00 2001 From: Bobbi Fox Date: Tue, 7 May 2019 11:05:21 -0400 Subject: [PATCH 2/5] temporary with debugging log messages --- frontend/controllers/concerns/linked_objects.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/controllers/concerns/linked_objects.rb b/frontend/controllers/concerns/linked_objects.rb index 76bc1a5..945ae3d 100644 --- a/frontend/controllers/concerns/linked_objects.rb +++ b/frontend/controllers/concerns/linked_objects.rb @@ -36,13 +36,13 @@ def self.build(row, type, num) def self.get_or_create(row, type, num, resource_uri, report) agent = build(row, type, num) agent_key = key_for(agent) + Rails.logger.error("Created Agent: #{agent.pretty_inspect} key: #{agent_key}") if !(agent_obj = stored(@@agents, agent[:id], agent_key)) unless agent[:id].blank? begin agent_obj = JSONModel("agent_#{agent[:type]}".to_sym).find(agent[:id]) rescue Exception => e if e.message != 'RecordNotFound' -# Pry::ColorPrinter.pp e raise ExcelImportException.new( I18n.t('plugins.aspace-import-excel.error.no_agent', :num => num, :why => e.message)) end end @@ -87,6 +87,7 @@ def self.get_or_create(row, type, num, resource_uri, report) end end end + Rails.logger.error("AGENTS: #{@@agents.pretty_inspect}") agent_link end From 999e0545ebec0fffc0bad7fe00a436cdedaff181 Mon Sep 17 00:00:00 2001 From: Bobbi Fox Date: Wed, 8 May 2019 11:35:22 -0400 Subject: [PATCH 3/5] exact match and disambiguation support --- .../controllers/concerns/linked_objects.rb | 21 ++++++++++++++----- frontend/models/handler.rb | 21 ++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/frontend/controllers/concerns/linked_objects.rb b/frontend/controllers/concerns/linked_objects.rb index 945ae3d..6b54cc0 100644 --- a/frontend/controllers/concerns/linked_objects.rb +++ b/frontend/controllers/concerns/linked_objects.rb @@ -15,6 +15,7 @@ class AgentHandler < Handler AGENT_TYPES = { 'families' => 'family', 'corporate_entities' => 'corporate_entity', 'people' => 'person'} def self.renew clear(@@agent_relators) + @@agents = {} end def self.key_for(agent) key = "#{agent[:type]} #{agent[:name]}" @@ -36,7 +37,6 @@ def self.build(row, type, num) def self.get_or_create(row, type, num, resource_uri, report) agent = build(row, type, num) agent_key = key_for(agent) - Rails.logger.error("Created Agent: #{agent.pretty_inspect} key: #{agent_key}") if !(agent_obj = stored(@@agents, agent[:id], agent_key)) unless agent[:id].blank? begin @@ -87,7 +87,6 @@ def self.get_or_create(row, type, num, resource_uri, report) end end end - Rails.logger.error("AGENTS: #{@@agents.pretty_inspect}") agent_link end @@ -110,8 +109,6 @@ def self.get_db_agent(agent, resource_uri, num) ret_ag = JSONModel("agent_#{agent[:type]}".to_sym).find(agent[:id]) rescue Exception => e if e.message != 'RecordNotFound' -# Pry::ColorPrinter.pp e.message -# Pry::ColorPrinter.pp e.backtrace raise ExcelImportException.new( I18n.t('plugins.aspace-import-excel.error.no_agent', :num => num, :why => e.message)) end end @@ -336,6 +333,7 @@ class SubjectHandler < Handler def self.renew clear(@@subject_term_types) clear(@@subject_sources) + @@subjects = {} end def self.key_for(subject) @@ -368,11 +366,24 @@ def self.get_or_create(row, num, repo_id, report) end end begin - unless subj || (subj = get_db_subj(subject)) + if !subj + begin + subj = get_db_subj(subject) + rescue Exception => e + if e.message == 'More than one match found in the database' + subject[:term] = subject[:term] + " DISAMBIGUATE ME!" + report.add_info(I18n.t('plugins.aspace-import-excel.warn.disam', :name => subject[:term])) + else + raise e + end + end + end + if !subj subj = create_subj(subject, num) report.add_info(I18n.t('plugins.aspace-import-excel.created', :what =>"#{I18n.t('plugins.aspace-import-excel.subj')}[#{subject[:term]}]", :id => subj.uri)) end rescue Exception => e + Rails.logger.error(e.backtrace) raise ExcelImportException.new( I18n.t('plugins.aspace-import-excel.error.no_subject',:num => num, :why => e.message)) end if subj diff --git a/frontend/models/handler.rb b/frontend/models/handler.rb index 1a1a5de..3adc551 100644 --- a/frontend/models/handler.rb +++ b/frontend/models/handler.rb @@ -36,25 +36,32 @@ def self.search(repo_id,params,jmsym, type = '', match = '') end end total_hits = search['total_hits'] || 0 -# Pry::ColorPrinter.pp "Total hits: #{total_hits}" if total_hits == 1 && !search['results'].blank? # for some reason, you get a hit of '1' but still have empty results?? obj = JSONModel(jmsym).find_by_uri(search['results'][0]['id']) elsif total_hits > 1 if matches.length == 2 + match_ct = 0 + disam = matches[1] + " DISAMBIGUATE ME!" + disam_obj = nil search['results'].each do |result| - if result[matches[0]] == matches[1] - # if we have more than one exact match, then bail! - if obj - raise Exception.new(I18n.t('plugins.aspace-import-excel.error.too_many')) - end + # if we have a disambiguate result get it + if result[matches[0]] == disam + disam_obj = JSONModel(jmsym).find_by_uri(result['id']) + elsif result[matches[0]] == matches[1] + match_ct += 1 obj = JSONModel(jmsym).find_by_uri(result['id']) end end + # if we have more than one exact match, then return disam_obj if we have one, or bail! + if match_ct > 1 + return disam_obj if disam_obj + raise Exception.new(I18n.t('plugins.aspace-import-excel.error.too_many')) + end else raise Exception.new(I18n.t('plugins.aspace-import-excel.error.too_many')) end elsif total_hits == 0 -# Pry::ColorPrinter.pp search +# Rails.logger.info("No hits found") end obj end From 12f598caa64a62b383dbb88b45543cb331606306 Mon Sep 17 00:00:00 2001 From: Bobbi Fox Date: Wed, 8 May 2019 12:40:57 -0400 Subject: [PATCH 4/5] Updated Agent and Subject documentation --- .../archival_objects_instructions.md | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/user_documentation/archival_objects_instructions.md b/user_documentation/archival_objects_instructions.md index 48c959e..9ab33ef 100644 --- a/user_documentation/archival_objects_instructions.md +++ b/user_documentation/archival_objects_instructions.md @@ -128,11 +128,21 @@ URL of thumbnail| URL String || if defined, this becomes the File version with The ingester allows you to link Agents (*CREATOR role only!*) to Archival objects. You can specify up to 3 Person Agents, up to 2 Corporate Agents, and one Family Agent per Archival object. -If you have previously defined the Agent(s) you are using, you may use the Record ID number (e.g.: for the Agent URI /agents */agent_person/1249*, you would use **1249**) OR the full header header string, with all capitalization and punctuation. +If you have previously defined the Agent(s) you are using, you may use the Record ID number (e.g.: for the Agent URI /agents */agent_person/1249*, you would use **1249**) OR the full header string, with all capitalization and punctuation. + +Either the Record ID *or* the header string is **required**. + +If you include both, or only the header, and the record isn't found, a new Agent record will be created. The header string will be used as the **family_name** if it's a Family Agent, and the **primary_name** +otherwise. + +If you enter the header string *without* the ID, the ingester will try to do an **exact match** against the header; if it finds more than one match (for example, if the database contains two agents with identical headers, but different sources): + + * The ingester will create a **new** agent (with publish=false) containing the header with ' DISAMBIGUATE ME!' appended to it. For example, given a person agent with a header of 'George Washington', a new person agent would be created with a primary name of 'George Washington DISAMBIGUATE ME!'. + * After ingest, you can use the *merge* functionality to resolve the ambiguities. + +If you enter a Record ID and **not** the header string, and that ID is not found, a new Agent record will be created with the name "PLACEHOLDER FOR *{agent type}* ID *{ id number}* NOT FOUND", so that you may easily find that record later and edit/merge it. In this case, the new Agent would be marked publish=false. When you correct the record, change publish to true if appropriate. -Either the Record ID *or* the header string is **required**; if you include both, and the record isn't found, a new Agent record will be created. The header string will be used as the **family_name** if it's a Family Agent, and the **primary_name** otherwise. -If for some reason you enter a Record ID and **not** the header string, and that ID is not found, a new Agent record will be created with the name "PLACEHOLDER FOR *{agent type}* ID *{ id number}* NOT FOUND", so that you may easily find that record later and edit/merge it. In this case, the new Agent would be marked publish=false. When you correct the record, change publish to true if appropriate. #### Person agents: @@ -172,6 +182,11 @@ Corporate Agent/Creator Relator (2)|String|| If supplying relator, term must be As with Agents, you may associate Subjects with the Archival Object. You may associate up to two Subject records. If you know the Record ID, you may use that instead of the **term**, **type**, and **source** in a manner similar to the way that Agent specifications are made, with the same database lookup and handling done there. Again, if you want the ingest to look up the **term** in the database, you must use the entire Subject header, including any punctuation or capitalization. +If you enter the subject header string *without* the ID, the ingester will try to do an **exact match** against the header; if it finds more than one match (for example, if the database contains two subjects with identical headers, but different sources): + + * The ingester will create a **new** agent (with publish=false) containing the header with ' DISAMBIGUATE ME!' appended to it. For example, given a subject with a header of 'Black Lives Matter', a new subject would be created with the header 'Black Lives Matter DISAMBIGUATE ME!'. + * After ingest, you can use the *merge* functionality to resolve the ambiguities.* + Column | Value | Default | Comment -------|-------|---------|--------- Subject (1) Record ID|Number|| From 0921ceb1f27272a0a1f4fc890f8139fec2545712 Mon Sep 17 00:00:00 2001 From: Bobbi Fox Date: Wed, 8 May 2019 15:40:08 -0400 Subject: [PATCH 5/5] correctly set publish flag --- frontend/controllers/concerns/linked_objects.rb | 7 ++++--- frontend/models/handler.rb | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/controllers/concerns/linked_objects.rb b/frontend/controllers/concerns/linked_objects.rb index 6b54cc0..b54c104 100644 --- a/frontend/controllers/concerns/linked_objects.rb +++ b/frontend/controllers/concerns/linked_objects.rb @@ -53,7 +53,7 @@ def self.get_or_create(row, type, num, resource_uri, report) agent_obj = get_db_agent(agent, resource_uri, num) rescue Exception => e if e.message == 'More than one match found in the database' - agent[:name] = agent[:name] + " DISAMBIGUATE ME!" + agent[:name] = agent[:name] + DISAMB_STR report.add_info(I18n.t('plugins.aspace-import-excel.warn.disam', :name => agent[:name])) else raise e @@ -94,7 +94,7 @@ def self.create_agent(agent, num) begin ret_agent = JSONModel("agent_#{agent[:type]}".to_sym).new._always_valid! ret_agent.names = [name_obj(agent)] - ret_agent.publish = !agent[:id_but_no_name] + ret_agent.publish = !(agent[:id_but_no_name] || agent[:name].ends_with?(DISAMB_STR)) ret_agent.save rescue Exception => e raise Exception.new(I18n.t('plugins.aspace-import-excel.error.no_agent', :num => num, :why => e.message)) @@ -371,7 +371,7 @@ def self.get_or_create(row, num, repo_id, report) subj = get_db_subj(subject) rescue Exception => e if e.message == 'More than one match found in the database' - subject[:term] = subject[:term] + " DISAMBIGUATE ME!" + subject[:term] = subject[:term] + DISAMB_STR report.add_info(I18n.t('plugins.aspace-import-excel.warn.disam', :name => subject[:term])) else raise e @@ -408,6 +408,7 @@ def self.create_subj(subject, num) subj.terms.push term subj.source = subject[:source] subj.vocabulary = '/vocabularies/1' # we're making a gross assumption here + subj.publish = !(subject[:id_but_no_term] || subject[:term].ends_with?(DISAMB_STR)) subj.save rescue Exception => e raise ExcelImportException.new(I18n.t('plugins.aspace-import-excel.error.no_subject',:num => num, :why => e.message)) diff --git a/frontend/models/handler.rb b/frontend/models/handler.rb index 3adc551..4ae3258 100644 --- a/frontend/models/handler.rb +++ b/frontend/models/handler.rb @@ -11,6 +11,8 @@ class Handler require 'enum_list' require 'pp' + DISAMB_STR = ' DISAMBIGUATE ME!' + # centralize the checking for an already-found object def self.stored(hash, id, key) ret_obj = hash.fetch(id, nil) || hash.fetch(key, nil) @@ -41,7 +43,7 @@ def self.search(repo_id,params,jmsym, type = '', match = '') elsif total_hits > 1 if matches.length == 2 match_ct = 0 - disam = matches[1] + " DISAMBIGUATE ME!" + disam = matches[1] + DISAMB_STR disam_obj = nil search['results'].each do |result| # if we have a disambiguate result get it