Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/linting.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ jobs:

- name: Install Nextflow
uses: nf-core/setup-nextflow@v2
version: 25.10

- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/linting_comment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Download lint results
uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6
with:
workflow: linting.yml
workflow_conclusion: completed
Expand Down
7 changes: 6 additions & 1 deletion bin/parse_sequenceQC.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,12 @@ def load_json(path):
"n50_value",
"nr_contigs",
"length",
"wgmlst_loci_count",
}

CHECKM_TEST = "checkm_contamination"
CHECKM_TEST = "checkm2_contamination"

WGMLST_TEST = "wgmlst_loci_count"

PNC_FAIL_MESSAGE = (
"[SEQ_FAIL] Resequencing is recommended. QUALITY_METRICS did not meet PNC requirements."
Expand Down Expand Up @@ -96,6 +99,8 @@ def build_typingQC_message(sample_data, species, failed_tests):
base_message = CHECKM_FAIL_MESSAGE
elif failed_tests.intersection(ABSOLUTE_FAIL_TESTS):
base_message = PNC_FAIL_MESSAGE
elif WGMLST_TEST in failed_tests:
base_message = PNC_FAIL_MESSAGE
elif failed_tests.intersection(WARNING_TESTS):
base_message = WARNING_MESSAGE
else:
Expand Down
37 changes: 34 additions & 3 deletions bin/parse_sistrQC.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
import csv
import argparse
import re
from pathlib import Path
import gzip

Expand Down Expand Up @@ -56,6 +57,29 @@ def load_reportable_serovars(reportable_file):
SEROVAR_KEY = f"{SISTR_PREFIX}serovar"
SEROVAR_CGMLST = f"{SISTR_PREFIX}serovar_cgmlst"


#QC_MESSAGES Updates
#Matches the non-actionable cgMLST loci found INFO message and strips it from qc_messages
CGMLST_LOCI_INFO_PATTERN = re.compile(r"INFO: Number of cgMLST\d+ loci found \(n=\d+\)")
def filter_qc_messages(qc_messages):
Comment on lines +61 to +64

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I'm understanding this correctly that this is parsing SISTR output (and not output we easily have control over), then this seem reasonable to me.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes exactly - it causes a lot of extra text in IRIDA Next metadata fields and serves no use for those in the lab who are analyzing the data!

if not qc_messages:
return qc_messages
cleaned = CGMLST_LOCI_INFO_PATTERN.sub("", qc_messages)
# Collapse leftover separators/whitespace left behind by the removal
cleaned = re.sub(r"\s*\|\s*", " | ", cleaned)
cleaned = re.sub(r"^\s*\|\s*|\s*\|\s*$", "", cleaned)
cleaned = re.sub(r"\n\s*\n", "\n", cleaned)
return cleaned.strip()

WZX_WZY_PATTERN = "Wzx/Wzy genes missing"
WZX_WZY_WARNING_MESSAGE = "WARNING: Wzx/Wzy genes missing. Cannot determine O-antigen serogroup."

def wzx_wzy_only_failure(qc_messages):
#Checks if a SISTR FAIL is solely due to missing Wzx/Wzy genes: does not affect serovar prediction and should not result in FAIL for typingQC
if not qc_messages:
return False
return WZX_WZY_PATTERN in qc_messages

def H1_warning(qc_messages):
# Check if SISTR WARNING contains identification of the inability to identify H1 antigens, and therefore, unable to predict serovar repliably
if not qc_messages:
Expand All @@ -70,12 +94,16 @@ def H1_warning(qc_messages):

def extract_sistr_qc(sample_data):
qc_status = sample_data.get(QC_STATUS_KEY, "Unknown")
qc_messages = sample_data.get(QC_MESSAGES_KEY, "")
qc_messages = filter_qc_messages(sample_data.get(QC_MESSAGES_KEY, ""))

if qc_status.upper() == "PASS":
# For PASS samples, leave QUALITY_ANALYSIS column blank
return ""

if qc_status.upper() == "FAIL" and wzx_wzy_only_failure(qc_messages):
# O-antigen serogroup couldn't be determined, but this doesn't block typingQC and reports it as a warning rather than the raw FAIL message.
return WZX_WZY_WARNING_MESSAGE

elif qc_status.upper() == "WARNING":
# If sample raises a WARNING in qc_status record the QC_message
return qc_messages if qc_messages else "SISTR analysis completed with warnings. Please review results manually."
Expand All @@ -86,13 +114,16 @@ def extract_sistr_qc(sample_data):

def build_typingQC_message(sample_data, reportable_serovars):
qc_status = sample_data.get(QC_STATUS_KEY, "Unknown")
qc_messages = sample_data.get(QC_MESSAGES_KEY, "")
qc_messages = filter_qc_messages(sample_data.get(QC_MESSAGES_KEY, ""))

# Check for H1 antigen warning from SISTR and treat it as an RDS [FAIL]
if qc_status.upper() == "WARNING" and H1_warning(qc_messages):
return "[SISTR_FAIL] Serotyping unsuccessful. RESEQUENCING or TRADITIONAL SEROTYPING is advised."

if qc_status.upper() in ["PASS", "WARNING"]:
# A FAIL solely due to missing Wzx/Wzy genes: fall through to the serovar reportability check below and prepend a warning instead of SISTR_FAIL
is_wzx_wzy_fail = qc_status.upper() == "FAIL" and wzx_wzy_only_failure(qc_messages)

if qc_status.upper() in ["PASS", "WARNING"] or is_wzx_wzy_fail:
# Extract serovar predictions
serovar = sample_data.get(SEROVAR_KEY, "")
serovar_cgmlst = sample_data.get(SEROVAR_CGMLST, "")
Expand Down
69 changes: 55 additions & 14 deletions bin/validate_serotypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,35 +39,66 @@ def create_serotype_lookup(sample_data_list):

return serotype_lookup

def determine_validated_serotype(typingqc_message, sample_serotype, approve_verification=False):
# Determine the validated serotype based on typingQC message and sample serotype.
# Serovar rename mappings: predicted serovar -> validated reportable name (CDC)
# Add new entried here as nomenclature changes are required
SEROVAR_RENAME_MAP = {
"Panama": "Panama/Houston",
"Sendai": "Miami"
}

# Quality metrics warnings to append when a rename is applied
SEROVAR_RENAME_WARNINGS = {
"Sendai": "WARNING: Predicted serovar Sendai has been reported as Miami to conform with current CDC serovar nomenclature."
}

def apply_serovar_rename(serotype, quality_metrics):
# If serotype matched a knoen rename mappung, return the renamed validated serotype and updated quality_metrics string. Otherwsie return the inputs unchanged.
if serotype not in SEROVAR_RENAME_MAP:
return serotype, quality_metrics

renamed = SEROVAR_RENAME_MAP[serotype]
warning = SEROVAR_RENAME_WARNINGS.get(serotype)

if warning:
#Append warning to existing quality_metics content if present
updated_metrics = f"{quality_metrics}; {warning}" if quality_metrics.strip() else warning
else:
updated_metrics = quality_metrics

return renamed, updated_metrics

def determine_validated_serotype(typingqc_message, sample_serotype, quality_metrics, approve_verification=False):
# Determine the validated serotype based on typingQC message and sample serotype. Returns validated_serotype, updated_quality_metrics
typingqc_upper = str(typingqc_message).upper()

if 'FAIL' in typingqc_upper:
return 'pending'
return 'pending', quality_metrics

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think two returned items is okay for now, but if gets more complicated, then it might be worth making a simple object to encapsulate all of it and return that object back.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed!


elif 'WARNING' in typingqc_upper:
if approve_verification:
# If approving verification, treat like PASS
if not sample_serotype or sample_serotype.strip() == '':
return ''
return sample_serotype
return '', quality_metrics
return apply_serovar_rename(sample_serotype, quality_metrics)
else:
# Default behavior for warnings
return 'verification needed'
return 'verification needed', quality_metrics

elif 'PASS' in typingqc_upper:
# If no serotype provided, leave blank
if not sample_serotype or sample_serotype.strip() == '':
return ''
return sample_serotype
return '', quality_metrics
return apply_serovar_rename(sample_serotype, quality_metrics)
else:
return 'unknown'
return 'unknown', quality_metrics

def find_column_indices(header):
# Find the indices of important columns in the CSV header.
indices = {
'sample': None,
'sample_name': None,
'typingqc_message': None,
'quality_metrics': None,
'validated_serotype': None
}

Expand All @@ -79,13 +110,15 @@ def find_column_indices(header):
indices['sample_name'] = i
elif col_lower == 'typingqc_message':
indices['typingqc_message'] = i
elif col_lower == 'quality_metrics':
indices['quality_metrics'] = i
elif col_lower == 'validated_serotype':
indices['validated_serotype'] = i

return indices

def process_csv_file(input_path, serotype_lookup, output_path, approve_verification=False):
# Process the CSV file and add validated serotype column.
# Process the CSV file and add validated serotype column.
rows = []

with input_path.open('r', newline='', encoding='utf-8') as csvfile:
Expand Down Expand Up @@ -123,19 +156,27 @@ def process_csv_file(input_path, serotype_lookup, output_path, approve_verificat
if col_indices['typingqc_message'] is not None and len(row) > col_indices['typingqc_message']:
typingqc_message = row[col_indices['typingqc_message']]

# Get existing quality_metrics value
quality_metrics = ''
if col_indices['quality_metrics'] is not None and len(row) > col_indices['quality_metrics']:
quality_metrics = row[col_indices['quality_metrics']]

# Get the serotype for this sample
sample_serotype = serotype_lookup.get(sample_id, '')

# Determine validated serotype
validated_serotype = determine_validated_serotype(
# Determine validated serotype and updated quality_metrics
validated_serotype, updated_quality_metrics = determine_validated_serotype(
typingqc_message,
sample_serotype,
approve_verification
quality_metrics,
approve_verification,
)

# Set the validated serotype in the appropriate column
# Write validated serotype and (possibly updated) quality_metrics back to row
if col_indices['validated_serotype'] is not None:
row[col_indices['validated_serotype']] = validated_serotype
if col_indices['quality_metrics'] is not None:
row[col_indices['quality_metrics']] = updated_quality_metrics

rows.append(row)

Expand Down
Binary file modified tests/data/CAMP001.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EA001.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EC001.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EC002.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EC004.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EC005.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EC006.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/EC007.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/LIST001.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM001.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM002.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM003.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM004.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM005.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM007.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM008.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM009.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM010.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM011.flat_sample.json.gz
Binary file not shown.
Binary file modified tests/data/SALM012.flat_sample.json.gz
Binary file not shown.
Binary file added tests/data/SALM013.flat_sample.json.gz
Binary file not shown.
Binary file added tests/data/SALM014.flat_sample.json.gz
Binary file not shown.
14 changes: 7 additions & 7 deletions tests/data/samplesheets/samplesheet1.csv
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name
SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7
SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7
SAMPLE3,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,IV Y:gz51:-
SAMPLE4,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum
SAMPLE5,SALM003,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM003.flat_sample.json.gz,Salmonella enterica,FAILED,Kentucky
SAMPLE6,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED,
SAMPLE7,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED,
SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reminder to update these paths later.

SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7
SAMPLE3,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,IV Y:gz51:-
SAMPLE4,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum
SAMPLE5,SALM003,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM003.flat_sample.json.gz,Salmonella enterica,FAILED,Kentucky
SAMPLE6,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED,
SAMPLE7,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED,
14 changes: 7 additions & 7 deletions tests/data/samplesheets/samplesheet_ectyper.csv
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name
SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7
SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7
SAMPLE3,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2
SAMPLE4,EC005,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC005.flat_sample.json.gz,Escherichia coli,PASSED,O2/O137:H41
SAMPLE5,EC006,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC006.flat_sample.json.gz,Escherichia coli,PASSED,-:H21
SAMPLE6,EC007,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC007.flat_sample.json.gz,Escherichia coli,PASSED,O111:H8
SAMPLE8,EA001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EA001.flat_sample.json.gz,Escherichia coli,PASSED,
SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7
SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7
SAMPLE3,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2
SAMPLE4,EC005,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC005.flat_sample.json.gz,Escherichia coli,PASSED,O2/O137:H41
SAMPLE5,EC006,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC006.flat_sample.json.gz,Escherichia coli,PASSED,-:H21
SAMPLE6,EC007,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC007.flat_sample.json.gz,Escherichia coli,PASSED,O111:H8
SAMPLE8,EA001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EA001.flat_sample.json.gz,Escherichia coli,PASSED,
8 changes: 4 additions & 4 deletions tests/data/samplesheets/samplesheet_exclusions.csv
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name
SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,"IV Y:g,z51:-"
SAMPLE2,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum
SAMPLE4,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED,
SAMPLE5,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/CAMP001.flat_sample.json.gz,Campylobacter_coli,FAILED,
SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,"IV Y:g,z51:-"
SAMPLE2,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum
SAMPLE4,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED,
SAMPLE5,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/CAMP001.flat_sample.json.gz,Campylobacter_coli,FAILED,
2 changes: 1 addition & 1 deletion tests/data/samplesheets/samplesheet_incorrectJSON.csv
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name
SAMPLE1,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,Souza|Madjorio|Liverpool
SAMPLE1,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,Souza|Madjorio|Liverpool
10 changes: 5 additions & 5 deletions tests/data/samplesheets/samplesheet_sequence.csv
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name
SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,"IV Y:g,z51:-"
SAMPLE2,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM002.flat_sample.json.gz,Salmonella_enterica,FAILED,Souza|Madjorio|Liverpool
SAMPLE4,SALM005,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM005.flat_sample.json.gz,Salmonella_enterica,PASSED,Paratyphi A
SAMPLE6,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/CAMP001.flat_sample.json.gz,Campylobacter coli,FAILED,
SAMPLE7,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2
SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,"IV Y:g,z51:-"
SAMPLE2,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM002.flat_sample.json.gz,Salmonella_diarizonae,FAILED,"IIIb T:z35:z35"
SAMPLE4,SALM005,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM005.flat_sample.json.gz,Salmonella_enterica,PASSED,Paratyphi A
SAMPLE6,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/CAMP001.flat_sample.json.gz,Campylobacter coli,FAILED,
SAMPLE7,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2
Loading
Loading