-
Notifications
You must be signed in to change notification settings - Fork 0
Update typingQC to align with mikrokondo and CDC #21
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: dev
Are you sure you want to change the base?
Changes from all commits
85aeea0
cc7fdf4
165913b
7169029
ef91e9e
7d7f8c5
360f7ba
f6bc13f
a41733d
1ee636b
dc83ff0
df4bb83
abe96cc
2ebe88a
ac3f383
52f0d91
aea4ebd
5491355
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,35 +39,66 @@ def create_serotype_lookup(sample_data_list): | |
|
|
||
| return serotype_lookup | ||
|
|
||
| def determine_validated_serotype(typingqc_message, sample_serotype, approve_verification=False): | ||
| # Determine the validated serotype based on typingQC message and sample serotype. | ||
| # Serovar rename mappings: predicted serovar -> validated reportable name (CDC) | ||
| # Add new entried here as nomenclature changes are required | ||
| SEROVAR_RENAME_MAP = { | ||
| "Panama": "Panama/Houston", | ||
| "Sendai": "Miami" | ||
| } | ||
|
|
||
| # Quality metrics warnings to append when a rename is applied | ||
| SEROVAR_RENAME_WARNINGS = { | ||
| "Sendai": "WARNING: Predicted serovar Sendai has been reported as Miami to conform with current CDC serovar nomenclature." | ||
| } | ||
|
|
||
| def apply_serovar_rename(serotype, quality_metrics): | ||
| # If serotype matched a knoen rename mappung, return the renamed validated serotype and updated quality_metrics string. Otherwsie return the inputs unchanged. | ||
| if serotype not in SEROVAR_RENAME_MAP: | ||
| return serotype, quality_metrics | ||
|
|
||
| renamed = SEROVAR_RENAME_MAP[serotype] | ||
| warning = SEROVAR_RENAME_WARNINGS.get(serotype) | ||
|
|
||
| if warning: | ||
| #Append warning to existing quality_metics content if present | ||
| updated_metrics = f"{quality_metrics}; {warning}" if quality_metrics.strip() else warning | ||
| else: | ||
| updated_metrics = quality_metrics | ||
|
|
||
| return renamed, updated_metrics | ||
|
|
||
| def determine_validated_serotype(typingqc_message, sample_serotype, quality_metrics, approve_verification=False): | ||
| # Determine the validated serotype based on typingQC message and sample serotype. Returns validated_serotype, updated_quality_metrics | ||
| typingqc_upper = str(typingqc_message).upper() | ||
|
|
||
| if 'FAIL' in typingqc_upper: | ||
| return 'pending' | ||
| return 'pending', quality_metrics | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think two returned items is okay for now, but if gets more complicated, then it might be worth making a simple object to encapsulate all of it and return that object back.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed! |
||
|
|
||
| elif 'WARNING' in typingqc_upper: | ||
| if approve_verification: | ||
| # If approving verification, treat like PASS | ||
| if not sample_serotype or sample_serotype.strip() == '': | ||
| return '' | ||
| return sample_serotype | ||
| return '', quality_metrics | ||
| return apply_serovar_rename(sample_serotype, quality_metrics) | ||
| else: | ||
| # Default behavior for warnings | ||
| return 'verification needed' | ||
| return 'verification needed', quality_metrics | ||
|
|
||
| elif 'PASS' in typingqc_upper: | ||
| # If no serotype provided, leave blank | ||
| if not sample_serotype or sample_serotype.strip() == '': | ||
| return '' | ||
| return sample_serotype | ||
| return '', quality_metrics | ||
| return apply_serovar_rename(sample_serotype, quality_metrics) | ||
| else: | ||
| return 'unknown' | ||
| return 'unknown', quality_metrics | ||
|
|
||
| def find_column_indices(header): | ||
| # Find the indices of important columns in the CSV header. | ||
| indices = { | ||
| 'sample': None, | ||
| 'sample_name': None, | ||
| 'typingqc_message': None, | ||
| 'quality_metrics': None, | ||
| 'validated_serotype': None | ||
| } | ||
|
|
||
|
|
@@ -79,13 +110,15 @@ def find_column_indices(header): | |
| indices['sample_name'] = i | ||
| elif col_lower == 'typingqc_message': | ||
| indices['typingqc_message'] = i | ||
| elif col_lower == 'quality_metrics': | ||
| indices['quality_metrics'] = i | ||
| elif col_lower == 'validated_serotype': | ||
| indices['validated_serotype'] = i | ||
|
|
||
| return indices | ||
|
|
||
| def process_csv_file(input_path, serotype_lookup, output_path, approve_verification=False): | ||
| # Process the CSV file and add validated serotype column. | ||
| # Process the CSV file and add validated serotype column. | ||
| rows = [] | ||
|
|
||
| with input_path.open('r', newline='', encoding='utf-8') as csvfile: | ||
|
|
@@ -123,19 +156,27 @@ def process_csv_file(input_path, serotype_lookup, output_path, approve_verificat | |
| if col_indices['typingqc_message'] is not None and len(row) > col_indices['typingqc_message']: | ||
| typingqc_message = row[col_indices['typingqc_message']] | ||
|
|
||
| # Get existing quality_metrics value | ||
| quality_metrics = '' | ||
| if col_indices['quality_metrics'] is not None and len(row) > col_indices['quality_metrics']: | ||
| quality_metrics = row[col_indices['quality_metrics']] | ||
|
|
||
| # Get the serotype for this sample | ||
| sample_serotype = serotype_lookup.get(sample_id, '') | ||
|
|
||
| # Determine validated serotype | ||
| validated_serotype = determine_validated_serotype( | ||
| # Determine validated serotype and updated quality_metrics | ||
| validated_serotype, updated_quality_metrics = determine_validated_serotype( | ||
| typingqc_message, | ||
| sample_serotype, | ||
| approve_verification | ||
| quality_metrics, | ||
| approve_verification, | ||
| ) | ||
|
|
||
| # Set the validated serotype in the appropriate column | ||
| # Write validated serotype and (possibly updated) quality_metrics back to row | ||
| if col_indices['validated_serotype'] is not None: | ||
| row[col_indices['validated_serotype']] = validated_serotype | ||
| if col_indices['quality_metrics'] is not None: | ||
| row[col_indices['quality_metrics']] = updated_quality_metrics | ||
|
|
||
| rows.append(row) | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,8 @@ | ||
| sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name | ||
| SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7 | ||
| SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7 | ||
| SAMPLE3,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,IV Y:gz51:- | ||
| SAMPLE4,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum | ||
| SAMPLE5,SALM003,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM003.flat_sample.json.gz,Salmonella enterica,FAILED,Kentucky | ||
| SAMPLE6,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED, | ||
| SAMPLE7,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED, | ||
| SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7 | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Reminder to update these paths later. |
||
| SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7 | ||
| SAMPLE3,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,IV Y:gz51:- | ||
| SAMPLE4,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum | ||
| SAMPLE5,SALM003,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM003.flat_sample.json.gz,Salmonella enterica,FAILED,Kentucky | ||
| SAMPLE6,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED, | ||
| SAMPLE7,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED, | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,8 @@ | ||
| sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name | ||
| SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7 | ||
| SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7 | ||
| SAMPLE3,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2 | ||
| SAMPLE4,EC005,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC005.flat_sample.json.gz,Escherichia coli,PASSED,O2/O137:H41 | ||
| SAMPLE5,EC006,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC006.flat_sample.json.gz,Escherichia coli,PASSED,-:H21 | ||
| SAMPLE6,EC007,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC007.flat_sample.json.gz,Escherichia coli,PASSED,O111:H8 | ||
| SAMPLE8,EA001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EA001.flat_sample.json.gz,Escherichia coli,PASSED, | ||
| SAMPLE1,EC001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC001.flat_sample.json.gz,Escherichia coli,PASSED,O157:H7 | ||
| SAMPLE2,EC002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC002.flat_sample.json.gz,Escherichia coli,PASSED,O55:H7 | ||
| SAMPLE3,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2 | ||
| SAMPLE4,EC005,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC005.flat_sample.json.gz,Escherichia coli,PASSED,O2/O137:H41 | ||
| SAMPLE5,EC006,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC006.flat_sample.json.gz,Escherichia coli,PASSED,-:H21 | ||
| SAMPLE6,EC007,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC007.flat_sample.json.gz,Escherichia coli,PASSED,O111:H8 | ||
| SAMPLE8,EA001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EA001.flat_sample.json.gz,Escherichia coli,PASSED, |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name | ||
| SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,"IV Y:g,z51:-" | ||
| SAMPLE2,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum | ||
| SAMPLE4,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED, | ||
| SAMPLE5,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/CAMP001.flat_sample.json.gz,Campylobacter_coli,FAILED, | ||
| SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella houtenae,PASSED,"IV Y:g,z51:-" | ||
| SAMPLE2,SALM004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM004.flat_sample.json.gz,Salmonella enterica,PASSED,Gallinarum | ||
| SAMPLE4,LIST001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/LIST001.flat_sample.json.gz,Listeria monocytogenes,PASSED, | ||
| SAMPLE5,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/CAMP001.flat_sample.json.gz,Campylobacter_coli,FAILED, |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,2 @@ | ||
| sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name | ||
| SAMPLE1,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,Souza|Madjorio|Liverpool | ||
| SAMPLE1,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,Souza|Madjorio|Liverpool |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| sample,sample_name,mikrokondo_data,predicted_identification_name,qc_status_overall,predicted_primary_type_name | ||
| SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,"IV Y:g,z51:-" | ||
| SAMPLE2,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM002.flat_sample.json.gz,Salmonella_enterica,FAILED,Souza|Madjorio|Liverpool | ||
| SAMPLE4,SALM005,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/SALM005.flat_sample.json.gz,Salmonella_enterica,PASSED,Paratyphi A | ||
| SAMPLE6,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/CAMP001.flat_sample.json.gz,Campylobacter coli,FAILED, | ||
| SAMPLE7,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/dev/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2 | ||
| SAMPLE1,SALM001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM001.flat_sample.json.gz,Salmonella_houtenae,PASSED,"IV Y:g,z51:-" | ||
| SAMPLE2,SALM002,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM002.flat_sample.json.gz,Salmonella_diarizonae,FAILED,"IIIb T:z35:z35" | ||
| SAMPLE4,SALM005,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/SALM005.flat_sample.json.gz,Salmonella_enterica,PASSED,Paratyphi A | ||
| SAMPLE6,CAMP001,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/CAMP001.flat_sample.json.gz,Campylobacter coli,FAILED, | ||
| SAMPLE7,EC004,https://raw.githubusercontent.com/phac-nml/typingQC/update/tests/data/EC004.flat_sample.json.gz,Escherichia coli,FAILED,O103:H2 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If I'm understanding this correctly that this is parsing SISTR output (and not output we easily have control over), then this seem reasonable to me.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes exactly - it causes a lot of extra text in IRIDA Next metadata fields and serves no use for those in the lab who are analyzing the data!