diff --git a/CHANGELOG.md b/CHANGELOG.md index 29699099a..db358454c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Added` +- Parameters `manta_call_regions`, `ml_prob_threshold`, `sv_freq_filter_expression`, `filter_expansionhunter_htt`, `filter_sv_to_manta`, `skip_vep_sv`, and `sv_size_threshold` to expose optional filtering and calling controls as pipeline parameters [#](https://github.com/nf-core/raredisease/pull/) - Interval parameter in the default retroseq call [#717](https://github.com/nf-core/raredisease/pull/717) - Tests for call_repeat_expansions and qc_bam subworkflows [#713](https://github.com/nf-core/raredisease/pull/713) - Feature to subsample mitochondrial alignments based on number of reads [#748](https://github.com/nf-core/raredisease/pull/748) @@ -52,6 +53,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `--skip_split_multiallelics` parameter to allow users to skip the `bcftools norm --multiallelics -both` step in SNV calling (DeepVariant and Sentieon), which can cause indel quality degradation in single-interval runs [#823](https://github.com/nf-core/raredisease/pull/823) - Extended vcfanno database sanity check to include extra vcfanno resources (`vcfanno_extra`) alongside the main resources, and moved the check upstream to `raredisease.nf` so it covers both genome and mitochondrial SNV annotation subworkflows [#834](https://github.com/nf-core/raredisease/pull/834) +### `Fixed` + +- Remove `imNotification` import and call from `utils_nfcore_raredisease_pipeline` after it was removed in the updated `utils_nfcore_pipeline` subworkflow [#](https://github.com/nf-core/raredisease/pull/) +- Use conditional output prefix for `BCFTOOLS_DECOMPRESS_MERGE` in `call_repeat_expansions` so that the file is named `_repeat_expansion.vcf` by default and `_filtered_htt_exphunter.vcf` only when `--filter_expansionhunter_htt` is set [#](https://github.com/nf-core/raredisease/pull/) + ### `Changed` - Use distinct output filenames for bcfools (in call_mobile_elements subworkflow) and svdb (in call_sv_tiddit subworkflow) [#716](https://github.com/nf-core/raredisease/pull/716) diff --git a/conf/modules/annotate_structural_variants.config b/conf/modules/annotate_structural_variants.config index b9c98b920..23bdec80d 100644 --- a/conf/modules/annotate_structural_variants.config +++ b/conf/modules/annotate_structural_variants.config @@ -33,6 +33,15 @@ process { ext.prefix = { "${meta.id}_view" } } + withName: '.*ANNOTATE_STRUCTURAL_VARIANTS:BCFTOOLS_FILTER_SV' { + ext.args = { + params.sv_freq_filter_expression + ? "-e '${params.sv_freq_filter_expression}' -Oz --write-index=tbi" + : "-Oz --write-index=tbi" + } + ext.prefix = { "${meta.id}_sv_filtered" } + } + withName: '.*ANNOTATE_STRUCTURAL_VARIANTS:ENSEMBLVEP_SV' { ext.args = { [ '--dir_cache cache', diff --git a/conf/modules/call_repeat_expansions.config b/conf/modules/call_repeat_expansions.config index 88b4320f2..56f072007 100644 --- a/conf/modules/call_repeat_expansions.config +++ b/conf/modules/call_repeat_expansions.config @@ -36,6 +36,20 @@ process { ext.prefix = { "${meta.id}_split_exp" } } + withName: '.*CALL_REPEAT_EXPANSIONS:BCFTOOLS_FILTER_HTT' { + ext.args = "-e 'INFO/REPID=\"HTT\" || INFO/REPID~\"^HTT_\"' -Oz --write-index=tbi" + ext.prefix = { "${meta.id}_filtered_exp" } + } + + withName: '.*CALL_REPEAT_EXPANSIONS:BCFTOOLS_DECOMPRESS_MERGE' { + ext.args = '--output-type v' + ext.prefix = { + params.filter_expansionhunter_htt + ? "${meta.id}_filtered_htt_exphunter" + : "${meta.id}_repeat_expansion" + } + } + withName: '.*CALL_REPEAT_EXPANSIONS:SVDB_MERGE_REPEATS' { ext.args = {"--notag"} } diff --git a/conf/modules/call_snv_sentieon.config b/conf/modules/call_snv_sentieon.config index 39085f55d..2d273158d 100644 --- a/conf/modules/call_snv_sentieon.config +++ b/conf/modules/call_snv_sentieon.config @@ -29,8 +29,16 @@ process { } withName: '.*CALL_SNV:CALL_SNV_SENTIEON:BCF_FILTER_ONE' { - ext.args = "-s 'ML_FAIL' -i 'INFO/ML_PROB <= 0.95' -m x --output-type z --write-index=tbi" - ext.prefix = { "${meta.id}_mlfiltered_0.95" } + ext.args = { + params.ml_prob_threshold + ? "-s 'ML_FAIL' -i 'INFO/ML_PROB <= ${params.ml_prob_threshold}' -m x -Oz --write-index=tbi" + : "-s 'ML_FAIL' -e 'FILTER==\"MLrejected\"' -m x -Oz --write-index=tbi" + } + ext.prefix = { + params.ml_prob_threshold + ? "${meta.id}_mlfiltered_${params.ml_prob_threshold}" + : "${meta.id}_mlfiltered" + } } withName: '.*CALL_SNV:CALL_SNV_SENTIEON:BCF_FILTER_TWO' { diff --git a/conf/modules/call_sv_manta.config b/conf/modules/call_sv_manta.config index 52b1d7875..346b7dd53 100644 --- a/conf/modules/call_sv_manta.config +++ b/conf/modules/call_sv_manta.config @@ -17,10 +17,17 @@ process { withName: '.*CALL_STRUCTURAL_VARIANTS:CALL_SV_MANTA:MANTA' { - ext.args = { (params.analysis_type == "wes") ? '--exome' : '' } + ext.args = { + def args = (params.analysis_type == "wes") ? '--exome' : '' + params.manta_call_regions ? [args, "--callRegions ${params.manta_call_regions}"].findAll { it }.join(' ') : args + } } withName: ".*CALL_STRUCTURAL_VARIANTS:CALL_SV_MANTA:BCFTOOLS_VIEW_MANTA" { ext.prefix = { "${meta.id}_manta" } - ext.args = { '--apply-filters .,PASS --output-type z' } + ext.args = { + def args = '--apply-filters .,PASS' + if (params.sv_size_threshold) args += " -e 'abs(INFO/SVLEN) > ${params.sv_size_threshold}'" + args + ' -Oz --write-index=tbi' + } } } diff --git a/conf/modules/qc_bam.config b/conf/modules/qc_bam.config index 52444ff73..55324e94b 100644 --- a/conf/modules/qc_bam.config +++ b/conf/modules/qc_bam.config @@ -22,8 +22,9 @@ process { } withName: '.*QC_BAM:PICARD_COLLECTHSMETRICS' { - ext.args = "--TMP_DIR ." + ext.args = "--TMP_DIR ." ext.prefix = { "${meta.id}_hsmetrics" } + ext.when = { params.target_bed != null } } withName: '.*QC_BAM:TIDDIT_COV' { diff --git a/docs/output.md b/docs/output.md index 7e486eb1f..96bf9ec6c 100644 --- a/docs/output.md +++ b/docs/output.md @@ -279,6 +279,10 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ The pipeline performs variant calling using [Sentieon DNAscope](https://support.sentieon.com/manual/DNAscope_usage/dnascope/) with a machine learning model. This approach identifies the candidate sites with a higher accuracy, and calculates genotypes for each sample at that site. These files are treated as intermediates and are not placed in the output folder by default. DNAscope is not run by default. To use DNAscope instead of DeepVariant, set `--variant_caller` to sentieon. +Variant filtering behaviour can be controlled with an optional parameter: + +- `--ml_prob_threshold`: When set (e.g. `0.6`), variants with `INFO/ML_PROB <= threshold` are tagged `ML_FAIL` and subsequently removed. When not set, variants already tagged `MLrejected` by Sentieon are tagged `ML_FAIL` instead. +
Output files @@ -294,6 +298,12 @@ The pipeline performs variant calling using [Sentieon DNAscope](https://support. [Manta](https://github.com/Illumina/manta) calls structural variants (SVs) and indels from mapped paired-end sequencing reads. It combines paired and split-read evidence during SV discovery and scoring to improve accuracy, but does not require split-reads or successful breakpoint assemblies to report a variant in cases where there is strong evidence otherwise. Output vcf files are treated as intermediates and are not placed in the output folder. +Two optional parameters affect Manta calling and the downstream SV merge: + +- `--manta_call_regions`: Path to a bgzipped and indexed BED file restricting Manta to call only within specified genomic regions (passed via `--callRegions`). Useful for WES or targeted panels, or to exclude decoy and unplaced sequences from SV calling. +- `--sv_size_threshold`: Manta SVs larger than this value (in bp) are removed before merging. For example, `--sv_size_threshold 1000000` removes SVs larger than 1 Mb. +- `--filter_sv_to_manta`: When `true`, only Manta variants are included in the SVDB merge step; TIDDIT variants are excluded. + #### TIDDIT sv [TIDDIT's sv](https://github.com/SciLifeLab/TIDDIT) is used to identify chromosomal rearrangements using sequencing data. TIDDIT identifies intra and inter-chromosomal translocations, deletions, tandem-duplications and inversions, using supplementary alignments as well as discordant pairs. TIDDIT searches for discordant reads and split reads (supplementary alignments). Output vcf files are treated as intermediates and are not placed in the output folder. @@ -325,11 +335,14 @@ The pipeline performs variant calling using [Sentieon DNAscope](https://support. [Expansion Hunter](https://github.com/Illumina/ExpansionHunter) aims to estimate sizes of repeat sequences by performing a targeted search through alignments that span, flank, and are fully contained in each repeat. The files generated are ready to be used with [REViewer](https://github.com/Illumina/REViewer). +The optional parameter `--filter_expansionhunter_htt` removes HTT (Huntington's disease) records from the ExpansionHunter output before downstream processing. This is useful when HTT calls are handled separately or are not clinically relevant in the current analysis. +
Output files - `repeat_expansions/` - - `_repeat_expansion.vcf`: file containing variant calls. + - `_repeat_expansion.vcf`: file containing variant calls (default). + - `_filtered_htt_exphunter.vcf`: file containing variant calls with HTT records removed (when `--filter_expansionhunter_htt` is set). - `_exphunter_sorted.bam`: A BAMlet containing alignments of reads that overlap or located in close proximity to each variant identified by ExpansionHunter - `_exphunter_sorted.bam.bai`: Index of the BAMlet file @@ -427,19 +440,24 @@ Based on VEP annotations, custom scripts used by the pipeline further annotate e #### SVDB query -[SVDB query](https://github.com/J35P312/SVDB#Query) allows you to quickly annotate your VCF with data from one or more structural variant databases. The output files are not published in the output folder, and is passed to vep for further annotation. +[SVDB query](https://github.com/J35P312/SVDB#Query) allows you to quickly annotate your VCF with data from one or more structural variant databases. The output files are not published in the output folder, and is passed to a frequency filter step and then to VEP for further annotation. + +After SVDB query, SVs can optionally be filtered by population frequency using `--sv_freq_filter_expression`. This parameter accepts a full [bcftools -e expression](https://samtools.github.io/bcftools/bcftools.html#expressions), for example `'INFO/SWEFRQ >= 0.02'`. Multiple conditions can be combined, e.g. `'INFO/SWEFRQ >= 0.02 || INFO/AF >= 0.02'`. When not set, the filtering step is skipped entirely. #### VEP [VEP](https://www.ensembl.org/info/docs/tools/vep/index.html) determines the effect of your variants on genes, transcripts, and protein sequence, as well as regulatory regions. We recommend annotating with pLI plugin, along with any other custom plugins you may want too use. Based on VEP annotations, custom scripts used by the pipeline further annotate each record with the most severe consequence, and pli scores. +VEP annotation of structural variants can be skipped by setting `--skip_vep_sv true`. SVDB query and frequency filtering still run; only the Picard sort, bcftools view, and VEP steps are skipped. In that case the frequency-filtered VCF is published directly. +
Output files - `annotate_sv/` - - `_svdbquery_vep.vcf.gz`: file containing svdb query, and vep annotations. - - `_svdbquery_vep.vcf.gz.tbi`: index of the file containing bcftools roh, vcfanno, and vep annotations. + - `_svdbquery_vep.vcf.gz`: file containing svdb query, frequency filter, and vep annotations. Published when `--skip_vep_sv` is not set. + - `_svdbquery_vep.vcf.gz.tbi`: index of the annotated SV vcf file. - `_svdbquery_vep_summary.html`: vep summary. + - `_sv_filtered.vcf.gz`: frequency-filtered SV vcf. Published instead of the VEP output when `--skip_vep_sv true` is set.
diff --git a/docs/usage.md b/docs/usage.md index e5963d4d2..870661891 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -238,14 +238,23 @@ The mandatory and optional parameters for each category are tabulated below. ##### 3. Repeat expansions -| Mandatory | Optional | -| --------------------------- | -------- | -| variant_catalog1 | | +| Mandatory | Optional | +| --------------------------- | ------------------------------------ | +| variant_catalog1 | filter_expansionhunter_htt2 | -1 We reccomend using the catalogs found [here](https://github.com/Clinical-Genomics/reference-files/tree/master/rare-disease/disease_loci/ExpansionHunter-v5.0.0). These catalogs have been extended from the illumina ones to include information on pathogenicity, which is neccesarry for the workflow. +1 We reccomend using the catalogs found [here](https://github.com/Clinical-Genomics/reference-files/tree/master/rare-disease/disease_loci/ExpansionHunter-v5.0.0). These catalogs have been extended from the illumina ones to include information on pathogenicity, which is neccesarry for the workflow.
+2 Set to `true` to remove HTT (Huntington's disease) records from ExpansionHunter output before downstream processing. Default is `false`. ##### 4. Variant calling - SNV +| Mandatory | Optional | +| -------------------------- | ------------------------------------- | +| variant_caller1 | known_dbsnp2 | +| ml_model2 | known_dbsnp_tbi2 | +| analysis_type3 | call_interval2 | +| | known_dbsnp_tbi2 | +| | par_bed4 | +| | ml_prob_threshold2 | | Mandatory | Optional | | -------------------------- | ------------------------------------ | | variant_caller1 | known_dbsnp2 | @@ -263,10 +272,17 @@ The mandatory and optional parameters for each category are tabulated below. ##### 5. Variant calling - Structural variants -| Mandatory | Optional | -| --------- | ---------- | -| | target_bed | -| | bwa | +| Mandatory | Optional | +| --------- | --------------------------------- | +| | target_bed | +| | bwa | +| | manta_call_regions1 | +| | sv_size_threshold2 | +| | filter_sv_to_manta3 | + +1 BED.gz file (with `.tbi` index) restricting Manta SV calling to specific regions. Useful for WES, targeted panels, or to exclude decoy and unplaced sequences.
+2 Filter Manta SVs larger than this value (in bp) before SVDB merge. For example, `--sv_size_threshold 1000000` removes SVs larger than 1 Mb. Default is `null` (no filter).
+3 Set to `true` to include only Manta variants in the SVDB merge step, excluding TIDDIT. Default is `false`.
##### 6. Copy number variant calling @@ -317,17 +333,19 @@ We use CADD only to annotate small indels. To annotate SNVs with precomputed CAD ##### 8. SV annotation & Ranking -| Mandatory | Optional | -| ---------------------------------------------- | --------------------------------- | -| genome | reduced_penetrance | -| svdb_query_dbs/svdb_query_bedpedbs1 | | -| vep_cache_version | vep_filters/vep_filters_scout_fmt | -| vep_cache | vep_plugin_files | -| score_config_sv | | -| variant_consequences_sv2 | | - -1 A CSV file that describes the databases (VCFs or BEDPEs) used by SVDB for annotating structural variants. Sample file [here](https://github.com/nf-core/test-datasets/blob/raredisease/reference/svdb_querydb_files.csv). Information about the column headers can be found [here](https://github.com/J35P312/SVDB#Query). -2 File containing list of SO terms listed in the order of severity from most severe to lease severe for annotating genomic SVs. Sample file [here](https://github.com/nf-core/test-datasets/blob/raredisease/reference/variant_consequences_v2.txt). You can learn more about these terms [here](https://grch37.ensembl.org/info/genome/variation/prediction/predicted_data.html). +| Mandatory | Optional | +| ---------------------------------------------- | ------------------------------------- | +| genome | reduced_penetrance | +| svdb_query_dbs/svdb_query_bedpedbs1 | vep_filters/vep_filters_scout_fmt | +| vep_cache_version | vep_plugin_files | +| vep_cache | sv_freq_filter_expression3 | +| score_config_sv | skip_vep_sv4 | +| variant_consequences_sv2 | | + +1 A CSV file that describes the databases (VCFs or BEDPEs) used by SVDB for annotating structural variants. Sample file [here](https://github.com/nf-core/test-datasets/blob/raredisease/reference/svdb_querydb_files.csv). Information about the column headers can be found [here](https://github.com/J35P312/SVDB#Query).
+2 File containing list of SO terms listed in the order of severity from most severe to lease severe for annotating genomic SVs. Sample file [here](https://github.com/nf-core/test-datasets/blob/raredisease/reference/variant_consequences_v2.txt). You can learn more about these terms [here](https://grch37.ensembl.org/info/genome/variation/prediction/predicted_data.html).
+3 A bcftools `-e` expression to filter SVs by population frequency after SVDB query, e.g. `'INFO/SWEFRQ >= 0.02'`. Supports any INFO field and multiple conditions. Default is `null` (no filter).
+4 Set to `true` to skip VEP annotation of structural variants. SVDB query and frequency filtering still run. Default is `false`.
##### 9. Mitochondrial annotation diff --git a/main.nf b/main.nf index f847e20e6..95a0ec700 100644 --- a/main.nf +++ b/main.nf @@ -346,7 +346,7 @@ workflow NFCORE_RAREDISEASE { // if (skip_germlinecnvcaller) { if (val_analysis_type.equals("wgs")) { - ch_svcaller_priority = channel.value(["tiddit", "manta", "cnvnator"]) + ch_svcaller_priority = channel.value(["tiddit", "manta"]) } else { ch_svcaller_priority = channel.value([]) } @@ -354,7 +354,7 @@ workflow NFCORE_RAREDISEASE { if (val_analysis_type.equals("wgs")) { ch_svcaller_priority = channel.value(["tiddit", "manta", "gcnvcaller", "cnvnator"]) } else { - ch_svcaller_priority = channel.value(["manta", "gcnvcaller"]) + ch_svcaller_priority = channel.value(["manta", "gcnvcaller", "cnvnator"]) } } diff --git a/modules.json b/modules.json index 4bb0c67b5..e11e6ea2e 100644 --- a/modules.json +++ b/modules.json @@ -572,12 +572,12 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "271e7fc14eb1320364416d996fb077421f3faed2", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", "installed_by": ["subworkflows"] }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "4b406a74dc0449c0401ed87d5bfff4252fd277fd", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", "installed_by": ["subworkflows"] }, "vcf_filter_bcftools_ensemblvep": { diff --git a/modules/nf-core/tiddit/sv/main.nf b/modules/nf-core/tiddit/sv/main.nf index ff097942d..280242e71 100644 --- a/modules/nf-core/tiddit/sv/main.nf +++ b/modules/nf-core/tiddit/sv/main.nf @@ -23,7 +23,7 @@ process TIDDIT_SV { script: def args = task.ext.args ?: '' prefix = task.ext.prefix ?: "${meta.id}" - def bwa_command = bwa_index ? "[[ -d ${bwa_index} ]] && for i in ${bwa_index}/*; do [[ -f ${fasta} && ! \"\$i\" =~ .*\"${fasta}.\".* ]] && ln -s \$i ${fasta}.\${i##*.} || ln -s \$i .; done" : "" + def bwa_command = bwa_index ? "[[ -d ${bwa_index} ]] && for i in ${bwa_index}/*; do [[ -f ${fasta} && ! \"\$i\" =~ .*\"${fasta}.\".* ]] && ln -s \$i ${fasta}.\${i##*.} || ln -sf \$i .; done" : "" """ $bwa_command diff --git a/nextflow.config b/nextflow.config index 13a0d776e..154da8c19 100644 --- a/nextflow.config +++ b/nextflow.config @@ -87,6 +87,14 @@ params { svdb_query_dbs = null target_bed = null variant_catalog = null + filter_expansionhunter_htt = false + filter_sv_to_manta = false + skip_vep_sv = false + sv_freq_filter_expression = null // bcftools -e expression for SV frequency filtering, e.g. "INFO/SWEFRQ >= 0.02 || INFO/gnomAD_AF >= 0.01" + sv_size_threshold = null // e.g. 1000000 — filter SVs larger than this (bp) + manta_call_regions = null // BED.gz file restricting Manta to specific regions; null = whole genome + ml_prob_threshold = null // ML_PROB filter threshold (e.g. 0.6); null = use pipeline default (0.95) + variant_consequences_snv = null variant_consequences_sv = null vep_filters = null diff --git a/nextflow_schema.json b/nextflow_schema.json index a5b135431..399d48c0e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -620,6 +620,36 @@ "fa_icon": "fas fa-forward", "description": "Disable specified tools.", "help_text": "Multiple tools can be specified, separated by commas.", + "pattern": "^((fastp|gens|germlinecnvcaller|peddy|smncopynumbercaller|vcf2cytosure|fastqc|ngsbits)?,?)*(?= 0.02'). Null disables the filter.", + "help_text": "Full bcftools exclude expression, supporting any INFO field. Variants matching the expression are removed. Example: 'INFO/SWEFRQ >= 0.02 || INFO/AF >= 0.02'." + }, + "sv_size_threshold": { + "type": "integer", + "fa_icon": "fas fa-filter", + "description": "Filter Manta SVs larger than this value in bp before merge (e.g. 1000000). Null disables the filter." "pattern": "^((fastp|gens|germlinecnvcaller|peddy|smncopynumbercaller|vcf2cytosure|fastqc|ngsbits|mitosalt)?,?)*(? [meta, vcf, []] }, + [], [], [] + ) + ch_vcf = BCFTOOLS_FILTER_SV.out.vcf + } - PICARD_SORTVCF.out.vcf - .map { meta, vcf -> return [meta,vcf,[]] } - .set { ch_sortvcf } + // Optionally skip VEP annotation (set params.skip_vep_sv = true to enable). + // SVDB_QUERY still runs; only PICARD_SORTVCF, BCFTOOLS_VIEW and ENSEMBLVEP_SV are skipped. + if (!params.skip_vep_sv) { + PICARD_SORTVCF(ch_vcf, ch_genome_fasta, ch_genome_dictionary) - BCFTOOLS_VIEW(ch_sortvcf, [], [], []) - .vcf - .map { meta, vcf -> return [meta, vcf, []]} - .set { ch_vep_in } + PICARD_SORTVCF.out.vcf + .map { meta, vcf -> return [meta,vcf,[]] } + .set { ch_sortvcf } - ENSEMBLVEP_SV( - ch_vep_in, - val_genome, - "homo_sapiens", - val_vep_cache_version, - ch_vep_cache, - ch_genome_fasta, - ch_vep_extra_files - ) + BCFTOOLS_VIEW(ch_sortvcf, [], [], []) + .vcf + .map { meta, vcf -> return [meta, vcf, []]} + .set { ch_vep_in } - ch_publish = ENSEMBLVEP_SV.out.vcf - .mix(ENSEMBLVEP_SV.out.tbi) - .mix(ENSEMBLVEP_SV.out.report.map{ meta, _process, _vep, html -> return [meta, html] }) - .map { meta, value -> ['annotate_sv/', [meta, value]] } + ENSEMBLVEP_SV( + ch_vep_in, + val_genome, + "homo_sapiens", + val_vep_cache_version, + ch_vep_cache, + ch_genome_fasta, + ch_vep_extra_files + ) + ch_vcf_ann = ENSEMBLVEP_SV.out.vcf + ch_tbi_ann = ENSEMBLVEP_SV.out.tbi + ch_publish = ENSEMBLVEP_SV.out.vcf + .mix(ENSEMBLVEP_SV.out.tbi) + .mix(ENSEMBLVEP_SV.out.report.map{ meta, _process, _vep, html -> return [meta, html] }) + .map { meta, value -> ['annotate_sv/', [meta, value]] } + } else { + ch_vcf_ann = ch_vcf + ch_tbi_ann = Channel.empty() + ch_publish = ch_vcf + .map { meta, vcf -> ['annotate_sv/', [meta, vcf]] } + } emit: - publish = ch_publish // channel: [ val(destination), val(value) ] - tbi = ENSEMBLVEP_SV.out.tbi // channel: [ val(meta), path(tbi) ] - vcf_ann = ENSEMBLVEP_SV.out.vcf // channel: [ val(meta), path(vcf) ] + publish = ch_publish // channel: [ val(destination), val(value) ] + tbi = ch_tbi_ann // channel: [ val(meta), path(tbi) ] + vcf_ann = ch_vcf_ann // channel: [ val(meta), path(vcf) ] } diff --git a/subworkflows/local/call_repeat_expansions/main.nf b/subworkflows/local/call_repeat_expansions/main.nf index a3bac56e0..156a7c96a 100644 --- a/subworkflows/local/call_repeat_expansions/main.nf +++ b/subworkflows/local/call_repeat_expansions/main.nf @@ -4,6 +4,8 @@ include { BCFTOOLS_NORM as SPLIT_MULTIALLELICS_EXP } from '../../../modules/nf-core/bcftools/norm/main' include { BCFTOOLS_REHEADER as BCFTOOLS_REHEADER_EXP } from '../../../modules/nf-core/bcftools/reheader/main' +include { BCFTOOLS_VIEW as BCFTOOLS_FILTER_HTT } from '../../../modules/nf-core/bcftools/view/main' +include { BCFTOOLS_VIEW as BCFTOOLS_DECOMPRESS_MERGE } from '../../../modules/nf-core/bcftools/view/main' include { EXPANSIONHUNTER } from '../../../modules/nf-core/expansionhunter/main' include { PICARD_RENAMESAMPLEINVCF as RENAMESAMPLE_EXP } from '../../../modules/nf-core/picard/renamesampleinvcf/main' include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' @@ -44,8 +46,21 @@ workflow CALL_REPEAT_EXPANSIONS { ch_genome_fasta ) + // Optionally filter out HTT records (set params.filter_expansionhunter_htt = true to enable) + if (params.filter_expansionhunter_htt) { + BCFTOOLS_FILTER_HTT ( + SPLIT_MULTIALLELICS_EXP.out.vcf.map { meta, vcf -> [ meta, vcf, [] ] }, + [], + [], + [] + ) + ch_exp_vcfs_pre_merge = BCFTOOLS_FILTER_HTT.out.vcf + } else { + ch_exp_vcfs_pre_merge = SPLIT_MULTIALLELICS_EXP.out.vcf + } + // Merge indiviual repeat expansions - SPLIT_MULTIALLELICS_EXP.out.vcf + ch_exp_vcfs_pre_merge .collect{_meta, vcf -> vcf} .toList() .collect() @@ -57,9 +72,14 @@ workflow CALL_REPEAT_EXPANSIONS { SVDB_MERGE_REPEATS ( ch_svdb_merge_input, [], true ) + BCFTOOLS_DECOMPRESS_MERGE( + SVDB_MERGE_REPEATS.out.vcf.map { meta, vcf -> [meta, vcf, []] }, + [], [], [] + ) + ch_publish = SAMTOOLS_SORT.out.bam .mix(SAMTOOLS_SORT.out.bai) - .mix(BCFTOOLS_REHEADER_EXP.out.vcf) + .mix(BCFTOOLS_DECOMPRESS_MERGE.out.vcf) .map { meta, value -> ['repeat_expansions/', [meta, value]] } emit: diff --git a/subworkflows/local/call_structural_variants/main.nf b/subworkflows/local/call_structural_variants/main.nf index b5407b11e..f58d0da6c 100644 --- a/subworkflows/local/call_structural_variants/main.nf +++ b/subworkflows/local/call_structural_variants/main.nf @@ -71,10 +71,11 @@ workflow CALL_STRUCTURAL_VARIANTS { .collect{ _meta, vcf -> vcf } .set { ch_tiddit_vcf } - CALL_SV_CNVNATOR (ch_genome_bam_bai, ch_genome_fasta, ch_case_info) - .vcf + CALL_SV_CNVNATOR (ch_genome_bam_bai, ch_genome_fasta, ch_case_info) + .vcf .collect{ _meta, vcf -> vcf } - .set { ch_cnvnator_vcf } + .set { ch_cnvnator_vcf } + } if (!skip_germlinecnvcaller) { @@ -121,9 +122,14 @@ workflow CALL_STRUCTURAL_VARIANTS { // Merge - with consistent ordering using concat if (!val_analysis_type.equals("mito")) { + // When filter_sv_to_manta = true: exclude TIDDIT from merge and drop priority + // (clinic has only validated Manta; set params.filter_sv_to_manta = false to include TIDDIT) + ch_tiddit_vcf_for_merge = params.filter_sv_to_manta ? Channel.empty() : ch_tiddit_vcf + ch_priority_for_merge = params.filter_sv_to_manta ? [] : ch_svcaller_priority + // Concatenate in specific order: tiddit -> manta -> gcnvcaller -> cnvnator -> mitosalt // Empty channels won't contribute any items - ch_tiddit_vcf + ch_tiddit_vcf_for_merge .concat(ch_manta_vcf) .concat(ch_gcnvcaller_vcf) .concat(ch_cnvnator_vcf) @@ -134,7 +140,7 @@ workflow CALL_STRUCTURAL_VARIANTS { ch_case_info .combine(ch_vcf_paths) .set { ch_merge_vcfs_in } - SVDB_MERGE (ch_merge_vcfs_in, ch_svcaller_priority, true) + SVDB_MERGE (ch_merge_vcfs_in, ch_priority_for_merge, true) TABIX_TABIX (SVDB_MERGE.out.vcf) ch_merged_svs = SVDB_MERGE.out.vcf diff --git a/subworkflows/local/utils_nfcore_raredisease_pipeline/main.nf b/subworkflows/local/utils_nfcore_raredisease_pipeline/main.nf index ad87a4dc8..53883ce93 100644 --- a/subworkflows/local/utils_nfcore_raredisease_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_raredisease_pipeline/main.nf @@ -14,7 +14,6 @@ include { samplesheetToList } from 'plugin/nf-schema' include { paramsHelp } from 'plugin/nf-schema' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' @@ -202,9 +201,6 @@ workflow PIPELINE_COMPLETION { } completionSummary(monochrome_logs) - if (hook_url) { - imNotification(summary_params, hook_url) - } } workflow.onError { diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 2f30e9a46..afca54390 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -17,7 +17,7 @@ workflow UTILS_NFCORE_PIPELINE { checkProfileProvided(nextflow_cli_args) emit: - valid_config + valid_config = valid_config } /* @@ -353,67 +353,3 @@ def completionSummary(monochrome_logs=true) { log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } - -// -// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack -// -def imNotification(summary_params, hook_url) { - def summary = [:] - summary_params - .keySet() - .sort() - .each { group -> - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) { - misc_fields['repository'] = workflow.repository - } - if (workflow.commitId) { - misc_fields['commitid'] = workflow.commitId - } - if (workflow.revision) { - misc_fields['revision'] = workflow.revision - } - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = getWorkflowVersion() - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("${workflow.projectDir}/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection() - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")) - def postRC = post.getResponseCode() - if (!postRC.equals(200)) { - log.warn(post.getErrorStream().getText()) - } -} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test similarity index 100% rename from subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test rename to subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap similarity index 100% rename from subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap rename to subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf index e69b7aaec..1df8b76fb 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -38,7 +38,7 @@ workflow UTILS_NFSCHEMA_PLUGIN { } log.info paramsHelp( help_options, - params.help instanceof String && params.help != "true" ? params.help : "", + (params.help instanceof String && params.help != "true") ? params.help : "", ) exit 0 } @@ -71,4 +71,3 @@ workflow UTILS_NFSCHEMA_PLUGIN { emit: dummy_emit = true } - diff --git a/tests/default.nf.test b/tests/default.nf.test index 59e505297..fd07cd509 100644 --- a/tests/default.nf.test +++ b/tests/default.nf.test @@ -31,4 +31,100 @@ nextflow_pipeline { ) } } + + test("-profile test -- filter_expansionhunter_htt") { + + options "-stub" + + when { + params { + outdir = "$outputDir" + skip_tools = "germlinecnvcaller,gens" + filter_expansionhunter_htt = true + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("-profile test -- filter_sv_to_manta") { + + options "-stub" + + when { + params { + outdir = "$outputDir" + skip_tools = "germlinecnvcaller,gens" + filter_sv_to_manta = true + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("-profile test -- skip_vep_sv") { + + options "-stub" + + when { + params { + outdir = "$outputDir" + skip_tools = "germlinecnvcaller,gens" + skip_vep_sv = true + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("-profile test -- sv_size_threshold") { + + options "-stub" + + when { + params { + outdir = "$outputDir" + skip_tools = "germlinecnvcaller,gens" + sv_size_threshold = 1000000 + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("-profile test -- sv_freq_filter_expression") { + + options "-stub" + + when { + params { + outdir = "$outputDir" + skip_tools = "germlinecnvcaller,gens" + sv_freq_filter_expression = "INFO/SWEFRQ >= 0.02" + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + } diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 572d85273..2a49fc221 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -15,6 +15,9 @@ "BCFTOOLS_CONCAT_ME": { "bcftools": 1.22 }, + "BCFTOOLS_DECOMPRESS_MERGE": { + "bcftools": 1.22 + }, "BCFTOOLS_MERGE_MT": { "bcftools": 1.22 }, @@ -671,8 +674,6 @@ "call_snv/mitochondria/justhusky_mitochondria.vcf.gz", "call_snv/mitochondria/justhusky_mitochondria.vcf.gz.tbi", "call_sv", - "call_sv/earlycasualcaiman.saltshaker.png", - "call_sv/earlycasualcaiman.saltshaker_classify.html", "call_sv/earlycasualcaiman_mitochondria_deletions.txt", "call_sv/hugelymodelbat.saltshaker.png", "call_sv/hugelymodelbat.saltshaker_classify.html", @@ -893,15 +894,13 @@ "repeat_expansions", "repeat_expansions/earlycasualcaiman_exphunter_sorted.bam", "repeat_expansions/earlycasualcaiman_exphunter_sorted.bam.bai", - "repeat_expansions/earlycasualcaiman_repeat_expansion.vcf", "repeat_expansions/hugelymodelbat_exphunter_sorted.bam", "repeat_expansions/hugelymodelbat_exphunter_sorted.bam.bai", - "repeat_expansions/hugelymodelbat_repeat_expansion.vcf", + "repeat_expansions/justhusky_repeat_expansion.vcf", "repeat_expansions/justhusky_repeat_expansion_stranger.vcf.gz", "repeat_expansions/justhusky_repeat_expansion_stranger.vcf.gz.tbi", "repeat_expansions/slowlycivilbuck_exphunter_sorted.bam", "repeat_expansions/slowlycivilbuck_exphunter_sorted.bam.bai", - "repeat_expansions/slowlycivilbuck_repeat_expansion.vcf", "smncopynumbercaller", "smncopynumbercaller/out", "smncopynumbercaller/out/justhusky_smncopynumbercaller.json", diff --git a/workflows/raredisease.nf b/workflows/raredisease.nf index b3b1c1470..0e6c0a6f1 100644 --- a/workflows/raredisease.nf +++ b/workflows/raredisease.nf @@ -699,30 +699,32 @@ workflow RAREDISEASE { ch_ann_csq_sv_in = ch_clinical_sv_vcf.mix(ch_clin_research_sv_vcf.research) - ANN_CSQ_PLI_SV ( - ch_variant_consequences_sv, - ch_ann_csq_sv_in, - false, - '' - ) + if (!params.skip_vep_sv) { + ANN_CSQ_PLI_SV ( + ch_variant_consequences_sv, + ch_ann_csq_sv_in, + false, + '' + ) - ANN_CSQ_PLI_SV.out.vcf_ann - .filter { meta, _vcf -> - if (meta.probands.size()==0) { - log.warn("Skipping SV ranking since no affected samples are detected in the case") + ANN_CSQ_PLI_SV.out.vcf_ann + .filter { meta, _vcf -> + if (meta.probands.size()==0) { + log.warn("Skipping SV ranking since no affected samples are detected in the case") + } + meta.probands.size()>0 } - meta.probands.size()>0 - } - .set {ch_ranksnv_sv_in} + .set {ch_ranksnv_sv_in} - RANK_VARIANTS_SV ( - ch_pedfile, - ch_reduced_penetrance, - ch_score_config_sv, - ch_ranksnv_sv_in, - true - ) - ch_rank_sv_publish = RANK_VARIANTS_SV.out.publish + RANK_VARIANTS_SV ( + ch_pedfile, + ch_reduced_penetrance, + ch_score_config_sv, + ch_ranksnv_sv_in, + true + ) + ch_rank_sv_publish = RANK_VARIANTS_SV.out.publish + } } } /* @@ -847,7 +849,7 @@ workflow RAREDISEASE { Generate CGH files from sequencing data ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if (!skip_vcf2cytosure && val_analysis_type.equals("wgs") && !skip_sv_calling && !skip_sv_annotation) { + if (!skip_vcf2cytosure && val_analysis_type.equals("wgs") && !skip_sv_calling && !skip_sv_annotation && !params.skip_vep_sv) { GENERATE_CYTOSURE_FILES ( ch_mapped.genome_marked_bam_bai, ch_vcf2cytosure_blacklist,