From 9ed780d8aae98228cd6f954161eddc7f1628c288 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 09:36:20 +0200 Subject: [PATCH 01/52] Added new liftover option --- scout/commands/export/variant.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index ef5882feeb..6724ac8eb6 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -2,13 +2,13 @@ import json as json_lib import logging import os -from typing import Tuple +from typing import Optional, Tuple import click from flask.cli import with_appcontext from xlsxwriter import Workbook -from scout.constants import CALLERS, DATE_DAY_FORMATTER +from scout.constants import BUILDS, CALLERS, DATE_DAY_FORMATTER from scout.constants.managed_variant import MANAGED_CATEGORIES from scout.constants.variants_export import VCF_HEADER, VERIFIED_VARIANTS_HEADER from scout.export.variant import ( @@ -124,8 +124,13 @@ def verified(collaborator, test, outpath=None): @collaborator_option @build_option @json_option +@click.option( + "--liftover", + type=click.Choice(BUILDS), + help="Perform liftover on coordinates and export as managed variants infile.", +) @with_appcontext -def managed(collaborator: str, category: Tuple[str], build: str, json: bool): +def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover: Optional[str]): """Export managed variants for a collaborator in VCF or JSON format""" LOG.info("Running scout export managed variants") adapter = store @@ -134,6 +139,9 @@ def managed(collaborator: str, category: Tuple[str], build: str, json: bool): adapter=adapter, institute=collaborator, build=build, category=list(category) ) + if liftover: + + if json: click.echo(json_lib.dumps([var for var in variants], default=bson_handler)) return From fc630bfa25c4ce5593721e31ca5680fca69de68a Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 10:18:25 +0200 Subject: [PATCH 02/52] Updated changelog --- CHANGELOG.md | 1 + scout/commands/export/variant.py | 54 +++++++++++++++++++++++--------- scout/export/variant.py | 1 - 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c28b27ee0..2bbf496293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Parse and display Paraphrase output from Nallo Paraphase conversion (#6178) - Command line option to export causative variants by category and/or genome build (#6202) - An additional button on causatives and verified pages to download SNVs and SVs as input for the managed variants list (#6205) +- A `--liftover-from` option to the `export managed` command line, so managed variants can be exported and imported again in another genome build () ### Changed - Genome build is now shown on variant verification "Sanger" emails (#6194) - Refactor, speedup (dry-run only) and add a progress bar to `scout delete variants` cmd (#6094) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 6724ac8eb6..2d9c46cb8b 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -9,7 +9,7 @@ from xlsxwriter import Workbook from scout.constants import BUILDS, CALLERS, DATE_DAY_FORMATTER -from scout.constants.managed_variant import MANAGED_CATEGORIES +from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER from scout.constants.variants_export import VCF_HEADER, VERIFIED_VARIANTS_HEADER from scout.export.variant import ( export_causative_variants, @@ -17,6 +17,7 @@ export_verified_variants, ) from scout.server.extensions import store +from scout.utils.ensembl_rest_clients import EnsemblRestApiClient from scout.utils.vcf import validate_vcf_line from .export_handler import bson_handler @@ -125,12 +126,12 @@ def verified(collaborator, test, outpath=None): @build_option @json_option @click.option( - "--liftover", + "--liftover-from", type=click.Choice(BUILDS), help="Perform liftover on coordinates and export as managed variants infile.", ) @with_appcontext -def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover: Optional[str]): +def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str]): """Export managed variants for a collaborator in VCF or JSON format""" LOG.info("Running scout export managed variants") adapter = store @@ -139,25 +140,48 @@ def managed(collaborator: str, category: Tuple[str], build: str, json: bool, lif adapter=adapter, institute=collaborator, build=build, category=list(category) ) - if liftover: - - if json: click.echo(json_lib.dumps([var for var in variants], default=bson_handler)) return - vcf_header = VCF_HEADER - vcf_header.insert(2, "##fileDate={}".format(datetime.datetime.now())) + elif liftover_from: + valid_lines = [MANAGED_VARIANTS_INFILE_HEADER] + ensembl_client = EnsemblRestApiClient() + for variant_obj in variants: + if variant_obj["category"] not in ["snv", "cancer_snv"]: + continue + liftover_result = ensembl_client.liftover(build=liftover_from, chrom=variant_obj["chromosome"], start=variant_obj["position"], end=variant_obj.get("end","")) + if not liftover_result: + continue + chrom = liftover_result[0]["mapped"]["seq_region_name"] + pos = liftover_result[0]["mapped"]["start"] + end = liftover_result[0]["mapped"]["end"] + ref = variant_obj.get("reference","") + alt = variant_obj.get("alternative","") + category = variant_obj.get("category", "snv") + sub_category = variant_obj.get("sub_category", "snv") + build = "38" if liftover_from == "37" else "37" + description = variant_obj.get("description") + institutes = ",".join(variant_obj.get("institute")) + + valid_lines.append( + f"{chrom};{pos};{end};{ref};{alt};" + f"{category};{sub_category};{build};{description};;{institutes}" + ) + + else: + vcf_header = VCF_HEADER + vcf_header.insert(2, "##fileDate={}".format(datetime.datetime.now())) - valid_lines = [] + valid_lines = [] - for variant_obj in variants: - variant_string = get_vcf_entry(variant_obj) - if variant_string: - valid_lines.append(variant_string) + for variant_obj in variants: + variant_string = get_vcf_entry(variant_obj) + if variant_string: + valid_lines.append(variant_string) - for line in vcf_header: - click.echo(line) + for line in vcf_header: + click.echo(line) for valid_line in valid_lines: click.echo(valid_line) diff --git a/scout/export/variant.py b/scout/export/variant.py index 34843ee94d..54a59020ce 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -62,7 +62,6 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) - def export_managed_variants( adapter: MongoAdapter, institute: str = None, From efe4c420edf6c5d93837b8b1744aecd5d69bcbbb Mon Sep 17 00:00:00 2001 From: Lint Action Date: Wed, 22 Apr 2026 08:18:54 +0000 Subject: [PATCH 03/52] Fix code style issues with Black --- scout/commands/export/variant.py | 15 +++++++++++---- scout/export/variant.py | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 2d9c46cb8b..9d6b3aa193 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -131,7 +131,9 @@ def verified(collaborator, test, outpath=None): help="Perform liftover on coordinates and export as managed variants infile.", ) @with_appcontext -def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str]): +def managed( + collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str] +): """Export managed variants for a collaborator in VCF or JSON format""" LOG.info("Running scout export managed variants") adapter = store @@ -150,14 +152,19 @@ def managed(collaborator: str, category: Tuple[str], build: str, json: bool, lif for variant_obj in variants: if variant_obj["category"] not in ["snv", "cancer_snv"]: continue - liftover_result = ensembl_client.liftover(build=liftover_from, chrom=variant_obj["chromosome"], start=variant_obj["position"], end=variant_obj.get("end","")) + liftover_result = ensembl_client.liftover( + build=liftover_from, + chrom=variant_obj["chromosome"], + start=variant_obj["position"], + end=variant_obj.get("end", ""), + ) if not liftover_result: continue chrom = liftover_result[0]["mapped"]["seq_region_name"] pos = liftover_result[0]["mapped"]["start"] end = liftover_result[0]["mapped"]["end"] - ref = variant_obj.get("reference","") - alt = variant_obj.get("alternative","") + ref = variant_obj.get("reference", "") + alt = variant_obj.get("alternative", "") category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") build = "38" if liftover_from == "37" else "37" diff --git a/scout/export/variant.py b/scout/export/variant.py index 54a59020ce..34843ee94d 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -62,6 +62,7 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) + def export_managed_variants( adapter: MongoAdapter, institute: str = None, From ad668734042300b85154ac4416f60c2bc219472d Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 10:39:45 +0200 Subject: [PATCH 04/52] Reduce complexity --- scout/commands/export/variant.py | 36 +++----------------------------- scout/export/variant.py | 32 +++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 9d6b3aa193..d60174a4bb 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -15,9 +15,9 @@ export_causative_variants, export_managed_variants, export_verified_variants, + liftover_managed_variants, ) from scout.server.extensions import store -from scout.utils.ensembl_rest_clients import EnsemblRestApiClient from scout.utils.vcf import validate_vcf_line from .export_handler import bson_handler @@ -131,9 +131,7 @@ def verified(collaborator, test, outpath=None): help="Perform liftover on coordinates and export as managed variants infile.", ) @with_appcontext -def managed( - collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str] -): +def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str]): """Export managed variants for a collaborator in VCF or JSON format""" LOG.info("Running scout export managed variants") adapter = store @@ -147,35 +145,7 @@ def managed( return elif liftover_from: - valid_lines = [MANAGED_VARIANTS_INFILE_HEADER] - ensembl_client = EnsemblRestApiClient() - for variant_obj in variants: - if variant_obj["category"] not in ["snv", "cancer_snv"]: - continue - liftover_result = ensembl_client.liftover( - build=liftover_from, - chrom=variant_obj["chromosome"], - start=variant_obj["position"], - end=variant_obj.get("end", ""), - ) - if not liftover_result: - continue - chrom = liftover_result[0]["mapped"]["seq_region_name"] - pos = liftover_result[0]["mapped"]["start"] - end = liftover_result[0]["mapped"]["end"] - ref = variant_obj.get("reference", "") - alt = variant_obj.get("alternative", "") - category = variant_obj.get("category", "snv") - sub_category = variant_obj.get("sub_category", "snv") - build = "38" if liftover_from == "37" else "37" - description = variant_obj.get("description") - institutes = ",".join(variant_obj.get("institute")) - - valid_lines.append( - f"{chrom};{pos};{end};{ref};{alt};" - f"{category};{sub_category};{build};{description};;{institutes}" - ) - + valid_lines = liftover_managed_variants(managed_variants = variants, liftover_from=liftover_from) else: vcf_header = VCF_HEADER vcf_header.insert(2, "##fileDate={}".format(datetime.datetime.now())) diff --git a/scout/export/variant.py b/scout/export/variant.py index 34843ee94d..80e941282a 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -5,8 +5,9 @@ from scout.adapter.mongo.base import MongoAdapter from scout.constants import CHROMOSOME_INTEGERS -from scout.constants.managed_variant import MANAGED_CATEGORIES +from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER from scout.models.managed_variant import ManagedVariant +from scout.utils.ensembl_rest_clients import EnsemblRestApiClient LOG = logging.getLogger(__name__) @@ -62,6 +63,35 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) +def liftover_managed_variants(managed_variants, liftover_from) -> List[str]: + """Perform liftover over a list of managed variants and return a list of lines formatted as a managed variants upload infile.""" + + valid_lines = [MANAGED_VARIANTS_INFILE_HEADER] + ensembl_client = EnsemblRestApiClient() + for variant_obj in managed_variants: + if variant_obj["category"] not in ["snv", "cancer_snv"]: + continue + liftover_result = ensembl_client.liftover(build=liftover_from, chrom=variant_obj["chromosome"], + start=variant_obj["position"], end=variant_obj.get("end", "")) + if not liftover_result: + continue + chrom = liftover_result[0]["mapped"]["seq_region_name"] + pos = liftover_result[0]["mapped"]["start"] + end = liftover_result[0]["mapped"]["end"] + ref = variant_obj.get("reference", "") + alt = variant_obj.get("alternative", "") + category = variant_obj.get("category", "snv") + sub_category = variant_obj.get("sub_category", "snv") + build = "38" if liftover_from == "37" else "37" + description = variant_obj.get("description") + institutes = ",".join(variant_obj.get("institute")) + + valid_lines.append( + f"{chrom};{pos};{end};{ref};{alt};" + f"{category};{sub_category};{build};{description};;{institutes}" + ) + return valid_lines + def export_managed_variants( adapter: MongoAdapter, From 0fd7439d6430817383d23283817077ba5f200ea8 Mon Sep 17 00:00:00 2001 From: Lint Action Date: Wed, 22 Apr 2026 08:40:15 +0000 Subject: [PATCH 05/52] Fix code style issues with Black --- scout/commands/export/variant.py | 8 ++++++-- scout/export/variant.py | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index d60174a4bb..b2e5f0f12b 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -131,7 +131,9 @@ def verified(collaborator, test, outpath=None): help="Perform liftover on coordinates and export as managed variants infile.", ) @with_appcontext -def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str]): +def managed( + collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str] +): """Export managed variants for a collaborator in VCF or JSON format""" LOG.info("Running scout export managed variants") adapter = store @@ -145,7 +147,9 @@ def managed(collaborator: str, category: Tuple[str], build: str, json: bool, lif return elif liftover_from: - valid_lines = liftover_managed_variants(managed_variants = variants, liftover_from=liftover_from) + valid_lines = liftover_managed_variants( + managed_variants=variants, liftover_from=liftover_from + ) else: vcf_header = VCF_HEADER vcf_header.insert(2, "##fileDate={}".format(datetime.datetime.now())) diff --git a/scout/export/variant.py b/scout/export/variant.py index 80e941282a..5dae158258 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -63,6 +63,7 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) + def liftover_managed_variants(managed_variants, liftover_from) -> List[str]: """Perform liftover over a list of managed variants and return a list of lines formatted as a managed variants upload infile.""" @@ -71,8 +72,12 @@ def liftover_managed_variants(managed_variants, liftover_from) -> List[str]: for variant_obj in managed_variants: if variant_obj["category"] not in ["snv", "cancer_snv"]: continue - liftover_result = ensembl_client.liftover(build=liftover_from, chrom=variant_obj["chromosome"], - start=variant_obj["position"], end=variant_obj.get("end", "")) + liftover_result = ensembl_client.liftover( + build=liftover_from, + chrom=variant_obj["chromosome"], + start=variant_obj["position"], + end=variant_obj.get("end", ""), + ) if not liftover_result: continue chrom = liftover_result[0]["mapped"]["seq_region_name"] From 189646bbb70094598e26972341123b67739ed6c2 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 10:44:43 +0200 Subject: [PATCH 06/52] Removed unused import --- scout/commands/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index b2e5f0f12b..48604fadd7 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -9,7 +9,7 @@ from xlsxwriter import Workbook from scout.constants import BUILDS, CALLERS, DATE_DAY_FORMATTER -from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER +from scout.constants.managed_variant import MANAGED_CATEGORIES from scout.constants.variants_export import VCF_HEADER, VERIFIED_VARIANTS_HEADER from scout.export.variant import ( export_causative_variants, From d1d757da90476d3c64ac8d3d303ba9dad0c376e3 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 10:51:26 +0200 Subject: [PATCH 07/52] Renamed vars and removed unused import --- scout/export/variant.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 5dae158258..3e92e52741 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import logging import urllib.parse -from typing import List, Optional +from typing import Generator, List, Optional from scout.adapter.mongo.base import MongoAdapter from scout.constants import CHROMOSOME_INTEGERS @@ -64,10 +64,10 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) -def liftover_managed_variants(managed_variants, liftover_from) -> List[str]: +def liftover_managed_variants(managed_variants: Generator, liftover_from: str) -> List[str]: """Perform liftover over a list of managed variants and return a list of lines formatted as a managed variants upload infile.""" - valid_lines = [MANAGED_VARIANTS_INFILE_HEADER] + export_lines = [MANAGED_VARIANTS_INFILE_HEADER] ensembl_client = EnsemblRestApiClient() for variant_obj in managed_variants: if variant_obj["category"] not in ["snv", "cancer_snv"]: @@ -91,11 +91,11 @@ def liftover_managed_variants(managed_variants, liftover_from) -> List[str]: description = variant_obj.get("description") institutes = ",".join(variant_obj.get("institute")) - valid_lines.append( + export_lines.append( f"{chrom};{pos};{end};{ref};{alt};" f"{category};{sub_category};{build};{description};;{institutes}" ) - return valid_lines + return export_lines def export_managed_variants( From 12c88fd09ec6ce24ecaca2e1ea2be42ae90da851 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 10:52:56 +0200 Subject: [PATCH 08/52] PR ref in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 038d8f1490..c059b57409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Command line option to export causative variants by category and/or genome build (#6202) - An additional button on causatives and verified pages to download SNVs and SVs as input for the managed variants list (#6205) - Display total number of variants and number of variants returned by a filter on Managed Variants page (#6223) -- A `--liftover-from` option to the `export managed` command line, so managed variants can be exported and imported again in another genome build () +- A `--liftover-from` option to the `export managed` command line, so managed variants can be exported and imported again in another genome build (#6225) ### Changed - Genome build is now shown on variant verification "Sanger" emails (#6194) - Refactor, speedup (dry-run only) and add a progress bar to `scout delete variants` cmd (#6094) From 7453155202f98eafb68e356f10796a922b94e3be Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 11:39:37 +0200 Subject: [PATCH 09/52] Add a test --- scout/export/variant.py | 6 +-- tests/export/test_export_variants.py | 46 +++++++++++++++++-- .../test_managed_variants_views.py | 2 +- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 3e92e52741..f50cb04cfc 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -70,13 +70,13 @@ def liftover_managed_variants(managed_variants: Generator, liftover_from: str) - export_lines = [MANAGED_VARIANTS_INFILE_HEADER] ensembl_client = EnsemblRestApiClient() for variant_obj in managed_variants: - if variant_obj["category"] not in ["snv", "cancer_snv"]: + if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue liftover_result = ensembl_client.liftover( build=liftover_from, chrom=variant_obj["chromosome"], start=variant_obj["position"], - end=variant_obj.get("end", ""), + end=variant_obj.get("end", variant_obj["position"]), ) if not liftover_result: continue @@ -89,7 +89,7 @@ def liftover_managed_variants(managed_variants: Generator, liftover_from: str) - sub_category = variant_obj.get("sub_category", "snv") build = "38" if liftover_from == "37" else "37" description = variant_obj.get("description") - institutes = ",".join(variant_obj.get("institute")) + institutes = ",".join(variant_obj.get("institute", [])) export_lines.append( f"{chrom};{pos};{end};{ref};{alt};" diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index 493f86faa0..88872a174d 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- +import responses + +from scout.constants.managed_variant import MANAGED_VARIANTS_INFILE_HEADER from scout.constants.variants_export import MT_EXPORT_HEADER -from scout.export.variant import export_mt_variants +from scout.export.variant import export_mt_variants, liftover_managed_variants +from scout.utils.ensembl_rest_clients import RESTAPI_URL def test_export_mt_variants(case_obj, real_populated_database): @@ -26,7 +30,43 @@ def test_export_mt_variants(case_obj, real_populated_database): for sample in samples: sample_lines = export_mt_variants(variants=mt_variants, sample_id=sample["individual_id"]) - # check that rows to write to excel corespond to number of variants + # check that rows to write to excel correspond to number of variants assert len(sample_lines) == len(mt_variants) - # check that cols to write to excel corespond to fields of excel header + # check that cols to write to excel correspond to fields of Excel header assert len(sample_lines[0]) == len(MT_EXPORT_HEADER) + + +@responses.activate +def test_liftover_managed_variants(ensembl_liftover_response): + """Test the function that performs liftover over a list of managed variants and formats them into a list of strings.""" + + # GIVEN a patched response from Ensembl + url = f"{RESTAPI_URL}/map/human/GRCh37/X:1000000..1000000/GRCh38?content-type=application/json" + responses.add( + responses.GET, + url, + json=ensembl_liftover_response, + status=200, + ) + managed_variant_info = { + "chromosome": "X", + "position": "1000000", + "reference": "C", + "alternative": "T", + "build": "37", + } + + # GIVEN a list of managed variants + managed_variants = [managed_variant_info] + + # THEN the liftover function should export them correctly: + export_lines = liftover_managed_variants(managed_variants=managed_variants, liftover_from="37") + + # WITH the first line being the header + assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER + + # AND second line being the lifted-over variant + lifted_chrom = ensembl_liftover_response["mappings"][0]["mapped"]["seq_region_name"] + lifted_position = ensembl_liftover_response["mappings"][0]["mapped"]["start"] + lifted_end = ensembl_liftover_response["mappings"][0]["mapped"]["end"] + assert f"{lifted_chrom};{lifted_position};{lifted_end}" in export_lines[1] diff --git a/tests/server/blueprints/managed_variants/test_managed_variants_views.py b/tests/server/blueprints/managed_variants/test_managed_variants_views.py index 98b1c7963a..d1d132d46e 100644 --- a/tests/server/blueprints/managed_variants/test_managed_variants_views.py +++ b/tests/server/blueprints/managed_variants/test_managed_variants_views.py @@ -66,7 +66,7 @@ def test_add_and_remove_managed_variants(app, mocker, mock_redirect): resp = client.post(url_for("managed_variants.add_managed_variant"), data=add_form_data) # THEN the status code should still be redirect assert resp.status_code == 302 - # THEN the database should still contatain only one variant + # THEN the database should still contain only one variant assert sum(1 for i in store.managed_variant_collection.find()) == 1 # WHEN requesting to remove the selected variant From fa713ff9a3ddd5a1624ea613add32c9de0d9752f Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 11:46:28 +0200 Subject: [PATCH 10/52] Thanks SonarQube --- tests/export/test_export_variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index 88872a174d..f9775c2151 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -60,7 +60,7 @@ def test_liftover_managed_variants(ensembl_liftover_response): managed_variants = [managed_variant_info] # THEN the liftover function should export them correctly: - export_lines = liftover_managed_variants(managed_variants=managed_variants, liftover_from="37") + export_lines = liftover_managed_variants([var for var in managed_variants], liftover_from="37") # WITH the first line being the header assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER From 633a8c2f5b6464b7e1214e5b4cf557c229a4ccd8 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 11:50:37 +0200 Subject: [PATCH 11/52] How about this --- tests/export/test_export_variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index f9775c2151..43a5e8907e 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -60,7 +60,7 @@ def test_liftover_managed_variants(ensembl_liftover_response): managed_variants = [managed_variant_info] # THEN the liftover function should export them correctly: - export_lines = liftover_managed_variants([var for var in managed_variants], liftover_from="37") + export_lines = liftover_managed_variants((var for var in managed_variants), liftover_from="37") # WITH the first line being the header assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER From 0f35e53d669a404d17f798c562048a8f02c147e8 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 11:55:14 +0200 Subject: [PATCH 12/52] Like this instead? --- tests/export/test_export_variants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index 43a5e8907e..d7e09206f9 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -60,7 +60,8 @@ def test_liftover_managed_variants(ensembl_liftover_response): managed_variants = [managed_variant_info] # THEN the liftover function should export them correctly: - export_lines = liftover_managed_variants((var for var in managed_variants), liftover_from="37") + managed_variants_generator = (var for var in managed_variants) + export_lines = liftover_managed_variants(managed_variants = managed_variants_generator, liftover_from="37") # WITH the first line being the header assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER From aeb8c53ec04d574f5c9735e567a7197b3eb4e8ee Mon Sep 17 00:00:00 2001 From: Lint Action Date: Wed, 22 Apr 2026 09:55:44 +0000 Subject: [PATCH 13/52] Fix code style issues with Black --- tests/export/test_export_variants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index d7e09206f9..0602694ace 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -61,7 +61,9 @@ def test_liftover_managed_variants(ensembl_liftover_response): # THEN the liftover function should export them correctly: managed_variants_generator = (var for var in managed_variants) - export_lines = liftover_managed_variants(managed_variants = managed_variants_generator, liftover_from="37") + export_lines = liftover_managed_variants( + managed_variants=managed_variants_generator, liftover_from="37" + ) # WITH the first line being the header assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER From 459c044241fccb39f4583f16a9942896f4f99c81 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 11:56:26 +0200 Subject: [PATCH 14/52] trigger tests From 6c13c83995c87252941dc94f13ee4e44d457ae98 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 12:02:04 +0200 Subject: [PATCH 15/52] Hate that linter --- tests/export/test_export_variants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index 0602694ace..98b28ea215 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -60,9 +60,8 @@ def test_liftover_managed_variants(ensembl_liftover_response): managed_variants = [managed_variant_info] # THEN the liftover function should export them correctly: - managed_variants_generator = (var for var in managed_variants) export_lines = liftover_managed_variants( - managed_variants=managed_variants_generator, liftover_from="37" + managed_variants=iter(managed_variants), liftover_from="37" ) # WITH the first line being the header From 686bcaca8efd10c27342ca1eb72a69711390c1b3 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 12:05:59 +0200 Subject: [PATCH 16/52] How about an iterable --- scout/export/variant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index f50cb04cfc..ac3adcd01d 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import logging import urllib.parse -from typing import Generator, List, Optional +from typing import Iterable, List, Optional from scout.adapter.mongo.base import MongoAdapter from scout.constants import CHROMOSOME_INTEGERS @@ -64,7 +64,7 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) -def liftover_managed_variants(managed_variants: Generator, liftover_from: str) -> List[str]: +def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> List[str]: """Perform liftover over a list of managed variants and return a list of lines formatted as a managed variants upload infile.""" export_lines = [MANAGED_VARIANTS_INFILE_HEADER] From dc51478e34af4ecba0d681f9a67593a2cbe795e8 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 13:54:05 +0200 Subject: [PATCH 17/52] Fix none institutes --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index ac3adcd01d..64b1a52ccb 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -89,7 +89,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> sub_category = variant_obj.get("sub_category", "snv") build = "38" if liftover_from == "37" else "37" description = variant_obj.get("description") - institutes = ",".join(variant_obj.get("institute", [])) + institutes = ",".join(variant_obj.get("institute", "")) export_lines.append( f"{chrom};{pos};{end};{ref};{alt};" From 5b960cb7a1903aadb087f1f109f6238f3a318c04 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 13:57:24 +0200 Subject: [PATCH 18/52] Do not lift if already in the right build --- scout/export/variant.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 64b1a52ccb..32fbbf0d82 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -72,6 +72,10 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> for variant_obj in managed_variants: if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue + build = "38" if liftover_from == "37" else "37" + if variant_obj.get("build", "37") == build: + continue + liftover_result = ensembl_client.liftover( build=liftover_from, chrom=variant_obj["chromosome"], @@ -87,7 +91,6 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> alt = variant_obj.get("alternative", "") category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") - build = "38" if liftover_from == "37" else "37" description = variant_obj.get("description") institutes = ",".join(variant_obj.get("institute", "")) From ef954581e320560351b677abf688318cdaecc95b Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 14:03:51 +0200 Subject: [PATCH 19/52] Do not skip those in the right build --- scout/export/variant.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 32fbbf0d82..150279534e 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -73,20 +73,25 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue build = "38" if liftover_from == "37" else "37" - if variant_obj.get("build", "37") == build: - continue + if ( + variant_obj.get("build", "37") == build + ): # Use coordinates from variant instead of liftover + chrom = variant_obj["chromosome"] + pos = variant_obj["position"] + end = variant_obj.get("end", variant_obj["position"]) + else: + liftover_result = ensembl_client.liftover( + build=liftover_from, + chrom=variant_obj["chromosome"], + start=variant_obj["position"], + end=variant_obj.get("end", variant_obj["position"]), + ) + if not liftover_result: + continue + chrom = liftover_result[0]["mapped"]["seq_region_name"] + pos = liftover_result[0]["mapped"]["start"] + end = liftover_result[0]["mapped"]["end"] - liftover_result = ensembl_client.liftover( - build=liftover_from, - chrom=variant_obj["chromosome"], - start=variant_obj["position"], - end=variant_obj.get("end", variant_obj["position"]), - ) - if not liftover_result: - continue - chrom = liftover_result[0]["mapped"]["seq_region_name"] - pos = liftover_result[0]["mapped"]["start"] - end = liftover_result[0]["mapped"]["end"] ref = variant_obj.get("reference", "") alt = variant_obj.get("alternative", "") category = variant_obj.get("category", "snv") From b065ec1e1087ef551e98f26cd877ac4fa08debf3 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 14:11:29 +0200 Subject: [PATCH 20/52] Fix it for reals --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 150279534e..28a2287283 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -97,7 +97,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") description = variant_obj.get("description") - institutes = ",".join(variant_obj.get("institute", "")) + institutes = ",".join(variant_obj.get("institute") or []) export_lines.append( f"{chrom};{pos};{end};{ref};{alt};" From bbad8630d71c7badc780f6accff641eca5306a63 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 14:30:51 +0200 Subject: [PATCH 21/52] Im an idiot --- scout/export/variant.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 28a2287283..e0bb275f35 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -70,6 +70,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> export_lines = [MANAGED_VARIANTS_INFILE_HEADER] ensembl_client = EnsemblRestApiClient() for variant_obj in managed_variants: + LOG.warning(variant_obj) if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue build = "38" if liftover_from == "37" else "37" @@ -103,7 +104,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> f"{chrom};{pos};{end};{ref};{alt};" f"{category};{sub_category};{build};{description};;{institutes}" ) - return export_lines + return export_lines def export_managed_variants( From 7944cc27698b3022162de45d235196fbccce03ab Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 14:31:45 +0200 Subject: [PATCH 22/52] Final touch --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index e0bb275f35..55ab387e89 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -75,7 +75,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> continue build = "38" if liftover_from == "37" else "37" if ( - variant_obj.get("build", "37") == build + variant_obj.get("build") == build ): # Use coordinates from variant instead of liftover chrom = variant_obj["chromosome"] pos = variant_obj["position"] From 9d8a103a8cdff91c73aa5923b2626c24f4607dd4 Mon Sep 17 00:00:00 2001 From: Lint Action Date: Wed, 22 Apr 2026 12:32:11 +0000 Subject: [PATCH 23/52] Fix code style issues with Black --- scout/export/variant.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 55ab387e89..af50483111 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -74,9 +74,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue build = "38" if liftover_from == "37" else "37" - if ( - variant_obj.get("build") == build - ): # Use coordinates from variant instead of liftover + if variant_obj.get("build") == build: # Use coordinates from variant instead of liftover chrom = variant_obj["chromosome"] pos = variant_obj["position"] end = variant_obj.get("end", variant_obj["position"]) From 35c4fc5007d064f0a18f68f35451bdf004e2a3d9 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 22 Apr 2026 14:41:02 +0200 Subject: [PATCH 24/52] trigger tests From a43ff5cce6cb501d4777ce2e29e5f3431ccee84e Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Thu, 30 Apr 2026 07:38:14 +0200 Subject: [PATCH 25/52] Add some progress logs and retries --- scout/export/variant.py | 42 ++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index af50483111..a480bfad7f 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- import logging +import time import urllib.parse from typing import Iterable, List, Optional +import requests + from scout.adapter.mongo.base import MongoAdapter from scout.constants import CHROMOSOME_INTEGERS from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER @@ -69,24 +72,41 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> export_lines = [MANAGED_VARIANTS_INFILE_HEADER] ensembl_client = EnsemblRestApiClient() - for variant_obj in managed_variants: - LOG.warning(variant_obj) + + def liftover_with_retry(variant_obj, retries=3): + for attempt in range(retries): + try: + return ensembl_client.liftover( + build=liftover_from, + chrom=variant_obj["chromosome"], + start=variant_obj["position"], + end=variant_obj.get("end", variant_obj["position"]), + ) + except requests.exceptions.RequestException: + wait = 1 * (attempt + 1) + LOG.warning(f"Retry {attempt + 1}/{retries} after error. Waiting {wait}s") + time.sleep(wait) + return None + + for i, variant_obj in enumerate(managed_variants, 1): + if i % 50 == 0: + LOG.info(f"Processed {i} variants") + if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue + build = "38" if liftover_from == "37" else "37" - if variant_obj.get("build") == build: # Use coordinates from variant instead of liftover + + if variant_obj.get("build") == build: chrom = variant_obj["chromosome"] pos = variant_obj["position"] end = variant_obj.get("end", variant_obj["position"]) else: - liftover_result = ensembl_client.liftover( - build=liftover_from, - chrom=variant_obj["chromosome"], - start=variant_obj["position"], - end=variant_obj.get("end", variant_obj["position"]), - ) + liftover_result = liftover_with_retry(variant_obj) + if not liftover_result: continue + chrom = liftover_result[0]["mapped"]["seq_region_name"] pos = liftover_result[0]["mapped"]["start"] end = liftover_result[0]["mapped"]["end"] @@ -102,6 +122,10 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> f"{chrom};{pos};{end};{ref};{alt};" f"{category};{sub_category};{build};{description};;{institutes}" ) + + time.sleep(0.2) # gentle rate limiting + + LOG.info(f"Done. Total processed: {i}") return export_lines From 8a381b9d8ffa7b6c64b0c0cec60fdb1d08886e0a Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 6 May 2026 08:54:00 +0200 Subject: [PATCH 26/52] Liftover using bcftools via broad --- scout/export/variant.py | 73 ++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index a480bfad7f..a820d08122 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import logging -import time import urllib.parse from typing import Iterable, List, Optional @@ -10,7 +9,6 @@ from scout.constants import CHROMOSOME_INTEGERS from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER from scout.models.managed_variant import ManagedVariant -from scout.utils.ensembl_rest_clients import EnsemblRestApiClient LOG = logging.getLogger(__name__) @@ -71,48 +69,51 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> """Perform liftover over a list of managed variants and return a list of lines formatted as a managed variants upload infile.""" export_lines = [MANAGED_VARIANTS_INFILE_HEADER] - ensembl_client = EnsemblRestApiClient() - - def liftover_with_retry(variant_obj, retries=3): - for attempt in range(retries): - try: - return ensembl_client.liftover( - build=liftover_from, - chrom=variant_obj["chromosome"], - start=variant_obj["position"], - end=variant_obj.get("end", variant_obj["position"]), - ) - except requests.exceptions.RequestException: - wait = 1 * (attempt + 1) - LOG.warning(f"Retry {attempt + 1}/{retries} after error. Waiting {wait}s") - time.sleep(wait) - return None - - for i, variant_obj in enumerate(managed_variants, 1): + LIFTOVER_API_URL = "https://liftover-xwkwwwxdwq-uc.a.run.app/liftover/" + + lifted_build = "38" if liftover_from == "37" else "37" + build_to = "hg38" if lifted_build == "38" else "hg19" + build_from = "hg19" if build_to == "hg38" else "hg38" + + nfailed = 0 + for i, variant_obj in enumerate(list(managed_variants)[:500], 1): if i % 50 == 0: LOG.info(f"Processed {i} variants") if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue - build = "38" if liftover_from == "37" else "37" - - if variant_obj.get("build") == build: + if variant_obj.get("build") == lifted_build: chrom = variant_obj["chromosome"] pos = variant_obj["position"] end = variant_obj.get("end", variant_obj["position"]) - else: - liftover_result = liftover_with_retry(variant_obj) - - if not liftover_result: + ref = variant_obj.get("reference") + alt = variant_obj.get("alternative") + + else: # Do liftover + params = { + "hg": f"{build_from}-to-{build_to}", + "format": "variant", + "chrom": f"{variant_obj.get('chromosome')}", + "pos": variant_obj.get("position"), + "end": variant_obj.get("end"), + "ref": variant_obj.get("reference", ""), + "alt": variant_obj.get("alternative", ""), + } + response = requests.get(LIFTOVER_API_URL, params=params) + + if response.status_code == 200: + result = response.json() + chrom = result["output_chrom"].replace("chr", "") + pos = result["output_pos"] + end = result.get("output_end") or result.get("output_pos") + ref = result["output_ref"] + alt = result["output_alt"] + else: + nfailed += 1 + LOG.error(response.json()) continue - chrom = liftover_result[0]["mapped"]["seq_region_name"] - pos = liftover_result[0]["mapped"]["start"] - end = liftover_result[0]["mapped"]["end"] - - ref = variant_obj.get("reference", "") - alt = variant_obj.get("alternative", "") category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") description = variant_obj.get("description") @@ -120,12 +121,10 @@ def liftover_with_retry(variant_obj, retries=3): export_lines.append( f"{chrom};{pos};{end};{ref};{alt};" - f"{category};{sub_category};{build};{description};;{institutes}" + f"{category};{sub_category};{lifted_build};{description};;{institutes}" ) - time.sleep(0.2) # gentle rate limiting - - LOG.info(f"Done. Total processed: {i}") + LOG.info(f"Done. Total processed: {i} - total failed: {nfailed}") return export_lines From f06c7515cb63d80125e10fd0f30d76dd453060de Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 6 May 2026 09:07:03 +0200 Subject: [PATCH 27/52] Silence test for now --- tests/export/test_export_variants.py | 40 ++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index 98b28ea215..e43b65e35a 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -36,18 +36,41 @@ def test_export_mt_variants(case_obj, real_populated_database): assert len(sample_lines[0]) == len(MT_EXPORT_HEADER) +""" @responses.activate def test_liftover_managed_variants(ensembl_liftover_response): - """Test the function that performs liftover over a list of managed variants and formats them into a list of strings.""" + Test the function that performs liftover over a list of managed variants and formats them into a list of strings. + + # GIVEN a patched response from the Broad Liftover API + url = "https://liftover-xwkwwwxdwq-uc.a.run.app/liftover/?hg=hg19-to-hg38&format=variant&chrom=X&pos=1000000&end=1000000&ref=C&alt=T" + + mock_resp = { + "hg": "hg19-to-hg38", + "chrom": "X", + "start": 999999, + "end": "1000000", + "output_chrom": "chrX", + "output_pos": 1039265, + "output_ref": "G", + "output_alt": "C,T", + "liftover_tool": "bcftools liftover plugin", + "normalized_chrom": "X", + "normalized_pos": "1000000", + "normalized_ref": "C", + "normalized_alt": "T", + "ref": "C", + "format": "variant", + "alt": "T", + "pos": "1000000", + } - # GIVEN a patched response from Ensembl - url = f"{RESTAPI_URL}/map/human/GRCh37/X:1000000..1000000/GRCh38?content-type=application/json" responses.add( responses.GET, url, - json=ensembl_liftover_response, + json=mock_resp, status=200, ) + managed_variant_info = { "chromosome": "X", "position": "1000000", @@ -68,7 +91,8 @@ def test_liftover_managed_variants(ensembl_liftover_response): assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER # AND second line being the lifted-over variant - lifted_chrom = ensembl_liftover_response["mappings"][0]["mapped"]["seq_region_name"] - lifted_position = ensembl_liftover_response["mappings"][0]["mapped"]["start"] - lifted_end = ensembl_liftover_response["mappings"][0]["mapped"]["end"] - assert f"{lifted_chrom};{lifted_position};{lifted_end}" in export_lines[1] + #lifted_chrom = ensembl_liftover_response["mappings"][0]["mapped"]["seq_region_name"] + #lifted_position = ensembl_liftover_response["mappings"][0]["mapped"]["start"] + #lifted_end = ensembl_liftover_response["mappings"][0]["mapped"]["end"] + #assert f"{lifted_chrom};{lifted_position};{lifted_end}" in export_lines[1] +""" From d33a0939bcacd6494ffede631cc10f27ce9015ff Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 6 May 2026 09:11:49 +0200 Subject: [PATCH 28/52] Of course run them all, not just a part --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index a820d08122..ca035a72f2 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -76,7 +76,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> build_from = "hg19" if build_to == "hg38" else "hg38" nfailed = 0 - for i, variant_obj in enumerate(list(managed_variants)[:500], 1): + for i, variant_obj in enumerate(managed_variants, 1): if i % 50 == 0: LOG.info(f"Processed {i} variants") From 77d6560a46d7fce2daaf903d20d8d3ca56a873a3 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 6 May 2026 09:37:51 +0200 Subject: [PATCH 29/52] Error return 200 status also it seems --- scout/export/variant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index ca035a72f2..1103d4f021 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -101,9 +101,9 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> "alt": variant_obj.get("alternative", ""), } response = requests.get(LIFTOVER_API_URL, params=params) + result = response.json() - if response.status_code == 200: - result = response.json() + if "error" not in result or response.status_code == 200: chrom = result["output_chrom"].replace("chr", "") pos = result["output_pos"] end = result.get("output_end") or result.get("output_pos") From 3811fae8b623270d34e400fa1f1a7b5074ca47fc Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 6 May 2026 09:38:40 +0200 Subject: [PATCH 30/52] Simpler --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 1103d4f021..86d84cba85 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -103,7 +103,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> response = requests.get(LIFTOVER_API_URL, params=params) result = response.json() - if "error" not in result or response.status_code == 200: + if "error" not in result: chrom = result["output_chrom"].replace("chr", "") pos = result["output_pos"] end = result.get("output_end") or result.get("output_pos") From 23160c37419282e3c895c84ef28f074ff5344c5b Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Mon, 11 May 2026 09:10:22 +0200 Subject: [PATCH 31/52] Added liftover tag in managed description field --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 86d84cba85..11375bbcdc 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -116,7 +116,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") - description = variant_obj.get("description") + description = "Lift over variant (37->38). " + variant_obj.get("description") institutes = ",".join(variant_obj.get("institute") or []) export_lines.append( From 0be2fb1cc4bcd96956428559e2d2fac5b26fc69e Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Mon, 11 May 2026 09:12:37 +0200 Subject: [PATCH 32/52] Do not hardcode build --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 11375bbcdc..b91ae5754e 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -116,7 +116,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") - description = "Lift over variant (37->38). " + variant_obj.get("description") + description = f"Lift over variant ({build_from}->{build_to}). " + variant_obj.get("description") institutes = ",".join(variant_obj.get("institute") or []) export_lines.append( From a650ec1ae7b0c02af03779f2766ecf5b81f63e2f Mon Sep 17 00:00:00 2001 From: Lint Action Date: Mon, 11 May 2026 07:13:04 +0000 Subject: [PATCH 33/52] Fix code style issues with Black --- scout/export/variant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index b91ae5754e..899a50b5f8 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -116,7 +116,9 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") - description = f"Lift over variant ({build_from}->{build_to}). " + variant_obj.get("description") + description = f"Lift over variant ({build_from}->{build_to}). " + variant_obj.get( + "description" + ) institutes = ",".join(variant_obj.get("institute") or []) export_lines.append( From 9658f1b5a8562298db6e440b7480f7815a6c742f Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Tue, 12 May 2026 09:04:50 +0200 Subject: [PATCH 34/52] Fixed description string according to the latest convention --- scout/export/variant.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 899a50b5f8..887bc9203e 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -116,9 +116,10 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") - description = f"Lift over variant ({build_from}->{build_to}). " + variant_obj.get( - "description" - ) + if "(causatives" not in variant_obj.get("description"): + description = variant_obj.get("description") + " (managed, build37)" + else: + description = variant_obj.get("description") institutes = ",".join(variant_obj.get("institute") or []) export_lines.append( From ee1a5355d7dd5bdcbd0a3a9e9b35f95c17b26476 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Tue, 12 May 2026 09:05:36 +0200 Subject: [PATCH 35/52] Make it more flexible --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 887bc9203e..5188d20896 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -117,7 +117,7 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> category = variant_obj.get("category", "snv") sub_category = variant_obj.get("sub_category", "snv") if "(causatives" not in variant_obj.get("description"): - description = variant_obj.get("description") + " (managed, build37)" + description = variant_obj.get("description") + f" (managed, build{liftover_from})" else: description = variant_obj.get("description") institutes = ",".join(variant_obj.get("institute") or []) From 3f3923171861fc2fd046f00ea8b9e58482daa3d1 Mon Sep 17 00:00:00 2001 From: Lint Action Date: Wed, 13 May 2026 05:07:48 +0000 Subject: [PATCH 36/52] Fix code style issues with Black --- scout/commands/export/variant.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 3050c13736..7cd59a2754 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -161,9 +161,9 @@ def managed( valid_lines = [] for variant_obj in variants: - variant_string = get_vcf_entry(variant_obj, build=build) - if variant_string: - valid_lines.append(variant_string) + variant_string = get_vcf_entry(variant_obj, build=build) + if variant_string: + valid_lines.append(variant_string) for line in vcf_header: click.echo(line) From da0076f373481bcf81c63bea25ab46eba017004f Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 13 May 2026 07:09:41 +0200 Subject: [PATCH 37/52] Trigger tests From 9fafa5b5ce34df8ded002e4e070053f529592e2c Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Thu, 21 May 2026 15:22:48 +0200 Subject: [PATCH 38/52] Fix isort --- scout/commands/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 420ca174ea..bfc80aa393 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -2,9 +2,9 @@ import json as json_lib import logging import os -from typing import Optional, Tuple import sys from pathlib import Path +from typing import Optional, Tuple import click from flask import current_app From 71100365a43288009cf4949a0edfd5a13ee78f9e Mon Sep 17 00:00:00 2001 From: Lint Action Date: Fri, 22 May 2026 12:52:56 +0000 Subject: [PATCH 39/52] Fix code style issues with Black --- scout/commands/export/variant.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index ac0691ffcf..662a979003 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -130,7 +130,9 @@ def verified(collaborator, test, outpath=None): help="Perform liftover on coordinates and export as managed variants infile.", ) @with_appcontext -def managed(collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str]): +def managed( + collaborator: str, category: Tuple[str], build: str, json: bool, liftover_from: Optional[str] +): """Export managed variants for a collaborator in VCF or JSON format""" LOG.info("Running scout export managed variants") adapter = store @@ -147,7 +149,9 @@ def managed(collaborator: str, category: Tuple[str], build: str, json: bool, lif click.echo(json_lib.dumps([var for var in variants], default=bson_handler)) return - print_vcf(variants=variants, build=build, export_category="MANAGED", liftover_from=liftover_from) + print_vcf( + variants=variants, build=build, export_category="MANAGED", liftover_from=liftover_from + ) def resolve_case( @@ -249,4 +253,4 @@ def causatives( click.echo(line) return - print_vcf(variants=causatives, build=build, export_category="CAUSATIVE", case_obj=case_obj) \ No newline at end of file + print_vcf(variants=causatives, build=build, export_category="CAUSATIVE", case_obj=case_obj) From 770b82924a0f7f5873a696f6924d06529a5965e6 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 22 May 2026 15:22:09 +0200 Subject: [PATCH 40/52] How about this --- scout/commands/export/variant.py | 14 +++++++++----- scout/export/variant.py | 9 ++++++--- scout/utils/vcf.py | 3 ++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 662a979003..3b5f04e77a 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -8,16 +8,17 @@ from flask.cli import with_appcontext from xlsxwriter import Workbook -from scout.constants import CALLERS, DATE_DAY_FORMATTER +from scout.constants import BUILDS, CALLERS, DATE_DAY_FORMATTER from scout.constants.managed_variant import MANAGED_CATEGORIES from scout.constants.variants_export import VERIFIED_VARIANTS_HEADER from scout.export.variant import ( + export_lift_over_managed_variants, export_managed_variants, export_verified_variants, ) from scout.server.blueprints.institutes.controllers import variants_to_managed_variants from scout.server.extensions import store -from scout.utils.vcf import build_vcf_header, print_vcf, validate_vcf_line +from scout.utils.vcf import print_vcf from .export_handler import bson_handler from .utils import build_option, category_option, collaborator_option, json_option @@ -148,10 +149,13 @@ def managed( if json: click.echo(json_lib.dumps([var for var in variants], default=bson_handler)) return + if liftover_from: + lines = export_lift_over_managed_variants( + managed_variants=variants, liftover_from=liftover_from + ) + return - print_vcf( - variants=variants, build=build, export_category="MANAGED", liftover_from=liftover_from - ) + print_vcf(variants=variants, build=build, export_category="MANAGED") def resolve_case( diff --git a/scout/export/variant.py b/scout/export/variant.py index 45fbf0d6ce..82393cc65a 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- import logging import urllib.parse -from typing import Iterable, List, Optional +from typing import Iterable, List +import click import requests from scout.adapter.mongo.base import MongoAdapter @@ -25,8 +26,8 @@ def sort_key(var: dict): return sorted(variants, key=sort_key) -def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> List[str]: - """Perform liftover over a list of managed variants and return a list of lines formatted as a managed variants upload infile.""" +def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: str): + """Perform liftover over a list of managed variants and print a list of lines formatted as a managed variants upload infile.""" export_lines = [MANAGED_VARIANTS_INFILE_HEADER] LIFTOVER_API_URL = "https://liftover-xwkwwwxdwq-uc.a.run.app/liftover/" @@ -88,6 +89,8 @@ def liftover_managed_variants(managed_variants: Iterable, liftover_from: str) -> ) LOG.info(f"Done. Total processed: {i} - total failed: {nfailed}") + for line in export_lines: + click.echo(line) return export_lines diff --git a/scout/utils/vcf.py b/scout/utils/vcf.py index b50dbaf699..289bd84b3d 100644 --- a/scout/utils/vcf.py +++ b/scout/utils/vcf.py @@ -271,9 +271,9 @@ def print_vcf( ) -> None: """ Print variants in VCF format. - If a case_id is provided, the VCF header is extended with FORMAT and per-individual genotype columns. + If liftover_from is provided, then liftover is performed on variant before the export line is printed. """ argv = [Path(sys.argv[0]).name] + sys.argv[1:] @@ -290,6 +290,7 @@ def print_vcf( click.echo(line) for variant_obj in variants: + if variant_string := get_vcf_entry( variant_obj, case_id=case_obj["_id"] if case_obj else None, From 049ad3724f25edcc248fbc41a0575e8fe789caa8 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 22 May 2026 15:33:40 +0200 Subject: [PATCH 41/52] Fix renamed import in test --- tests/export/test_export_variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index e43b65e35a..2acc109b14 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -4,7 +4,7 @@ from scout.constants.managed_variant import MANAGED_VARIANTS_INFILE_HEADER from scout.constants.variants_export import MT_EXPORT_HEADER -from scout.export.variant import export_mt_variants, liftover_managed_variants +from scout.export.variant import export_mt_variants from scout.utils.ensembl_rest_clients import RESTAPI_URL From ac987294da33d81c9589bc519e53e0f202964467 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 11:35:06 +0200 Subject: [PATCH 42/52] Fix conflicts --- scout/commands/export/variant.py | 4 +--- scout/export/variant.py | 33 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/scout/commands/export/variant.py b/scout/commands/export/variant.py index 3b5f04e77a..ff3d9566e7 100644 --- a/scout/commands/export/variant.py +++ b/scout/commands/export/variant.py @@ -150,9 +150,7 @@ def managed( click.echo(json_lib.dumps([var for var in variants], default=bson_handler)) return if liftover_from: - lines = export_lift_over_managed_variants( - managed_variants=variants, liftover_from=liftover_from - ) + export_lift_over_managed_variants(managed_variants=variants, liftover_from=liftover_from) return print_vcf(variants=variants, build=build, export_category="MANAGED") diff --git a/scout/export/variant.py b/scout/export/variant.py index 82393cc65a..29ed9e9d0b 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -4,12 +4,12 @@ from typing import Iterable, List import click -import requests from scout.adapter.mongo.base import MongoAdapter from scout.constants import CHROMOSOME_INTEGERS from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER from scout.models.managed_variant import ManagedVariant +from scout.utils.broad_liftover_client import BroadLiftoverApiClient LOG = logging.getLogger(__name__) @@ -30,15 +30,16 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: """Perform liftover over a list of managed variants and print a list of lines formatted as a managed variants upload infile.""" export_lines = [MANAGED_VARIANTS_INFILE_HEADER] - LIFTOVER_API_URL = "https://liftover-xwkwwwxdwq-uc.a.run.app/liftover/" + client = BroadLiftoverApiClient() lifted_build = "38" if liftover_from == "37" else "37" build_to = "hg38" if lifted_build == "38" else "hg19" build_from = "hg19" if build_to == "hg38" else "hg38" nfailed = 0 - for i, variant_obj in enumerate(managed_variants, 1): - if i % 50 == 0: + nprocessed = 0 + for nprocessed, variant_obj in enumerate(managed_variants, 1): + if nprocessed % 50 == 0: LOG.info(f"Processed {i} variants") if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: @@ -52,17 +53,14 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: alt = variant_obj.get("alternative") else: # Do liftover - params = { - "hg": f"{build_from}-to-{build_to}", - "format": "variant", - "chrom": f"{variant_obj.get('chromosome')}", - "pos": variant_obj.get("position"), - "end": variant_obj.get("end"), - "ref": variant_obj.get("reference", ""), - "alt": variant_obj.get("alternative", ""), - } - response = requests.get(LIFTOVER_API_URL, params=params) - result = response.json() + result = client.liftover( + build_from=build_from, + chrom=variant_obj.get("chromosome"), + start=variant_obj.get("position"), + end=variant_obj.get("end"), + ref=variant_obj.get("reference", ""), + alt=variant_obj.get("alternative", ""), + ) if "error" not in result: chrom = result["output_chrom"].replace("chr", "") @@ -72,7 +70,7 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: alt = result["output_alt"] else: nfailed += 1 - LOG.error(response.json()) + LOG.error(result) continue category = variant_obj.get("category", "snv") @@ -88,10 +86,9 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: f"{category};{sub_category};{lifted_build};{description};;{institutes}" ) - LOG.info(f"Done. Total processed: {i} - total failed: {nfailed}") + LOG.info(f"Done. Total processed: {nprocessed} - total failed: {nfailed}") for line in export_lines: click.echo(line) - return export_lines def export_managed_variants( From 932e3865e25ec31caa4ecd95a0d146d79dc5acac Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 11:39:10 +0200 Subject: [PATCH 43/52] Remove old imports from test file --- tests/export/test_export_variants.py | 67 ---------------------------- 1 file changed, 67 deletions(-) diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index 2acc109b14..b1f57faf76 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -1,11 +1,6 @@ # -*- coding: utf-8 -*- - -import responses - -from scout.constants.managed_variant import MANAGED_VARIANTS_INFILE_HEADER from scout.constants.variants_export import MT_EXPORT_HEADER from scout.export.variant import export_mt_variants -from scout.utils.ensembl_rest_clients import RESTAPI_URL def test_export_mt_variants(case_obj, real_populated_database): @@ -34,65 +29,3 @@ def test_export_mt_variants(case_obj, real_populated_database): assert len(sample_lines) == len(mt_variants) # check that cols to write to excel correspond to fields of Excel header assert len(sample_lines[0]) == len(MT_EXPORT_HEADER) - - -""" -@responses.activate -def test_liftover_managed_variants(ensembl_liftover_response): - Test the function that performs liftover over a list of managed variants and formats them into a list of strings. - - # GIVEN a patched response from the Broad Liftover API - url = "https://liftover-xwkwwwxdwq-uc.a.run.app/liftover/?hg=hg19-to-hg38&format=variant&chrom=X&pos=1000000&end=1000000&ref=C&alt=T" - - mock_resp = { - "hg": "hg19-to-hg38", - "chrom": "X", - "start": 999999, - "end": "1000000", - "output_chrom": "chrX", - "output_pos": 1039265, - "output_ref": "G", - "output_alt": "C,T", - "liftover_tool": "bcftools liftover plugin", - "normalized_chrom": "X", - "normalized_pos": "1000000", - "normalized_ref": "C", - "normalized_alt": "T", - "ref": "C", - "format": "variant", - "alt": "T", - "pos": "1000000", - } - - responses.add( - responses.GET, - url, - json=mock_resp, - status=200, - ) - - managed_variant_info = { - "chromosome": "X", - "position": "1000000", - "reference": "C", - "alternative": "T", - "build": "37", - } - - # GIVEN a list of managed variants - managed_variants = [managed_variant_info] - - # THEN the liftover function should export them correctly: - export_lines = liftover_managed_variants( - managed_variants=iter(managed_variants), liftover_from="37" - ) - - # WITH the first line being the header - assert export_lines[0] == MANAGED_VARIANTS_INFILE_HEADER - - # AND second line being the lifted-over variant - #lifted_chrom = ensembl_liftover_response["mappings"][0]["mapped"]["seq_region_name"] - #lifted_position = ensembl_liftover_response["mappings"][0]["mapped"]["start"] - #lifted_end = ensembl_liftover_response["mappings"][0]["mapped"]["end"] - #assert f"{lifted_chrom};{lifted_position};{lifted_end}" in export_lines[1] -""" From 6c4d2fd74b9e125f07dd227396542467ce804d00 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 11:40:53 +0200 Subject: [PATCH 44/52] Full rever vcf utils --- scout/utils/vcf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scout/utils/vcf.py b/scout/utils/vcf.py index 92b87ce863..f44f49462c 100644 --- a/scout/utils/vcf.py +++ b/scout/utils/vcf.py @@ -272,9 +272,9 @@ def print_vcf( ) -> None: """ Print variants in VCF format. + If a case_id is provided, the VCF header is extended with FORMAT and per-individual genotype columns. - If liftover_from is provided, then liftover is performed on variant before the export line is printed. """ argv = [Path(sys.argv[0]).name] + sys.argv[1:] @@ -291,7 +291,6 @@ def print_vcf( click.echo(line) for variant_obj in variants: - if variant_string := get_vcf_entry( variant_obj, case_id=case_obj["_id"] if case_obj else None, From 0b40a0e3c2894f71a3d79649efaa7aebd971f790 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 11:41:48 +0200 Subject: [PATCH 45/52] Fixed changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ceb6ac151..74ba77c48b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). About changelog [here](https://keepachangelog.com/en/1.0.0/) ## [unreleased] +### Added +- A `--liftover-from` option to the `export managed` command line, so managed variants can be exported and imported again in another genome build (#6225) ### Changed - Replaced Ensembl rest liftover service with liftover API from the Broad Institute (#6293) - Avoid fetching genes and panels multiple times when loading variants (#6350) @@ -73,7 +75,6 @@ About changelog [here](https://keepachangelog.com/en/1.0.0/) - Command line option to export causative variants by category and/or genome build (#6202) - An additional button on causatives and verified pages to download SNVs and SVs as input for the managed variants list (admins only) (#6205 and #6231) - Display total number of variants and number of variants returned by a filter on Managed Variants page (#6223) -- A `--liftover-from` option to the `export managed` command line, so managed variants can be exported and imported again in another genome build (#6225) ### Changed - Genome build is now shown on variant verification "Sanger" emails (#6194) - Refactor, speedup (dry-run only) and add a progress bar to `scout delete variants` cmd (#6094) From f4bfa5a13f0e7de417a88679d627ce03da7ccb03 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 11:43:10 +0200 Subject: [PATCH 46/52] Fix final typo --- scout/export/variant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 29ed9e9d0b..625f0d187b 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -40,7 +40,7 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: nprocessed = 0 for nprocessed, variant_obj in enumerate(managed_variants, 1): if nprocessed % 50 == 0: - LOG.info(f"Processed {i} variants") + LOG.info(f"Processed {nprocessed} variants") if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: continue From d8eba1ab27390091965f1d550b4a29fe00662d2f Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 14:05:55 +0200 Subject: [PATCH 47/52] Remove redundant code --- scout/export/variant.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 625f0d187b..342073f018 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -32,10 +32,6 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: export_lines = [MANAGED_VARIANTS_INFILE_HEADER] client = BroadLiftoverApiClient() - lifted_build = "38" if liftover_from == "37" else "37" - build_to = "hg38" if lifted_build == "38" else "hg19" - build_from = "hg19" if build_to == "hg38" else "hg38" - nfailed = 0 nprocessed = 0 for nprocessed, variant_obj in enumerate(managed_variants, 1): @@ -54,7 +50,7 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: else: # Do liftover result = client.liftover( - build_from=build_from, + build_from=liftover_from, chrom=variant_obj.get("chromosome"), start=variant_obj.get("position"), end=variant_obj.get("end"), From 21d37ea6f2a090dadd5944584ce6746dcb4f1316 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 14:08:04 +0200 Subject: [PATCH 48/52] I removed too much --- scout/export/variant.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 342073f018..da88f6fb01 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -32,6 +32,10 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: export_lines = [MANAGED_VARIANTS_INFILE_HEADER] client = BroadLiftoverApiClient() + build_from, build_to + + lifted_build = "38" if liftover_from == "37" else "37" + nfailed = 0 nprocessed = 0 for nprocessed, variant_obj in enumerate(managed_variants, 1): @@ -50,7 +54,7 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: else: # Do liftover result = client.liftover( - build_from=liftover_from, + build_from=build_from, chrom=variant_obj.get("chromosome"), start=variant_obj.get("position"), end=variant_obj.get("end"), From ea401b396e5b03b37c26d9a032025d361b933c0d Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Fri, 29 May 2026 14:11:32 +0200 Subject: [PATCH 49/52] Some more fixes --- scout/export/variant.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index da88f6fb01..332e7d340f 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -32,8 +32,6 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: export_lines = [MANAGED_VARIANTS_INFILE_HEADER] client = BroadLiftoverApiClient() - build_from, build_to - lifted_build = "38" if liftover_from == "37" else "37" nfailed = 0 @@ -54,7 +52,7 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: else: # Do liftover result = client.liftover( - build_from=build_from, + build_from=liftover_from, chrom=variant_obj.get("chromosome"), start=variant_obj.get("position"), end=variant_obj.get("end"), From 9471d7aa6210b068bc219df108011a7d439af24d Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Mon, 1 Jun 2026 10:01:57 +0200 Subject: [PATCH 50/52] Simplify code --- scout/export/variant.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 332e7d340f..6d78581322 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -36,21 +36,24 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: nfailed = 0 nprocessed = 0 + for nprocessed, variant_obj in enumerate(managed_variants, 1): if nprocessed % 50 == 0: LOG.info(f"Processed {nprocessed} variants") - if variant_obj.get("category", "snv") not in ["snv", "cancer_snv"]: + category = variant_obj.get("category", "snv") + if category not in ["snv", "cancer_snv"]: continue - if variant_obj.get("build") == lifted_build: + build = variant_obj.get("build") + + if build == lifted_build: chrom = variant_obj["chromosome"] pos = variant_obj["position"] - end = variant_obj.get("end", variant_obj["position"]) + end = variant_obj.get("end", pos) ref = variant_obj.get("reference") alt = variant_obj.get("alternative") - - else: # Do liftover + else: result = client.liftover( build_from=liftover_from, chrom=variant_obj.get("chromosome"), @@ -60,23 +63,25 @@ def export_lift_over_managed_variants(managed_variants: Iterable, liftover_from: alt=variant_obj.get("alternative", ""), ) - if "error" not in result: - chrom = result["output_chrom"].replace("chr", "") - pos = result["output_pos"] - end = result.get("output_end") or result.get("output_pos") - ref = result["output_ref"] - alt = result["output_alt"] - else: + if "error" in result: nfailed += 1 LOG.error(result) continue - category = variant_obj.get("category", "snv") + chrom = result["output_chrom"].replace("chr", "") + pos = result["output_pos"] + end = result.get("output_end") or result.get("output_pos") + ref = result["output_ref"] + alt = result["output_alt"] + sub_category = variant_obj.get("sub_category", "snv") - if "(causatives" not in variant_obj.get("description"): - description = variant_obj.get("description") + f" (managed, build{liftover_from})" + + desc = variant_obj.get("description") + if "(causatives" not in (desc or ""): + description = f"{desc} (managed, build{liftover_from})" else: - description = variant_obj.get("description") + description = desc + institutes = ",".join(variant_obj.get("institute") or []) export_lines.append( From c816db65800aff8236904cf7ff80af86e4a533a5 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Wed, 17 Jun 2026 14:05:55 +0200 Subject: [PATCH 51/52] Add a test --- tests/conftest.py | 27 +++++++++++++ tests/export/test_export_variants.py | 60 +++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ec2b556f4e..a6ee132f42 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -230,6 +230,33 @@ def broad_ucsc_liftover_response(): return _response +@pytest.fixture +def broad_bcftools_liftover_response(): + """Returns a response from the Broad Institute's liftover API - BCFtools tool.""" + _response = { + "alt": "G", + "chrom": "chr8", + "ref": "T", + "pos": "141310715", + "end": "141310715", + "format": "variant", + "hg": "hg19-to-hg38", + "start": 141310714, + "output_chrom": "chr8", + "output_pos": 140300616, # This is liftover position + "output_ref": "T", + "output_alt": "G", + "output_reverse_complemented": False, + "output_ref_alt_swap": None, + "liftover_tool": "bcftools liftover plugin", + "normalized_chrom": "8", + "normalized_pos": "141310715", + "normalized_ref": "T", + "normalized_alt": "G", + } + return _response + + @pytest.fixture(scope="function") def gene_bulk(genes): """Return a list with HgncGene objects""" diff --git a/tests/export/test_export_variants.py b/tests/export/test_export_variants.py index b1f57faf76..e89a1fc876 100644 --- a/tests/export/test_export_variants.py +++ b/tests/export/test_export_variants.py @@ -1,6 +1,9 @@ -# -*- coding: utf-8 -*- +import responses + +from scout.constants.managed_variant import MANAGED_VARIANTS_INFILE_HEADER from scout.constants.variants_export import MT_EXPORT_HEADER -from scout.export.variant import export_mt_variants +from scout.export.variant import export_lift_over_managed_variants, export_mt_variants +from scout.utils.broad_liftover_client import LIFTOVER_URL def test_export_mt_variants(case_obj, real_populated_database): @@ -29,3 +32,56 @@ def test_export_mt_variants(case_obj, real_populated_database): assert len(sample_lines) == len(mt_variants) # check that cols to write to excel correspond to fields of Excel header assert len(sample_lines[0]) == len(MT_EXPORT_HEADER) + + +@responses.activate +def test_export_lift_over_managed_variants(broad_bcftools_liftover_response, capsys): + """Test the function lifts over and formats managed variants into a managed variants infile.""" + + # GIVEN an Iterable with managed variants: + managed_variant = { + "chromosome": "8", + "position": 141310715, + "end": 141310715, + "reference": "T", + "alternative": "G", + "category": "snv", + "build": "37", + } + + managed_variants = [managed_variant] + + url = ( + f"{LIFTOVER_URL}/" + "?hg=hg19-to-hg38" + "&format=variant" + f"&chrom={managed_variant['chromosome']}" + f"&pos={managed_variant['position']}" + f"&end={managed_variant['end']}" + f"&ref={managed_variant['reference']}" + f"&alt={managed_variant['alternative']}" + ) + + # GIVEN a mocked call to the liftover service (BCFTools via Broad institute's API) + resp = broad_bcftools_liftover_response + responses.add( + responses.GET, + url, + json=resp, + status=200, + ) + + # WHEN exporting the managed variants + export_lift_over_managed_variants( + managed_variants=managed_variants, + liftover_from="37", + ) + + # THEN a header and a lifted variant line should be printed + out = capsys.readouterr().out.splitlines() + + assert out[0] == MANAGED_VARIANTS_INFILE_HEADER + assert ( + out[1] + == f"{resp['chrom'].replace('chr','')};{resp['output_pos']};{resp['output_pos']};{resp['output_ref']};{resp['output_alt']};snv;snv;38;None (managed, build37);;" + ) From 9b234bbe3da5a09ea08059ef6431778b5082f474 Mon Sep 17 00:00:00 2001 From: Chiara Rasi Date: Tue, 23 Jun 2026 10:49:17 +0200 Subject: [PATCH 52/52] Fixed sorting of a file --- scout/export/variant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scout/export/variant.py b/scout/export/variant.py index 487ed7d834..b4e49cb805 100644 --- a/scout/export/variant.py +++ b/scout/export/variant.py @@ -9,7 +9,6 @@ from scout.constants import CHROMOSOME_INTEGERS from scout.constants.managed_variant import MANAGED_CATEGORIES, MANAGED_VARIANTS_INFILE_HEADER from scout.constants.query_terms import GT_NO_ALT_CALL - from scout.models.managed_variant import ManagedVariant from scout.utils.broad_liftover_client import BroadLiftoverApiClient