diff --git a/.secrets.baseline b/.secrets.baseline index 7e4bc4ec..2f0505bd 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -128,7 +128,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "a87f1bc7c5a21db4b7dcee915801748078c6b7f1", "is_verified": false, - "line_number": 170, + "line_number": 169, "is_secret": false }, { @@ -136,7 +136,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "cb4e7ee7bb0fbd6e89ef3e033bcbd982ff5a7c7b", "is_verified": false, - "line_number": 193, + "line_number": 192, "is_secret": false }, { @@ -144,7 +144,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "07d5c4e95ae395857eaaead6ebbc48921f5d1d73", "is_verified": false, - "line_number": 206, + "line_number": 205, "is_secret": false }, { @@ -152,7 +152,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "e12f4062d4ace7aafbbf20e8231f6cd65bd4df9d", "is_verified": false, - "line_number": 1016, + "line_number": 1015, "is_secret": false }, { @@ -160,7 +160,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "e379fbc5150551e3a339911738efbc4ce8973f23", "is_verified": false, - "line_number": 1031, + "line_number": 1030, "is_secret": false }, { @@ -168,7 +168,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "a79227640041125c18d7253d944f60a061cbed77", "is_verified": false, - "line_number": 1047, + "line_number": 1046, "is_secret": false }, { @@ -176,7 +176,7 @@ "filename": "tests/integration/data.py", "hashed_secret": "7e804e59287fe4f3a27459fc546dc800870ea43f", "is_verified": false, - "line_number": 1071, + "line_number": 1070, "is_secret": false } ], @@ -191,5 +191,5 @@ } ] }, - "generated_at": "2024-03-12T19:34:21Z" + "generated_at": "2024-06-12T15:38:28Z" } diff --git a/esbuild/gdc_elasticsearch.py b/esbuild/gdc_elasticsearch.py index 6e2c4155..e8076e7c 100644 --- a/esbuild/gdc_elasticsearch.py +++ b/esbuild/gdc_elasticsearch.py @@ -357,6 +357,12 @@ def go(self, roll_alias=True, send_events=True): index_prefix=self.index_prefix, ) + if self.build_projects: + logger.info("Running partial build") + logger.info(f"Projects: {self.build_projects}") + else: + logger.info("Running full build") + cases, files, annotations, projects = self._cache_database(self.converter) logger.info("Validating docs produced") diff --git a/esbuild/graph/active/builder.py b/esbuild/graph/active/builder.py index 441c94a9..f1411f60 100644 --- a/esbuild/graph/active/builder.py +++ b/esbuild/graph/active/builder.py @@ -19,10 +19,21 @@ import itertools import logging -from typing import Iterable, List, Optional, Sequence, Set - +from collections.abc import ( + Collection, + Container, + Iterable, + Iterator, + Mapping, + Sequence, + Set, +) +from typing import Any, Optional + +import more_itertools import psqlgraph from gdcdatamodel2 import models +from indexclient import client from esbuild.graph.common import builder, validators @@ -30,46 +41,35 @@ FILTERED_FILE_STATUSES = frozenset(("ignore", "error")) -def reverse_and_skip_first_entry(path: List) -> List: - """Return a path that. - - 1. is reversed and - 2. has the first step (in reversed order) removed - - This was created to traverse paths in reverse order such that the - first entry in the path is skipped because we are already visiting - that node. Example: ``['a', 'b', 'c'] -> ['b', 'a']`` - - """ - return path[-2::-1] - - -def list_product(roots: List, subtrees: List) -> List: +def _path_product( + roots: Iterable[Iterable[str]], subtrees: Iterable[Iterable[str]] +) -> Iterable[Sequence[str]]: """Append each subtree to each root. It's not quite a cartesian product, example:: roots = [['a', 'b'], ['-', '#']] subtrees = [range(0, 2), range(2, 4), range(4, 8)] - list(list_product(roots, subtrees)) - [['a', 'b', 0, 1], - ['a', 'b', 2, 3], - ['a', 'b', 4, 5, 6, 7], - ['-', '#', 0, 1], - ['-', '#', 2, 3], - ['-', '#', 4, 5, 6, 7]] + _path_product(roots, subtrees) -> + ( + ('a', 'b', 0, 1), + ('a', 'b', 2, 3), + ('a', 'b', 4, 5, 6, 7), + ('-', '#', 0, 1), + ('-', '#', 2, 3), + ('-', '#', 4, 5, 6, 7) + ) """ - return [root + subtree for root in roots for subtree in subtrees] + return tuple((*p0, *p1) for p0 in roots for p1 in subtrees) -def subtree_paths_to_file( - cls, - paths: Optional[List[List[str]]] = None, - visited: Optional[List[str]] = None, - categories: Optional[Set[str]] = None, - exclude_paths_through: Optional[set] = None, -) -> List[List[str]]: +def _subtree_paths_to_file( + cls: type[models.Node], + visited: tuple[str, ...] = (), + categories: Container[str] = frozenset(("data_file", "analysis")), + exclude_paths_through: Container[str] = frozenset(), +) -> Iterator[Sequence[str]]: """Find paths to file nodes in subtree. Recurse through all child nodes in categories :param:`categories` @@ -86,17 +86,8 @@ def subtree_paths_to_file( Returns: paths to file nodes """ - if categories is None: - categories = {"data_file", "analysis"} - - if exclude_paths_through is None: - exclude_paths_through = set() - - visited = visited if visited is not None else [] - paths = paths if paths is not None else [] - if cls._dictionary["category"] == "data_file": - paths.append(visited) + yield visited for backref in cls._pg_backrefs.values(): child = backref["src_type"] @@ -108,56 +99,39 @@ def subtree_paths_to_file( ) if should_recur: - subtree_paths_to_file( + yield from _subtree_paths_to_file( child, - paths, - visited=visited + [child.label], + visited=(*visited, child.label), exclude_paths_through=exclude_paths_through, ) - return paths +def _get_case_to_file_paths() -> Iterable[Sequence[str]]: + """Build all paths from case to files. -class ActiveGraphIndexBuilder(builder.GraphIndexBuilder): - """The builder for the current graph indices. + Since the Active index has more complicated paths from case to file, this is an + attempt not to hard code them. This process, with the exception of files related to + read groups, builds the paths manually. - Since the Active index has more complicated paths from case to - file, this is an attempt not to hard code them. See module doc. - """ + TODO: DEV-2768: Currently this is done completely manually. However, we can and prob + should build these paths using the same logic we currently use for read groups + but start at the case node and walk all possible paths to our set of desired + file types. We should create a whitelist of files which should be included in + the build (preferably configurable) which will insure only the intended files + are release. This will help us from accidentally forgetting a path or missing a + path that is later introduced to the graph, but still be able to control which + files types are released. - # Skip any paths that traverse through nodes in - # ``exclude_paths_through``. - # - # In the index, AlignedReads were associated with two aliquots - # because they go through the Alignment Cocleaning - # Workflow. However, they should have edges directly back to a - # single SubmittedAlignedReads that goes back to a single - # aliquot. They should only be associated with this aliquot. - # - # The impact is that the user can not filter properly on the - # sample types, e.g. tumor versus normal as it returns all of the - # AlignedReads. - # - # The solution applied here is to simply remove paths through - # specific nodes and rely on the shortcut edges when traversing to - # Read Groups. - # - # See PGDC-2349 for details. - exclude_paths_through = { - "alignment_cocleaning_workflow", - } - - # Filter nodes out if their properties are a superset of any of - # the dictionaries listed here by label - unindexed_by_property = { - "annotation": [{"status": "Rescinded"}, {"classification": "Blocking Release"}], - } - - case_to_aliquot = [ - ["sample", "aliquot"], - ["sample", "analyte", "aliquot"], - ["sample", "portion", "analyte", "aliquot"], - ] + Returns: + A collection of string sequences which each represent a path (label to label; + node to node) starting with the expected child of a case node and traversing to + the file node label. + """ + case_to_aliquot = ( + ("sample", "aliquot"), + ("sample", "analyte", "aliquot"), + ("sample", "portion", "analyte", "aliquot"), + ) # BREADCRUMB # Holy hell. Ok, the following lists are paths to where @@ -168,10 +142,27 @@ class ActiveGraphIndexBuilder(builder.GraphIndexBuilder): # a list below, then add it to the case_to_file_paths # - a very tired joe sislow (3/15/2018) - readgroup_subtree = list_product( - [[models.ReadGroup.label]], - subtree_paths_to_file( - models.ReadGroup, exclude_paths_through=exclude_paths_through + readgroup_subtree = _path_product( + ((models.ReadGroup.label,),), + _subtree_paths_to_file( + models.ReadGroup, + # Skip any paths that traverse through nodes in + # ``exclude_paths_through``. + # + # In the index, AlignedReads were associated with two aliquots + # because they go through the Alignment Cocleaning + # Workflow. However, they should have edges directly back to a + # single SubmittedAlignedReads that goes back to a single + # aliquot. They should only be associated with this aliquot. + # + # The impact is that the user can not filter properly on the + # sample types, e.g. tumor versus normal as it returns all of the + # AlignedReads. + # + # The solution applied here is to simply remove paths through + # specific nodes and rely on the shortcut edges when traversing to + # Read Groups. + exclude_paths_through=("alignment_cocleaning_workflow",), ), ) @@ -187,203 +178,215 @@ class ActiveGraphIndexBuilder(builder.GraphIndexBuilder): # should be used. # - joe sislow (11/27/2018) - aliquot_to_copy_number_segment_paths = [ - [ + aliquot_to_copy_number_segment_paths = ( + ( "submitted_tangent_copy_number", "copy_number_liftover_workflow", "copy_number_segment", - ], - [ + ), + ( "submitted_genotyping_array", "somatic_copy_number_workflow", "copy_number_segment", - ], - ] + ), + ) - aliquot_to_copy_number_estimate_paths = [ - [ + aliquot_to_copy_number_estimate_paths = ( + ( "submitted_tangent_copy_number", "copy_number_liftover_workflow", "copy_number_segment", "copy_number_variation_workflow", "copy_number_estimate", - ], - [ + ), + ( "submitted_genotyping_array", "somatic_copy_number_workflow", "copy_number_estimate", - ], - ] + ), + ) - aliquot_to_methylation_value_paths = [ - [ + aliquot_to_methylation_value_paths = ( + ( "submitted_methylation_beta_value", "methylation_liftover_workflow", "methylation_beta_value", - ], - [ + ), + ( "raw_methylation_array", "methylation_array_harmonization_workflow", "methylation_beta_value", - ], - ] + ), + ) # added for slide_image by joe, 3/18 - case_to_slide_image_path = [ - ["sample", "slide", "slide_image"], - ["sample", "portion", "slide", "slide_image"], - ] - - case_to_file_paths = [ - ["biospecimen_supplement"], - ["clinical_supplement"], - ["sample", "pathology_report"], - ] - - case_to_copy_number_segment_paths = list_product( + case_to_slide_image_path = ( + ("sample", "slide", "slide_image"), + ("sample", "portion", "slide", "slide_image"), + ) + + case_to_copy_number_segment_paths = _path_product( case_to_aliquot, aliquot_to_copy_number_segment_paths ) - case_to_copy_number_estimate_paths = list_product( + case_to_copy_number_estimate_paths = _path_product( case_to_aliquot, aliquot_to_copy_number_estimate_paths ) - case_to_protein_expression = [ - ["sample", "protein_expression"], - ["sample", "portion", "protein_expression"], - ] + case_to_protein_expression = ( + ("sample", "protein_expression"), + ("sample", "portion", "protein_expression"), + ) - case_to_methylation_value_paths = list_product( + case_to_methylation_value_paths = _path_product( case_to_aliquot, aliquot_to_methylation_value_paths ) - case_to_raw_methylation_array_paths = list_product( - case_to_aliquot, [["raw_methylation_array"]] + case_to_raw_methylation_array_paths = _path_product( + case_to_aliquot, (("raw_methylation_array",),) ) - case_to_masked_methylation_array_paths = list_product( + case_to_masked_methylation_array_paths = _path_product( case_to_aliquot, - [ - [ + ( + ( "raw_methylation_array", "methylation_array_harmonization_workflow", "masked_methylation_array", - ] - ], + ), + ), ) - case_to_genotyping_array_paths = list_product( - case_to_aliquot, [["submitted_genotyping_array"]] + case_to_genotyping_array_paths = _path_product( + case_to_aliquot, (("submitted_genotyping_array",),) ) - case_to_germline_variation_paths = list_product( + case_to_germline_variation_paths = _path_product( case_to_genotyping_array_paths, - [["germline_mutation_calling_workflow", "simple_germline_variation"]], + (("germline_mutation_calling_workflow", "simple_germline_variation"),), ) - case_to_file_paths += list_product(case_to_aliquot, readgroup_subtree) - case_to_file_paths += case_to_copy_number_segment_paths - case_to_file_paths += case_to_copy_number_estimate_paths - case_to_file_paths += case_to_methylation_value_paths - case_to_file_paths += case_to_slide_image_path - case_to_file_paths += case_to_protein_expression - case_to_file_paths += case_to_raw_methylation_array_paths - case_to_file_paths += case_to_masked_methylation_array_paths - case_to_file_paths += case_to_genotyping_array_paths - case_to_file_paths += case_to_germline_variation_paths - - file_labels = builder.GraphIndexBuilder.node_labels_by_category( - [ - "data_file", - "index_file", - ] + return tuple( + itertools.chain( + ( + ("biospecimen_supplement",), + ("clinical_supplement",), + ("sample", "pathology_report"), + ), + _path_product(case_to_aliquot, readgroup_subtree), + case_to_copy_number_segment_paths, + case_to_copy_number_estimate_paths, + case_to_methylation_value_paths, + case_to_slide_image_path, + case_to_protein_expression, + case_to_raw_methylation_array_paths, + case_to_masked_methylation_array_paths, + case_to_genotyping_array_paths, + case_to_germline_variation_paths, + ) + ) + + +def _get_file_labels() -> Collection[str]: + """Get the file labels for the build. + + This will return all nodes with a category of `data_file` & `index_file` except for + the Archive and File node. + + Returns: + The node labels associated with the file types to include in the build. + """ + file_categories = frozenset(("data_file", "index_file")) + excluded_labels = frozenset(("archive", "file")) + + return frozenset( + n.label + for n in models.Node.get_subclasses() + if n._dictionary["category"] in file_categories + and n.label not in excluded_labels + ) + + +def _get_paths_from_files( + paths_to_files: Iterable[Sequence[str]], + destination: str, + included_files: Optional[Container[str]] = None, +) -> Mapping[str, Iterable[Sequence[str]]]: + def restructure_path(path: Sequence[str]) -> Sequence[str]: + restructured_path: Iterable[str] = reversed(path[:-1]) + restructured_path = more_itertools.takewhile_inclusive( + lambda e: e != destination, restructured_path + ) + + return tuple(restructured_path) + + def is_path_included(path: Sequence[str]) -> bool: + if included_files and path[-1] not in included_files: + return False + + return destination in path + + paths_to_files = filter(is_path_included, paths_to_files) + + return more_itertools.map_reduce( + paths_to_files, keyfunc=lambda p: p[-1], valuefunc=restructure_path ) - # Do not create file docs for archives - file_labels.remove("archive") - - # Do not include files that are of the general legacy File type - file_labels.remove("file") - - # Specify which analysis nodes get which types of - # `analysis.metadata` {'metadata type': set({'labels'})} - analysis_metadata = { - "read_groups": { - "alignment_workflow", - "alignment_cocleaning_workflow", - }, - } - - # Pre-calculate the paths to read_group from each type of file - file_to_read_group_paths = {} - for path in readgroup_subtree: - file_to_read_group_paths.setdefault(path[-1], []).append(path[-2::-1]) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Omit entities from these projects - self.omitted_projects.add(("CCLE", "CCLE_V2")) - self.omitted_projects.add(("CCLE", "ALL-P1")) - self.omitted_projects.add(("CCLE", "ACC")) - self.omitted_projects.add(("CCLE", "DLBC")) - self.omitted_projects.add(("CCLE", "READ")) - self.omitted_projects.add(("CCLE", "GBM")) - self.omitted_projects.add(("CCLE", "THCA")) - self.omitted_projects.add(("CCLE", "BLCA")) - self.omitted_projects.add(("CCLE", "UCEC")) - self.omitted_projects.add(("CCLE", "PCPG")) - self.omitted_projects.add(("CCLE", "LCML")) - self.omitted_projects.add(("CCLE", "CESC")) - self.omitted_projects.add(("CCLE", "UCS")) - self.omitted_projects.add(("CCLE", "THYM")) - self.omitted_projects.add(("CCLE", "LIHC")) - self.omitted_projects.add(("CCLE", "CHOL")) - self.omitted_projects.add(("CCLE", "HNSC")) - self.omitted_projects.add(("CCLE", "STAD")) - self.omitted_projects.add(("CCLE", "SKCM")) - self.omitted_projects.add(("CCLE", "COAD")) - self.omitted_projects.add(("CCLE", "UVM")) - self.omitted_projects.add(("CCLE", "PAAD")) - self.omitted_projects.add(("CCLE", "TGCT")) - self.omitted_projects.add(("CCLE", "LUSC")) - self.omitted_projects.add(("CCLE", "CNTL")) - self.omitted_projects.add(("CCLE", "MISC")) - self.omitted_projects.add(("CCLE", "MESO")) - self.omitted_projects.add(("CCLE", "FPPP")) - self.omitted_projects.add(("CCLE", "OV")) - self.omitted_projects.add(("CCLE", "ESCA")) - self.omitted_projects.add(("CCLE", "LCLL")) - self.omitted_projects.add(("CCLE", "MM")) - self.omitted_projects.add(("CCLE", "SARC")) - self.omitted_projects.add(("CCLE", "KIRP")) - self.omitted_projects.add(("CCLE", "LGG")) - self.omitted_projects.add(("CCLE", "LAML")) - self.omitted_projects.add(("CCLE", "PRAD")) - self.omitted_projects.add(("CCLE", "LUAD")) - self.omitted_projects.add(("CCLE", "BRCA")) - self.omitted_projects.add(("CCLE", "KIRC")) - self.omitted_projects.add(("CCLE", "KICH")) - - def denormalize_all(self): - cases, files, annotations, projects = super().denormalize_all() - - # Copy `primary_site` and `disease_type` from projects to cases.project: - projects_map = { - p["project_id"]: { - "primary_site": p["primary_site"], - "disease_type": p["disease_type"], - } - for p in projects - } - - for case in cases: - project_id = case["project"]["project_id"] - case["project"]["primary_site"] = projects_map[project_id]["primary_site"] - case["project"]["disease_type"] = projects_map[project_id]["disease_type"] - - return cases, files, annotations, projects - - def get_case_files(self, node): + +class ActiveGraphIndexBuilder(builder.GraphIndexBuilder): + """The builder for the current graph indices. + + Args: + psqlgraph_driver: The driver for interacting with the graph. + indexd_client: The client for accessing indexd documents. + index_prefix: The prefix of the index name being created (used for logging + only.) + build_projects: The projects which should be included in the build. + build_awg: A flag indicating if this is an AWG build. + selective_caching: A flag indicating if selective caching functionality + should be used. + versioned_files: Versioned files that haven't been released yet. + allowed_gencode_versions: The gencode version allowed when selecting file + objects to index. + """ + + def __init__( + self, + psqlgraph_driver: psqlgraph.PsqlGraphDriver, + indexd_client: client.IndexClient, + index_prefix: Optional[str], + build_projects: Iterable[str], + build_awg: bool, + selective_caching: bool, + versioned_files: Mapping[str, Mapping[str, Any]], + allowed_gencode_versions: Set[str], + ) -> None: + super().__init__( + psqlgraph_driver, + indexd_client, + index_prefix, + build_projects, + build_awg, + selective_caching, + versioned_files, + allowed_gencode_versions, + case_to_file_paths=_get_case_to_file_paths(), + file_labels=_get_file_labels(), + unindexed_by_property={ + "annotation": ( + {"status": "Rescinded"}, + {"classification": "Blocking Release"}, + ), + }, + ) + + self._file_to_aliquot_paths = _get_paths_from_files( + self.case_to_file_paths, destination=models.Aliquot.label + ) + self._file_to_read_group_paths = _get_paths_from_files( + self.case_to_file_paths, destination=models.ReadGroup.label + ) + + def get_case_files(self, node: models.Node) -> Collection[models.Node]: def file_filter(file) -> bool: metadata = self.file_metadata.get(file.node_id, {}) @@ -392,42 +395,46 @@ def file_filter(file) -> bool: unfiltered_files = super().get_case_files(node) return set(filter(file_filter, unfiltered_files)) - def denormalize_file(self, node, ptree): + def denormalize_file(self, node: models.Node, ptree: builder.PTree) -> dict: doc = super().denormalize_file(node, ptree) self.add_file_analysis(node, doc) self.add_file_downstream_analyses(node, doc) return doc - def get_file_index_files(self, node): + def get_file_index_files(self, node: models.Node) -> Iterator[models.Node]: """Given a file, return any neighboring index files.""" - return [ + return ( n - for n in list(self.get_child_with_category(node, "index_file")) + for n in self.get_child_with_category(node, "index_file") if self.is_index_file(n) - ] + ) - def get_parent_with_category(self, node, category): + def get_parent_with_category( + self, node: models.Node, category: str + ) -> Iterator[models.Node]: """Return iterable of neighbors from outbound edges with category.""" - labels = [ + labels = ( l["dst_type"].label for l in node._pg_links.values() if l["dst_type"]._dictionary["category"] == category - ] + ) return self.neighbors_labeled(node, labels) - def get_child_with_category(self, node, category): + def get_child_with_category( + self, node: models.Node, category: str + ) -> Iterator[models.Node]: """Return iterable of neighbors from inbound edges with category.""" - labels = [ + labels = ( l["src_type"].label for l in node._pg_backrefs.values() if l["src_type"]._dictionary["category"] == category - ] + ) return self.neighbors_labeled(node, labels) - def add_file_analysis(self, node, doc): + def add_file_analysis(self, node: models.Node, doc: dict) -> None: """Add the 'analysis' that produced the current file.""" analyses = list(self.get_parent_with_category(node, "analysis")) @@ -450,16 +457,16 @@ def add_file_analysis(self, node, doc): tags=[f"file_id:{node.node_id}"], ) - def add_file_downstream_analyses(self, node, doc): + def add_file_downstream_analyses(self, node: models.Node, doc: dict) -> None: """Add the 'analysis' that produced the current file.""" - analyses = list(self.get_child_with_category(node, "analysis")) + analyses = self.get_child_with_category(node, "analysis") for analysis in analyses: analysis_doc = self._get_base_doc(analysis) self.add_analysis_output_files(analysis, analysis_doc) doc.setdefault("downstream_analyses", []).append(analysis_doc) - def add_analysis_input_files(self, node, doc): + def add_analysis_input_files(self, node: models.Node, doc: dict) -> None: """For a given analysis node, add the input_files to the doc.""" input_files = [ f @@ -471,7 +478,7 @@ def add_analysis_input_files(self, node, doc): if input_file_docs: doc.setdefault("input_files", []).extend(input_file_docs) - def add_analysis_output_files(self, node, doc): + def add_analysis_output_files(self, node: models.Node, doc: dict) -> None: """For a given analysis node, add the output_files to the doc.""" output_files = [ f @@ -483,17 +490,23 @@ def add_analysis_output_files(self, node, doc): if output_file_docs: doc.setdefault("output_files", []).extend(output_file_docs) - def add_analysis_metadata(self, analysis, read_groups, doc): + def add_analysis_metadata( + self, analysis: models.Node, read_groups: Iterable[models.Node], doc: dict + ) -> None: """For a given analysis node, add the metadata to the doc.""" - metadata_doc = {} + metadata_doc: dict = {} - if analysis.label in self.analysis_metadata["read_groups"]: + # Specify which analysis nodes get which types of + # `analysis.metadata` {'metadata type': set({'labels'})} + if analysis.label in ("alignment_workflow", "alignment_cocleaning_workflow"): self.add_analysis_metadata_read_groups(read_groups, metadata_doc) if metadata_doc: doc["metadata"] = metadata_doc - def add_analysis_metadata_read_groups(self, read_groups, doc): + def add_analysis_metadata_read_groups( + self, read_groups: Iterable[models.Node], doc: dict + ) -> None: """For a given analysis node, add read_groups to the metadata subdoc.""" read_group_docs = [] @@ -509,7 +522,7 @@ def add_analysis_metadata_read_groups(self, read_groups, doc): if read_group_docs: doc["read_groups"] = read_group_docs - def get_read_group_qc_docs(self, read_group): + def get_read_group_qc_docs(self, read_group: models.Node) -> list[dict]: """Return a list of documents for Read Group QCs.""" read_group_qc_docs = [] rg_qcs = self.neighbors_labeled(read_group, "read_group_qc") @@ -518,28 +531,16 @@ def get_read_group_qc_docs(self, read_group): return read_group_qc_docs - def get_file_read_groups(self, node): + def get_file_read_groups(self, node: models.Node) -> Set[models.Node]: """Given a data_file node, traverse up the tree to read_groups. :returns: set of read_groups """ - paths = self.file_to_read_group_paths.get(node.label, []) - return set(self.walk_paths(node, paths)) + paths = self._file_to_read_group_paths.get(node.label, ()) + return self.walk_paths(node, paths) - def get_analysis_read_groups(self, node): - """Given a analysis node, traverse up the tree to read_groups. - - :returns: set of read_groups - - """ - return { - path - for file_ in self.get_parent_with_category(node, "data_file") - for path in self.get_file_read_groups(file_) - } - - def get_simple_file_doc(self, node): + def get_simple_file_doc(self, node: models.Node) -> dict: """Create a simple file doc for {input,output}_files.""" doc = self._get_base_doc(node) @@ -553,55 +554,25 @@ def get_simple_file_doc(self, node): return doc - def _get_custom_associated_entity_paths( - self, node: psqlgraph.Node - ) -> Iterable[Sequence[str]]: - # Special case paths to be traversed to possible associated entities - if node.label == "copy_number_segment": - return ( - reverse_and_skip_first_entry(path) - for path in list_product( - [["aliquot"]], self.aliquot_to_copy_number_segment_paths - ) - ) - elif node.label == "copy_number_estimate": - return ( - reverse_and_skip_first_entry(path) - for path in list_product( - [["aliquot"]], self.aliquot_to_copy_number_estimate_paths - ) - ) - elif node.label == "methylation_beta_value": - return ( - reverse_and_skip_first_entry(path) - for path in list_product( - [["aliquot"]], self.aliquot_to_methylation_value_paths - ) - ) - else: - return () - def _get_associated_entities_via_read_group( - self, node: psqlgraph.Node - ) -> Iterable[psqlgraph.Node]: + self, node: models.Node + ) -> Iterable[models.Node]: return itertools.chain.from_iterable( - self.neighbors_labeled(rg, self.possible_associated_entities) + self.neighbors_labeled(rg, builder.POSSIBLE_ASSOCIATED_ENTITIES) for rg in self.get_file_read_groups(node) ) def _get_associated_entities_via_data_files( - self, node: psqlgraph.Node - ) -> Iterable[psqlgraph.Node]: + self, node: models.Node + ) -> Iterable[models.Node]: return itertools.chain.from_iterable( - self.neighbors_labeled(parent, self.possible_associated_entities) + self.neighbors_labeled(parent, builder.POSSIBLE_ASSOCIATED_ENTITIES) for parent in self.get_parent_with_category(node, "data_file") ) - def get_file_associated_entities( - self, node: psqlgraph.Node - ) -> Iterable[psqlgraph.Node]: + def get_file_associated_entities(self, node: models.Node) -> Iterable[models.Node]: """Return all entities that are 'associated' with a file.""" - custom_paths = self._get_custom_associated_entity_paths(node) + custom_paths = self._file_to_aliquot_paths.get(node.label, ()) entities = super().get_file_associated_entities(node) entities = itertools.chain( entities, self._get_associated_entities_via_read_group(node) diff --git a/esbuild/graph/active/mimic.py b/esbuild/graph/active/mimic.py deleted file mode 100644 index 7c5d47b5..00000000 --- a/esbuild/graph/active/mimic.py +++ /dev/null @@ -1,13 +0,0 @@ -"""esbuild.graph.active.mimic. - -Mimics the isolation of functionality such as filtering nodes from the -index. - -""" - -from esbuild.graph.active.builder import ActiveGraphIndexBuilder -from esbuild.graph.common.mimic import CommonMimic - - -class ActiveMimic(CommonMimic, ActiveGraphIndexBuilder): - pass diff --git a/esbuild/graph/common/builder.py b/esbuild/graph/common/builder.py index bce07d36..200a05c5 100644 --- a/esbuild/graph/common/builder.py +++ b/esbuild/graph/common/builder.py @@ -5,54 +5,174 @@ """ +import collections +import copy +import functools import hashlib import itertools import logging -import random import re import traceback +import types import uuid -from collections import defaultdict -from copy import deepcopy -from functools import lru_cache -from typing import ( - Any, - Dict, - Generator, +from collections.abc import ( + Collection, + Container, Iterable, Iterator, - List, Mapping, - Optional, + Sequence, Set, - Tuple, - Union, ) -from uuid import UUID, uuid5 +from typing import Any, Optional, TypeVar, Union, cast +import more_itertools import networkx as nx +import progressbar import psqlgraph -from datadog import statsd -from gdcdatamodel2 import models as md +from datadog.dogstatsd.base import statsd +from gdcdatamodel2 import models from indexclient import client -from progressbar import ETA, Bar, Percentage, ProgressBar -from psqlgraph import Edge, Node -from sqlalchemy.orm import joinedload +from sqlalchemy import orm from esbuild.graph.common import mappings, validators -PTree = Dict[Node, "PTree"] -Document = Dict[str, Union[str, int]] +PTree = dict[models.Node, "PTree"] +TDocument = TypeVar("TDocument", dict, list[dict]) log = logging.getLogger(__name__) AVAILABLE_GENCODE_VERSIONS = frozenset(["neutral", "v22", "v36"]) -FILE_MISSING_GENCODE = {"error": "no gencode_version for generated data files"} +"""All possible gencode versions.""" +FILE_MISSING_GENCODE: dict[str, str] = { + "error": "no gencode_version for generated data files" +} +"""The error message associated with file which have no associated gencode version.""" ENTRY_FOR_WRONG_GENCODE = {"ignore": "wrong gencode_version for generated data files"} +"""The error message associated with file which have no document associated with an +appropriate gencode version. +""" FIELD_ALLOWLIST = frozenset({"wgs_coverage", "specimen_type"}) +"""Special fields which must be whitelisted to be included. +NOTE: New models should make this redundant. +""" +INDEX_FILE_EXTENSIONS: Set[str] = frozenset( + ( + ".bai", + ".tbi", + ) +) +"""File extensions associated with index files.""" +POSSIBLE_ASSOCIATED_ENTITIES: Collection[str] = ( + "portion", + "aliquot", + "case", + "slide", +) +"""All possible node types which can be considered associated entities.""" +DATA_FILE_INDEXD_FIELDS = ("acl", "file_size", "file_name", "file_state", "md5sum") +"""The fields associated with indexd records which should be included in the file +document. +""" +DIFFERENTIATED_EDGES: Set[tuple[str, str, str]] = frozenset( + ( + ("file", "member_of", "archive"), + ("archive", "member_of", "file"), + ("file", "describes", "case"), + ("case", "describes", "file"), + ("file", "related_to", "file"), + ) +) +"""Edges which will need extra data associated with them cached.""" +INDEXD_URL_TYPE = "cleversafe" +"""The url type associated with files to be indexed in the build.""" +FLATTENED_OBJECTS = types.MappingProxyType( + { + "tag": "name", + "platform": "name", + "data_format": "name", + "data_subtype": "name", + "experimental_strategy": "name", + "data_level": "name", + } +) +"""Nested object's who's associated property will be included in the indexed document of +the parent. E.g. the tag node's name property will be pulled up into the parent node's +document representation in the final indexed document. +""" +OMITTED_PROJECTS: Set[tuple[str, str]] = frozenset( + ( + ("TCGA", "CNTL"), + ("TCGA", "MISC"), + ("TCGA", "TEST"), + ("TCGA", "DEV1"), + ("TCGA", "DEV2"), + ("TCGA", "DEV3"), + ("TCGA", "FPPP"), + ("GDC", "INTERNAL"), + ("UAT08", "BROAD-BCR"), + ("TARGET", "AML-IF"), + ("CCLE", "CCLE_V2"), + ("CCLE", "ALL-P1"), + ("CCLE", "ACC"), + ("CCLE", "DLBC"), + ("CCLE", "READ"), + ("CCLE", "GBM"), + ("CCLE", "THCA"), + ("CCLE", "BLCA"), + ("CCLE", "UCEC"), + ("CCLE", "PCPG"), + ("CCLE", "LCML"), + ("CCLE", "CESC"), + ("CCLE", "UCS"), + ("CCLE", "THYM"), + ("CCLE", "LIHC"), + ("CCLE", "CHOL"), + ("CCLE", "HNSC"), + ("CCLE", "STAD"), + ("CCLE", "SKCM"), + ("CCLE", "COAD"), + ("CCLE", "UVM"), + ("CCLE", "PAAD"), + ("CCLE", "TGCT"), + ("CCLE", "LUSC"), + ("CCLE", "CNTL"), + ("CCLE", "MISC"), + ("CCLE", "MESO"), + ("CCLE", "FPPP"), + ("CCLE", "OV"), + ("CCLE", "ESCA"), + ("CCLE", "LCLL"), + ("CCLE", "MM"), + ("CCLE", "SARC"), + ("CCLE", "KIRP"), + ("CCLE", "LGG"), + ("CCLE", "LAML"), + ("CCLE", "PRAD"), + ("CCLE", "LUAD"), + ("CCLE", "BRCA"), + ("CCLE", "KIRC"), + ("CCLE", "KICH"), + ) +) +"""Projects which should always be omitted from the build.""" +SUPPLEMENT_PATTERNS = tuple( + map( + lambda s: re.compile(s), + ( + r"nationwidechildrens.org_biospecimen.([a-zA-Z0-9-]+).xml", + r"nationwidechildrens.org_control.([a-zA-Z0-9-]+).xml", + r"genome.wustl.edu_biospecimen.([a-zA-Z0-9-]+).xml", + r"genome.wustl.edu_control.([a-zA-Z0-9-]+).xml", + r"nationwidechildrens.org_clinical.([a-zA-Z0-9-]+).xml", + r"genome.wustl.edu_clinical.([a-zA-Z0-9-]+).xml", + ), + ) +) +"""The patterns which match older supplement file names.""" -@lru_cache(maxsize=32) +@functools.lru_cache(maxsize=32) def dfs_to_parent(node, target="case"): if node.label == target: return node @@ -72,9 +192,26 @@ def _get_gencode_version(doc: Optional[client.Document]) -> Optional[str]: return gencode_version +def _load_hidden_properties() -> Iterator[tuple[str, Set[str]]]: + default_properties = frozenset(("batch_id", "file_state")) + labels = (t.label for t in models.Node.get_subclasses() if t.label != "annotation") + + for label in labels: + yield label, default_properties + + yield "annotation", frozenset(("creator", *default_properties)) + + +HIDDEN_PROPERTIES = types.MappingProxyType(dict(_load_hidden_properties())) +"""A mapping of node types to the properties which should be omitted from the final +indexed document.""" + + class GraphIndexBuilder: """Build graph index from gdc psqlgraph. + The Process Explained: + This class handles all the JSON production for the GDC portal. Currently, the entire postgresql database is cached to memory. To save space, edge labels are only maintained if we need @@ -113,227 +250,111 @@ class GraphIndexBuilder: remembering which nodes are walked through a lot and remembering which neighbors they have with a given label. - NOTE: An attempt was made to do this whole thing in parallel, - however the memory footprint grew to large. The best method for - doing this is to use the main process as a workload distributer, - and have child processes denormalizing cases. This way, - the main thread can upsert files on an outbound queue from child - processes. - - - Josh (jsmiller@uchicago.edu) - - TODOS: - - figure out a way to parallelize without excess copies - - =============== - Transformations - =============== - - In order to update features in the index without propagating - renames, re-nestings, flattenings etc through the datamodel, the - denormalization process will reformat the data in (but not limited - to) the following ways: - - * flattening: - Some nodes are flattened into properties. These are typically - nodes like ``tag`` that only have a ``name`` property. See - ``self.flatten`` for a complete list. - - * hidden_properties: - Some nodes should have properties hidden, e.g. - ``annotation.creator``. In addition, ``project_id`` will be - hidden on all nodes. - - * s/data_type/data_category/g: - data_type is renamed data_category, viz. - https://jira.opensciencedatacloud.org/browse/PGDC-1472 - - * s/data_subtype/data_type/g: - data_subtype is renamed data_type, viz. - https://jira.opensciencedatacloud.org/browse/PGDC-1472 - - * s/related_files/metadata_files/g - related_files is renamed metadata_files, viz. - https://jira.opensciencedatacloud.org/browse/PGDC-1838 - + Args: + psqlgraph_driver: The driver for interacting with the graph. + indexd_client: The client for accessing indexd documents. + index_prefix: The prefix of the index name being created (used for logging + only.) + build_projects: The projects which should be included in the build. + build_awg: A flag indicating if this is an AWG build. + selective_caching: A flag indicating if selective caching functionality + should be used. + versioned_files: Versioned files that haven't been released yet. + allowed_gencode_versions: The gencode version allowed when selecting file + objects to index. + case_to_file_paths: All possible paths from a case node to all relevant file + nodes. + file_labels: All of the node labels which are associated with files to be + included in the build. + unindexed_by_property: A mapping of filtered nodes which if their properties + are a superset of any of the dictionaries listed here are removed from + being indexed. """ - data_file_categories = ["data_file", "metadata_file"] - data_file_indexd_fields = ["acl", "file_size", "file_name", "file_state", "md5sum"] - - # This defines the possible ways to get from case to indexed - # files. Should be an iterable of iterables, i.e. - # [['file'], ['sample', 'aliquot', 'file']] - case_to_file_paths = [] - - # in addition, project_id will be hidden on all nodes - # {node.label: {set of property keys}} - hidden_properties = { - "annotation": { - "creator", - } - } - # Set of properties to add to hidden_properties for all nodes - hidden_properties_for_all = { - "batch_id", - "file_state", - } - - for node_type in md.Node.get_subclasses(): - hidden_properties.setdefault(node_type.label, set()) - hidden_properties[node_type.label].update(hidden_properties_for_all) - - # Filter nodes out if their properties are a superset of any of - # the dictionaries listed here by label - unindexed_by_property = { - # "label": [{"key1": "value1", "key2": "value2"}] - } - - INDEXD_URL_TYPE = "cleversafe" - - required_attrs = [ - "case_to_file_paths", - "file_labels", - ] - - supplement_regexes = [ - re.compile(regex) - for regex in [ - "nationwidechildrens.org_biospecimen.([a-zA-Z0-9-]+).xml", - "nationwidechildrens.org_control.([a-zA-Z0-9-]+).xml", - "genome.wustl.edu_biospecimen.([a-zA-Z0-9-]+).xml", - "genome.wustl.edu_control.([a-zA-Z0-9-]+).xml", - "nationwidechildrens.org_clinical.([a-zA-Z0-9-]+).xml", - "genome.wustl.edu_clinical.([a-zA-Z0-9-]+).xml", - ] - ] - def __init__( self, psqlgraph_driver: psqlgraph.PsqlGraphDriver, indexd_client: client.IndexClient, - index_prefix: Optional[str] = "", - **kwargs: Any, + index_prefix: Optional[str], + build_projects: Iterable[str], + build_awg: bool, + selective_caching: bool, + versioned_files: Mapping[str, Mapping[str, Any]], + allowed_gencode_versions: Set[str], + case_to_file_paths: Iterable[Sequence[str]] = (), + file_labels: Collection[str] = (), + unindexed_by_property: Mapping[ + str, Iterable[Mapping[str, Any]] + ] = types.MappingProxyType({}), ) -> None: - """Walk the graph to produce elasticsearch json documents.""" + self.g = psqlgraph_driver self.indexd = indexd_client self.index_prefix = index_prefix - self.file_metadata = {} # Cache of file metadata from indexd - self.skipped_nodes = {} # Cache of skipped nodes and reason for skipping - # Versioned files that haven't been released yet - self.versioned_files = kwargs.pop("versioned_files", {}) + self.versioned_files = versioned_files - self.allowed_gencode_versions = kwargs.pop( - "allowed_gencode_versions", AVAILABLE_GENCODE_VERSIONS - ) - if not self.allowed_gencode_versions.issubset(AVAILABLE_GENCODE_VERSIONS): + if not (allowed_gencode_versions <= AVAILABLE_GENCODE_VERSIONS): raise NotImplementedError( - f"{self.allowed_gencode_versions} is not a valid gencode_version requirement" + f"{allowed_gencode_versions} is not a valid gencode_version requirement" f"The available gencode_versions are {AVAILABLE_GENCODE_VERSIONS}" ) - # Set all optional arguments as attributes: - # NOTE: Selective caching only works when all the non-project nodes - # that are expected to be picked up are populated with project_id - # As of Jan 2018, this is true only for newest active projects - self.build_awg = kwargs.get("build_awg") - self.selective_caching = kwargs.get("selective_caching") - self.build_projects = [ - tuple(p.split("-", 1)) for p in kwargs.get("build_projects", []) - ] - - # Verify required attributes are set - for required_attr in self.required_attrs: - if getattr(self, required_attr) is None: - raise NotImplementedError( - f"{self.__class__.__name__} must set {required_attr}" - ) - - if self.build_projects: - log.info("Running partial build") - log.info(f"Projects: {self.build_projects}") - else: - log.info("Running full build") + self.allowed_gencode_versions = allowed_gencode_versions + self.build_awg = build_awg + self.selective_caching = selective_caching + # The original project ids are split into program/project name as tuples + # E.g. "TCGA-ACC" -> ("TCGA", "ACC") + self.build_projects: Iterable[tuple[str, str]] = tuple( + cast(tuple[str, str], tuple(p.split("-", 1))) for p in build_projects + ) - self.g = psqlgraph_driver + self.case_to_file_paths = case_to_file_paths + self.file_to_case_paths = tuple( + tuple(more_itertools.value_chain(reversed(l[:-1]), "case")) + for l in self.case_to_file_paths + ) + """A calculated field based on the case to file paths reversed.""" + self.file_labels = file_labels + self.unindexed_by_property = unindexed_by_property + + # VARIOUS CACHES + self.annotation_entities: dict[models.Node, dict[str, dict]] = {} + """A cache mapping a node to to a mapping of annotation ids which are mapped + themselves to their denormalized doc. + """ + self.annotations: Collection[models.Node] = () + """A cache of the annotations for the build.""" + self.cases: Sequence[models.Node] = () + """A cache of the cases for the build.""" + self.data_categories: dict[str, set[models.Node]] = collections.defaultdict(set) + """A cache of data categories mapped to their associated files nodes.""" + self.entity_cases: dict[models.Node, models.Node] = {} + """A cache possible associated entities mapped to their associated case.""" + self.experimental_strategies: dict[ + str, set[models.Node] + ] = collections.defaultdict(set) + """A cache of experimental strategies mapped to their associated files.""" + self.file_metadata: dict[str, dict] = {} + """A cache of file metadata from indexd.""" self.G = nx.Graph() - - self.leaf_nodes = ["center", "tissue_source_site"] - self.experimental_strategies = defaultdict(set) - self.data_categories = defaultdict(set) - self.popular_nodes = {} - self.cases = None - self.projects: Optional[Iterable[Node]] = None - self.relevant_nodes: Dict[Node, Set[Node]] = None - self.annotations = None - self.annotation_entities: Dict[Node, Dict[uuid.UUID, dict]] = None - self.entity_cases = None - - # Different from ``self.data_categories`` in that it's a - # replacement for a hardcoded dict of data_type, data_subtype - # relationships. This is populated by - # ``self._cache_file_properties`` - self.existing_data_types = set() - - # Suppress entities with redaction annotation if - # entity.annotation.category not in this list - self.redacted_but_not_suppressed = ["Subject withdrew consent"] - - # Omit entities from these projects - self.omitted_projects = { - ("TCGA", "CNTL"), - ("TCGA", "MISC"), - ("TCGA", "TEST"), - ("TCGA", "DEV1"), - ("TCGA", "DEV2"), - ("TCGA", "DEV3"), - ("TCGA", "FPPP"), - ("GDC", "INTERNAL"), - ("UAT08", "BROAD-BCR"), - ("TARGET", "AML-IF"), - } - - # The body of these nested documents will be flattened into - # the parent document using the given key's value - self.flatten = { - "tag": "name", - "platform": "name", - "data_format": "name", - "data_subtype": "name", - "experimental_strategy": "name", - "data_level": "name", - } - - # The edges below will maintain labels in the in memory graph, - # all others will be discarded - self.differentiated_edges = [ - ("file", "member_of", "archive"), - ("archive", "member_of", "file"), - ("file", "describes", "case"), - ("case", "describes", "file"), - ("file", "related_to", "file"), - ] - - self.file_to_case_paths = [ - list(reversed(l))[1:] + ["case"] for l in self.case_to_file_paths - ] - - self.possible_associated_entities = [ - "portion", - "aliquot", - "case", - "slide", - ] - - self.index_file_extensions = { - ".bai", - ".tbi", - } - - def warning( - self, title: str, text: str, tags: Optional[List[str]] = None, *args, **kwargs - ) -> None: + """A cache of the data within the actual gdc graph.""" + self.popular_nodes: dict[ + models.Node, dict[tuple[str, ...], Set[models.Node]] + ] = collections.defaultdict(dict) + """A cache of common neighbors to a particular node. The node is mapped to a + mapping of particular sets of labels which in turn are mapped to the nodes of + those types associated with the original node. + """ + self.projects: Collection[models.Node] = () + """A cache of project nodes.""" + self.relevant_nodes: dict[models.Node, Set[models.Node]] = {} + """A cache which maps file nodes to their associated cased nodes.""" + self.skipped_nodes: dict[ + str, list[Union[models.Node, tuple[models.Node, str]]] + ] = collections.defaultdict(list) + """A cache of nodes skipped in the build and the reason for skipping them.""" + + def warning(self, title: str, text: str, tags: Optional[list[str]] = None) -> None: tags = tags or [] tags.append(f"index_group:{self.index_prefix}") @@ -346,9 +367,7 @@ def warning( tags=tags, ) - def error( - self, title: str, text: str, tags: Optional[List[str]] = None, *args, **kwargs - ): + def error(self, title: str, text: str, tags: Optional[list[str]] = None): tags = tags or [] tags.append(f"index_group:{self.index_prefix}") @@ -361,7 +380,7 @@ def error( tags=tags, ) - def pbar(self, title, maxval): + def pbar(self, title: str, maxval: int) -> progressbar.ProgressBar: """Create and initialize a custom progressbar. :param str title: The text of the progress bar @@ -369,14 +388,14 @@ def pbar(self, title, maxval): """ maxval = maxval or 1 # prevent maxal of 0 - pbar = ProgressBar( + pbar = progressbar.ProgressBar( widgets=[ title, - Percentage(), + progressbar.Percentage(), " ", - Bar(marker="#", left="[", right="]"), + progressbar.Bar(marker="#", left="[", right="]"), " ", - ETA(), + progressbar.ETA(), " ", ], max_value=maxval, @@ -388,22 +407,9 @@ def pbar(self, title, maxval): # Tree functions ################################################################### - def parse_tree(self, tree, result): - """Generate a simpler tree with just node labels. - - Recursively walk a mapping tree and generate a simpler tree with - just node labels and not correspondences. - - """ - for key in tree: - if key != "corr": - result[key] = {} - self.parse_tree(tree[key], result[key]) - return result - - def create_tree(self, node, mapping, tree): + def create_tree(self, node: models.Node, mapping: Mapping, tree: PTree) -> PTree: """Recursively walk a mapping to create a walkable tree.""" - if node.label in self.leaf_nodes: + if node.label in ("center", "tissue_source_site"): return {} submap = mapping[node.label] corr, plural = submap["corr"] @@ -416,13 +422,13 @@ def create_tree(self, node, mapping, tree): def walk_tree( self, - node: Node, - tree: Dict[Node, dict], - mapping: Dict[str, dict], - doc: Document, - level=0, - ids=None, - ): + node: models.Node, + tree: dict[models.Node, dict], + mapping: dict[str, dict], + doc: TDocument, + level: int = 0, + ids: Optional[Mapping[str, set[str]]] = None, + ) -> TDocument: """Add the properties of node neighbors to doc. Recursively walk from a node to all possible neighbors that are @@ -461,8 +467,10 @@ def walk_tree( ids[f"submitter_{child.label}_ids"].add(sub_id) if corr == mappings.ONE_TO_MANY: + assert isinstance(doc, list) doc.append(subdoc) else: + assert isinstance(doc, dict) doc.update(subdoc) return doc @@ -473,7 +481,7 @@ def copy_tree(self, original: PTree, new: PTree) -> PTree: self.copy_tree(original[node], new[node]) return new - def _get_base_doc(self, node: Node, include_id: bool = True) -> Document: + def _get_base_doc(self, node: models.Node, include_id: bool = True) -> dict: """Create a dictionary with all the properties of a node. This is the basic document generator. Take all the properties of a @@ -501,7 +509,7 @@ def _get_base_doc(self, node: Node, include_id: bool = True) -> Document: # Also include manually included fields form allowlist. if (key in node.__pg_properties__ or key in FIELD_ALLOWLIST) # Ignore certain keys by type - and key not in self.hidden_properties.get(node.label, []) + and key not in HIDDEN_PROPERTIES.get(node.label, ()) # Hide project_id for all nodes but project, viz. PGDC-1550 and (key != "project_id" or node.label == "project") } @@ -514,8 +522,8 @@ def _get_base_doc(self, node: Node, include_id: bool = True) -> Document: ################################################################### def walk_path( - self, node: Node, path: List[str], whole=False - ) -> Generator[Node, None, None]: + self, node: models.Node, path: Sequence[str], whole: bool = False + ) -> Iterator[models.Node]: """Get a node from end of a path or all the nodes along the path. Given a list of strings, treat it as a path, and yield the end of @@ -530,28 +538,29 @@ def walk_path( yield from self.walk_path(neighbor, path[1:], whole) - def walk_paths(self, node: Node, paths: List[List[str]], whole=False) -> Set[Node]: + def walk_paths( + self, node: models.Node, paths: Iterable[Sequence[str]], whole: bool = False + ) -> Set[models.Node]: """Get nodes from walking paths. Given a list of paths, yield the result of walking each path. If `whole` is true, return every node along each traversal. """ - return { - n - for n in itertools.chain( - *[self.walk_path(node, path, whole=whole) for path in paths] + return frozenset( + itertools.chain.from_iterable( + self.walk_path(node, path, whole=whole) for path in paths ) - } + ) - def remove_bam_index_files(self, files): + def remove_bam_index_files(self, files: Iterable[models.Node]) -> Set[models.Node]: return {f for f in files if not self.is_index_file(f)} ################################################################### # Cases ################################################################## - def remove_hidden_nodes(self, nodes): + def remove_hidden_nodes(self, nodes: Iterable[models.Node]) -> Set[models.Node]: """Get subset of nodes whose self.is_node_hidden(node) is False. Returns a subset of :param:`nodes` for which @@ -563,9 +572,9 @@ def remove_hidden_nodes(self, nodes): """ return {node for node in nodes if not validators.is_node_hidden(node)} - def get_case_files(self, node): + def get_case_files(self, node: models.Node) -> Collection[models.Node]: """Return a list of file nodes by walking out from case.""" - files = self.walk_paths(node, self.case_to_file_paths) + files: Iterable[models.Node] = self.walk_paths(node, self.case_to_file_paths) # Set file metadata fields from indexd as node properties files = (self.add_file_metadata_from_indexd(f) for f in files) @@ -574,60 +583,30 @@ def get_case_files(self, node): return files - def get_case_tree(self, node): + def get_case_tree(self, node: models.Node) -> tuple[dict, PTree]: """Use tree to create nested json. :returns: doc, ptree, visited_ids """ ptree = self.get_case_ptree(node) - visited_ids = defaultdict(set) - doc = self.walk_tree(node, ptree, mappings.CASE_TREE, [], ids=visited_ids)[0] - - # Convert to list for later serialization - visited_ids = {key: list(ids) for key, ids in visited_ids.items()} + visited_ids: Mapping[str, set[str]] = collections.defaultdict(set) + doc = self.walk_tree( + node, ptree, mappings.CASE_TREE, cast(list[dict], []), ids=visited_ids + )[0] # Inject a dictionary of ids for each visited entity (in # TOP_LEVEL_IDS) - doc.update(visited_ids) - - return doc, ptree, visited_ids - - def get_relevant_ids(self, node, visited_ids): - """Create flattened copy of visited_ids. - - Create a flattened copy of visited_ids to filter relevant - annotations by entity id + # Convert to list for later serialization + doc.update((key, list(ids)) for key, ids in visited_ids.items()) - """ - return [ - _entity_id - for _entity_type in visited_ids.values() - for _entity_id in _entity_type - ] + [node.node_id] + return doc, ptree - def get_case_ptree(self, node): + def get_case_ptree(self, node: models.Node) -> PTree: """Walk graph naturally for tree of node objects.""" return {node: self.create_tree(node, mappings.CASE_TREE, {})} - def get_relevant_annotations(self, file_docs, relevant_ids): - """Return a flat list of annotations who describe entities in relevant_ids.""" - return [ - annotation - for file_ in file_docs - for annotation in file_.get("annotations", []) - if annotation["entity_id"] in relevant_ids - ] - - def get_diagnosis_annotations(self, node): - """Return a flat list of annotations describing a case's diagnoses.""" - return [ - ann_doc - for diagnosis in self.neighbors_labeled(node, "diagnosis") - for ann_doc in self.annotation_entities.get(diagnosis, {}).values() - ] - - def denormalize_case(self, node: Node) -> Tuple[dict, List[dict], List]: + def denormalize_case(self, node: models.Node) -> tuple[dict, list[dict], list]: """Get the entire case document for a case node. Given a case node, return the entire case document, @@ -637,13 +616,13 @@ def denormalize_case(self, node: Node) -> Tuple[dict, List[dict], List]: """ # Walk from case to leaves (not files) and create a case doc, # a participant tree, and a list of visited ids - case, ptree, visited_ids = self.get_case_tree(node) + case, ptree = self.get_case_tree(node) # Get the file nodes related to the case files = self.get_case_files(node) # Create case summary - case["summary"] = self.get_case_summary(node, files) + case["summary"] = self.get_case_summary(files) # Take any out of place nodes and put then in correct place in tree self.reconstruct_biospecimen_paths(case) @@ -653,7 +632,7 @@ def denormalize_case(self, node: Node) -> Tuple[dict, List[dict], List]: project = self.patch_project(case["project"]) # Denormalize the cases files - returned_files = self.get_case_file_docs(node, ptree, files) + returned_files = self.get_case_file_docs(ptree, files) # Add files to cases # Do not add cases, annotations and associated entities to case.files @@ -663,7 +642,7 @@ def denormalize_case(self, node: Node) -> Tuple[dict, List[dict], List]: for k in f if k not in ["cases", "annotations", "associated_entities"] } - for f in deepcopy(returned_files) + for f in copy.deepcopy(returned_files) ] # Do not include input_files in case.files.analysis (TT-928) @@ -676,19 +655,12 @@ def denormalize_case(self, node: Node) -> Tuple[dict, List[dict], List]: return case, returned_files, [] def get_case_file_docs( - self, node: Node, ptree: PTree, files: Set[Node] - ) -> List[dict]: + self, ptree: PTree, files: Iterable[models.Node] + ) -> list[dict]: """Given a list of files, return a list of file docs.""" return [self.denormalize_file(file_, ptree) for file_ in files] - def patch_annotations(self, annotations, node, project): - """Add misc properties to annotations in-place.""" - for annotation in annotations: - annotation["project"] = project - annotation["case_id"] = node.node_id - annotation["case_submitter_id"] = node.submitter_id - - def get_exp_strats(self, files): + def get_exp_strats(self, files: Iterable[models.Node]) -> Iterator[dict]: """Get files experimental strategies. Get the set of experimental_strategies where intersection of the @@ -697,14 +669,14 @@ def get_exp_strats(self, files): """ for exp_strat, file_list in self.experimental_strategies.items(): - intersection = file_list & files + intersection = file_list.intersection(files) if intersection: yield { "experimental_strategy": exp_strat, "file_count": len(intersection), } - def get_data_categories(self, files): + def get_data_categories(self, files: Iterable[models.Node]) -> Iterator[dict]: """Get files data category. Get the set of data_categories where intersection of the @@ -713,7 +685,7 @@ def get_data_categories(self, files): """ for data_category, file_list in self.data_categories.items(): - intersection = file_list & files + intersection = file_list.intersection(files) if intersection: yield { # data_type is renamed data_category, viz. @@ -722,7 +694,7 @@ def get_data_categories(self, files): "file_count": len(intersection), } - def get_case_summary(self, node: Node, files): + def get_case_summary(self, files: Collection[models.Node]) -> dict: """Create a dictionary with summary information about case files. Generate a dictionary containing a summary of a cases files @@ -739,7 +711,7 @@ def get_case_summary(self, node: Node, files): } @staticmethod - def reconstruct_diagnoses_paths(case: Document) -> Document: + def reconstruct_diagnoses_paths(case: dict) -> dict: """Reconstruct path for molecular tests. There are two different paths from diagnoses to molecular tests: @@ -754,7 +726,7 @@ def reconstruct_diagnoses_paths(case: Document) -> Document: Returns: updated case dictionary """ - case_copy = deepcopy(case) + case_copy = copy.deepcopy(case) correct_molecular_tests = set() for follow_up in case_copy.get("follow_ups", []): for molecular_test in follow_up.get("molecular_tests", []): @@ -845,7 +817,7 @@ def reconstruct_biospecimen_paths(self, case: dict) -> None: ns="aliquots", seed=aliquot["aliquot_id"] ) portion_id = get_namespaced_uuid(ns="analytes", seed=analyte_id) - new_dict = { + new_dict: dict = { "analytes": [{"analyte_id": analyte_id, "aliquots": [aliquot]}] } # check if another entry already added the fake id @@ -872,7 +844,7 @@ def reconstruct_biospecimen_paths(self, case: dict) -> None: ] ) - def patch_project(self, project_doc): + def patch_project(self, project_doc: dict) -> dict: # Delete some keys from project document for key in mappings.HIDDEN_PROJECT_KEYS: project_doc.pop(key, None) @@ -889,7 +861,7 @@ def patch_project(self, project_doc): # File denormalization ################################################################### - def denormalize_file(self, node: Node, ptree: PTree) -> Dict[str, Union[int, str]]: + def denormalize_file(self, node: models.Node, ptree: PTree) -> dict: """Given a cases tree and a file node, create the file json document.""" # Add file metadata fields from indexd node = self.add_file_metadata_from_indexd(node) @@ -911,15 +883,15 @@ def denormalize_file(self, node: Node, ptree: PTree) -> Dict[str, Union[int, str self.add_archives(node, doc) doc["cases"] = [] relevant = self.add_cases(node, ptree, doc) - self.add_file_associated_entities(node, doc, case_id) - self.add_annotations(node, relevant, doc) + self.add_file_associated_entities(node, doc) + self.add_annotations(relevant, doc) self.add_acl(node, doc) self.add_file_data_format(node, doc) return doc def has_allowed_gencode_version( - self, node: Node, gencode_version: Optional[str] + self, node: models.Node, gencode_version: Optional[str] ) -> bool: # submittable nodes should always be included if node._dictionary.get("submittable", False): @@ -927,21 +899,19 @@ def has_allowed_gencode_version( return gencode_version in self.allowed_gencode_versions - def add_file_metadata_from_indexd(self, node: Node) -> Node: + def add_file_metadata_from_indexd(self, node: models.Node) -> models.Node: """Read file metadata from indexd and sets it to node.""" if node.node_id in self.versioned_files: for key, value in self.versioned_files[node.node_id].items(): setattr(node, key, value) return node - # Try to get cached metadata value - record = self.file_metadata.get(node.node_id) - - # If not found, get it from indexd - if not record: - record = self.indexd.get(node.node_id) + if node.node_id in self.file_metadata: + record = self.file_metadata[node.node_id] + else: + document = self.indexd.get(node.node_id) - if not record: + if not document: if node.sysan.get("to_delete"): self.file_metadata[node.node_id] = {"error": "to_delete file"} else: @@ -953,7 +923,7 @@ def add_file_metadata_from_indexd(self, node: Node) -> Node: self.file_metadata[node.node_id] = {"error": "no indexd record"} return node - gencode_version = _get_gencode_version(record) + gencode_version = _get_gencode_version(document) if not self.has_allowed_gencode_version(node, gencode_version): if gencode_version: self.file_metadata[node.node_id] = ENTRY_FOR_WRONG_GENCODE @@ -967,26 +937,22 @@ def add_file_metadata_from_indexd(self, node: Node) -> Node: self.file_metadata[node.node_id] = FILE_MISSING_GENCODE return node - record = record.to_json() # Cache indexd record - self.file_metadata[node.node_id] = record + record = self.file_metadata.setdefault(node.node_id, document.to_json()) # for to_delete nodes and nodes with wrong gencode_version if "error" in record or "ignore" in record: return node # Set node file metadata attributes according to indexd record - for key in self.data_file_indexd_fields: + for key in DATA_FILE_INDEXD_FIELDS: # Try to pick basic value value = record.get(key) if value is None: value = record["metadata"].get(key) if key == "file_state": for s3_url in record["urls_metadata"].keys(): - if ( - record["urls_metadata"][s3_url].get("type") - == self.INDEXD_URL_TYPE - ): + if record["urls_metadata"][s3_url].get("type") == INDEXD_URL_TYPE: value = record["urls_metadata"][s3_url].get("state") # Special values @@ -999,10 +965,10 @@ def add_file_metadata_from_indexd(self, node: Node) -> Node: return node - def add_node_type(self, node: Node, doc: Document) -> None: + def add_node_type(self, node: models.Node, doc: dict) -> None: doc["type"] = node.label - def get_data_format(self, node: Node) -> str: + def get_data_format(self, node: models.Node) -> Optional[str]: """Get data format of a file node. Return the ``data_format`` given a file node based on @@ -1037,7 +1003,9 @@ def get_data_format(self, node: Node) -> str: return format_ - def prune_case(self, relevant_nodes, ptree, keys): + def prune_case( + self, relevant_nodes: Set[models.Node], ptree: PTree, keys: Container[str] + ) -> None: """Remove node (whose label in keys but is not in relavent_nodes) from ptree. Start with whole case tree and remove any nodes that did not @@ -1045,15 +1013,11 @@ def prune_case(self, relevant_nodes, ptree, keys): .. note:: :param:`ptree` is edited **in place* - :param relevant_nodes: - The ancestors that should not be pruned from the tree - (most likely self.relevant_nodes[some_file]) - :param ptree: - The canonical ptree dict tree containing a the descendents - of a case - :param keys: - Only prune a given node ``node`` if ``node.label`` in keys - + Args: + relevant_nodes: The ancestors that should not be pruned from the tree (most + likely self.relevant_nodes[some_file]) + ptree: The canonical ptree dict tree containing a the descendents of a case. + keys: Only prune a given node ``node`` if ``node.label`` in keys. """ for node in list(ptree.keys()): if ptree[node]: @@ -1061,13 +1025,13 @@ def prune_case(self, relevant_nodes, ptree, keys): if node.label in keys and node not in relevant_nodes: ptree.pop(node) - def add_file_data_format(self, node, doc): + def add_file_data_format(self, node: models.Node, doc: dict) -> None: """Add (or overwrite) the data format if found.""" data_format = self.get_data_format(node) if data_format: doc["data_format"] = data_format - def add_file_neighbors(self, node: Node, doc: Document) -> None: + def add_file_neighbors(self, node: models.Node, doc: dict) -> None: """Add specified neighbors to file doc[label]. Given a file, walk to all of its neighbors specified by the schema @@ -1080,9 +1044,9 @@ def add_file_neighbors(self, node: Node, doc: Document) -> None: if n not in ["archive", "portion", "file"] ] for neighbor in set(self.neighbors_labeled(node, auto_neighbors)): - corr, label = mappings.FILE_TREE["file"][neighbor.label]["corr"] - if neighbor.label in self.flatten: - base = neighbor[self.flatten[neighbor.label]] + corr, label = mappings.FILE_TREE["file"][neighbor.label]["corr"] # type: ignore + if neighbor.label in FLATTENED_OBJECTS: + base = neighbor[FLATTENED_OBJECTS[neighbor.label]] else: base = self._get_base_doc(neighbor) @@ -1120,7 +1084,7 @@ def add_file_neighbors(self, node: Node, doc: Document) -> None: doc[label] = [] doc[label].append(base) - def is_index_file(self, node): + def is_index_file(self, node: models.Node) -> bool: """Given a node, return whether it is considered an 'index file'. :returns: bool @@ -1135,22 +1099,21 @@ def is_index_file(self, node): # Set file metadata fields node = self.add_file_metadata_from_indexd(node) - for extension in self.index_file_extensions: + for extension in INDEX_FILE_EXTENSIONS: if getattr(node, "file_name", "").endswith(extension): return True - else: - return False + return False - def get_file_index_files(self, node): + def get_file_index_files(self, node: models.Node) -> Iterator[models.Node]: """Given a file, return any neighboring index files.""" - return [ + return ( n for n in list(self.neighbors_labeled(node, "file")) if self.G[node][n].get("label") == "related_to" and self.is_index_file(n) - ] + ) - def add_index_files(self, node, doc): + def add_index_files(self, node: models.Node, doc: dict) -> None: """Add neighboring index files to file doc["index_files"]. Given a file, walk to any neighboring index files and add @@ -1160,8 +1123,6 @@ def add_index_files(self, node, doc): index_file_docs = [] index_files = self.get_file_index_files(node) - log.debug(f"Found index files for {node}: {index_files}") - for index_file in index_files: index_file_doc = self._get_base_doc(index_file) index_file_doc["data_format"] = self.get_data_format(index_file) @@ -1171,7 +1132,7 @@ def add_index_files(self, node, doc): if index_file_docs: doc["index_files"] = index_file_docs - def add_related_files(self, node: Node, doc: dict) -> None: + def add_related_files(self, node: models.Node, doc: dict) -> None: """Add neighboring files to file doc["metadata_files"]. Given a file, walk to any (non data-from) neighboring files and add @@ -1184,11 +1145,11 @@ def add_related_files(self, node: Node, doc: dict) -> None: """ rf_docs = [] - metadata_labels = [ + metadata_labels = ( "analysis_metadata", "run_metadata", "experiment_metadata", - ] + ) # Get related_files related_files = [ @@ -1256,7 +1217,7 @@ def add_related_files(self, node: Node, doc: dict) -> None: # viz. https://jira.opensciencedatacloud.org/browse/PGDC-1838 doc["metadata_files"] = rf_docs - def add_archives(self, node, doc): + def add_archives(self, node: models.Node, doc: dict) -> None: """Add attached archive to file doc["archive"]. For each archive attached to a given file node, multixplex on @@ -1290,7 +1251,7 @@ def add_archives(self, node, doc): doc["archive"] = archive_doc - def add_data_category(self, node, doc): + def add_data_category(self, node: models.Node, doc: dict) -> None: """Add the data_subtype to the file document with child data_category.""" data_categories = [ data_category @@ -1302,7 +1263,9 @@ def add_data_category(self, node, doc): # https://jira.opensciencedatacloud.org/browse/PGDC-1472 doc["data_category"] = data_categories[0] - def add_cases(self, node: Node, ptree: Dict[Node, dict], doc: dict) -> Set[Node]: + def add_cases( + self, node: models.Node, ptree: dict[models.Node, dict], doc: dict + ) -> Collection[models.Node]: """Add cases to file doc['cases']. Given a file and a case tree, re-insert the case as a @@ -1312,11 +1275,11 @@ def add_cases(self, node: Node, ptree: Dict[Node, dict], doc: dict) -> Set[Node] """ if not ptree: log.warning("No ptree (case tree) for %s", node) - return [] + return () if node not in self.relevant_nodes: log.warning("No relevant cases for %s", node) - return [] + return () relevant = self.relevant_nodes[node] prune_keys = ["sample", "portion", "analyte", "aliquot", "file"] @@ -1324,7 +1287,8 @@ def add_cases(self, node: Node, ptree: Dict[Node, dict], doc: dict) -> Set[Node] self.prune_case(relevant, ptree, prune_keys) doc["cases"] = [ - self.walk_tree(path, ptree, mappings.CASE_TREE, [])[0] for path in ptree + self.walk_tree(path, ptree, mappings.CASE_TREE, cast(list[dict], []))[0] + for path in ptree ] for case in doc["cases"]: @@ -1335,7 +1299,7 @@ def add_cases(self, node: Node, ptree: Dict[Node, dict], doc: dict) -> Set[Node] return relevant - def add_annotations(self, node, relevant, doc): + def add_annotations(self, relevant: Iterable[models.Node], doc: dict) -> None: """Loop relevant, get all annotation docs and add them to doc["annotations"]. Given a file node, aggregate all the annotations from a pruned @@ -1352,12 +1316,12 @@ def add_annotations(self, node, relevant, doc): if annotations: doc["annotations"] = annotations - def add_acl(self, node, doc): + def add_acl(self, node: models.Node, doc: dict) -> None: """Add the protection status of a file to the file document.""" self.add_file_access(node, doc) doc["acl"] = node.acl - def add_file_access(self, node: Node, doc: dict) -> None: + def add_file_access(self, node: models.Node, doc: dict) -> None: """Summarize file ACL and Add it to doc["access"]. Summarizes whether the ACL implies that the file is either ``open`` @@ -1369,11 +1333,11 @@ def add_file_access(self, node: Node, doc: dict) -> None: else: doc["access"] = "controlled" - def get_file_associated_entities(self, node: Node) -> Iterable[Node]: + def get_file_associated_entities(self, node: models.Node) -> Iterable[models.Node]: """Return a list of entities that are 'associated' with a file.""" - return self.neighbors_labeled(node, self.possible_associated_entities) + return self.neighbors_labeled(node, POSSIBLE_ASSOCIATED_ENTITIES) - def add_file_associated_entities(self, node: Node, doc, case_id): + def add_file_associated_entities(self, node: models.Node, doc: dict) -> None: self._cache_entity_cases() docs = [] @@ -1400,7 +1364,7 @@ def add_file_associated_entities(self, node: Node, doc, case_id): if docs: doc["associated_entities"] = docs - def upsert_file_into_dict(self, files, file_doc): + def upsert_file_into_dict(self, files: dict[str, dict], file_doc: dict) -> None: did = file_doc["file_id"] if did not in files: files[did] = file_doc @@ -1416,7 +1380,7 @@ def upsert_file_into_dict(self, files, file_doc): # Project summaries ################################################################### - def denormalize_project(self, p: Node) -> dict: + def denormalize_project(self, p: models.Node) -> dict: """Summarize a project.""" self._cache_all() doc = self._get_base_doc(p) @@ -1435,7 +1399,7 @@ def denormalize_project(self, p: Node) -> dict: # Get files log.info("Getting files") - files = set() + files: set[models.Node] = set() case_files = {} for case in cases: case_files[case] = self.remove_bam_index_files( @@ -1509,11 +1473,14 @@ def denormalize_project(self, p: Node) -> dict: doc["primary_site"] = list(primary_sites) # Compile summary - doc["summary"] = { - "case_count": len(cases), - "file_count": len(files), - "file_size": sum(f["file_size"] or 0 for f in files), - } + doc["summary"] = cast( + dict, + { + "case_count": len(cases), + "file_count": len(files), + "file_size": sum(f["file_size"] or 0 for f in files), + }, + ) if exp_strat_summaries: doc["summary"]["experimental_strategies"] = exp_strat_summaries @@ -1530,8 +1497,8 @@ def denormalize_project(self, p: Node) -> dict: ################################################################### def denormalize_cases( - self, cases: Iterable[Node] = None - ) -> Tuple[List[dict], List[dict], List[dict]]: + self, cases: Collection[models.Node] = () + ) -> tuple[list[dict], list[dict], list[dict]]: """Denormalize specified cases or all cases in graph. If cases is not specified, denormalize all cases in @@ -1539,11 +1506,12 @@ def denormalize_cases( given. :returns: - Tuple containing (case docs, file docs, annotation docs) + tuple containing (case docs, file docs, annotation docs) """ self._cache_all() - case_docs, ann_docs, file_docs = [], {}, {} + case_docs, ann_docs = [], {} + file_docs: dict[str, dict] = {} if not cases: cases = self.cases pbar = self.pbar("Denormalizing cases ", len(cases)) @@ -1560,7 +1528,9 @@ def denormalize_cases( pbar.finish() return case_docs, list(file_docs.values()), list(ann_docs.values()) - def denormalize_projects(self, projects: Iterable[Node] = None) -> List[dict]: + def denormalize_projects( + self, projects: Collection[models.Node] = () + ) -> list[dict]: """Denormalize specified projects or all projects in graph. If projects is not specified, denormalize all projects in @@ -1579,7 +1549,7 @@ def denormalize_projects(self, projects: Iterable[Node] = None) -> List[dict]: pbar.finish() return project_docs - def denormalize_annotation(self, node): + def denormalize_annotation(self, node: models.Node) -> dict: """Denormalize a specific annotation. .. note: The project of an annotation will be injected during @@ -1622,28 +1592,43 @@ def denormalize_annotation(self, node): with self.g.session_scope() as sxn, sxn.no_autoflush: annotation = self.g.nodes().get(node.node_id) - cases = [e for e in annotation.edges_out if e.label == "relates_to"] - if cases: - ann_doc["case_id"] = cases[0].dst_id + case = more_itertools.first( + ( + e + for e in annotation.edges_out # type: ignore + if e.label == "relates_to" + ), + default=None, + ) + + if case: + ann_doc["case_id"] = case.dst_id return ann_doc - def denormalize_annotations(self, annotations, projects=None): + def denormalize_annotations( + self, + annotations: Iterable[models.Node], + projects: Mapping[str, dict] = types.MappingProxyType({}), + ) -> list[dict]: g = self.g annotation_ids = [node.node_id for node in annotations] if not annotation_ids: return [] - projects = projects or {} + # NOTE: This santiy check is required as the linter thinks that Edges could be None. + assert models.Edge with g.session_scope(can_inherit=False) as sxn, sxn.no_autoflush: - edges_q = g.edges().filter(Edge.src_id.in_(annotation_ids)) + edges_q = g.edges().filter(models.Edge.src_id.in_(annotation_ids)) entities = dict() ann_to_entity = dict() ann_to_case = dict() + assert edges_q + for edge in edges_q: # Can't query on label since it's a hybrid property if edge.label != "annotates": @@ -1711,7 +1696,7 @@ def denormalize_annotations(self, annotations, projects=None): return docs - def denormalize_all(self): + def denormalize_all(self) -> tuple[list[dict], list[dict], list[dict], list[dict]]: """Return an entire index worth of case, file, annotation, and project documents.""" cases, files, _ = self.denormalize_cases() projects = self.denormalize_projects() @@ -1729,34 +1714,32 @@ def denormalize_all(self): ), ) - self.annotations = self.annotations or [] + self.annotations = self.annotations or () annotations = self.denormalize_annotations( self.annotations, projects=project_lookup ) - return cases, files, annotations, projects + # NOTE: This is new logic from the active builder. + projects_map = { + p["project_id"]: { + "primary_site": p["primary_site"], + "disease_type": p["disease_type"], + } + for p in projects + } - def denormalize_cases_sample(self, k=10): - """Return an entire index worth of case, file, annotation documents.""" - self._cache_all() - cases = random.sample(self.cases, k) - cases, files, annotations = self.denormalize_cases(cases) - return cases, files, annotations + for case in cases: + project_id = case["project"]["project_id"] + case["project"]["primary_site"] = projects_map[project_id]["primary_site"] + case["project"]["disease_type"] = projects_map[project_id]["disease_type"] - def denormalize_sample(self, k=10): - """Return an entire index worth of case, file, annotation, and project documents.""" - cases, files, annotations = self.denormalize_sample_cases(k) - projs = random.sample(self.projects, 1) - projects = self.denormalize_projects(projs) return cases, files, annotations, projects ################################################################### # Graph functions ################################################################### - def nodes_labeled( - self, labels: Union[Iterable, str] - ) -> Generator[Node, None, None]: + def nodes_labeled(self, labels: Union[Iterable, str]) -> Iterator[models.Node]: """Return an iterator over the edges in the graph with label `label`. Args: @@ -1776,25 +1759,12 @@ def nodes_labeled( if n.label in labels: yield n - @staticmethod - def node_labels_by_category(categories: Union[Iterable, str]) -> List[str]: - """Return an iterator of node labels that are files.""" - categories = ( - tuple(categories) if hasattr(categories, "__iter__") else (categories,) - ) - - return [ - n.label - for n in Node.get_subclasses() - if n._dictionary["category"] in categories - ] - def neighbors_labeled( self, - node: Node, + node: models.Node, labels: Union[str, Iterable[str]], expected: Optional[int] = None, - ) -> Generator[Node, None, None]: + ) -> Iterator[models.Node]: """Get neighbors of node, with desired labels. For a given node, return an iterator with generates neighbors to @@ -1844,70 +1814,6 @@ def neighbors_labeled( # Validation functions ################################################################### - def validate_project_file_counts(self, project_doc, file_docs): - log.info("Validating {}".format(project_doc["project_id"])) - actual = len( - [ - f - for f in file_docs - if project_doc["project_id"] - in {p["project"]["project_id"] for p in f["cases"]} - ] - ) - expected = project_doc["summary"]["file_count"] - if actual != expected: - self.error( - "File count mismatch", - "{} file count mismatch: {} != {}".format( - project_doc["project_id"], actual, expected - ), - tags=["project_id:{}".format(project_doc["project_id"])], - ) - - def validate_docs(self, case_docs, file_docs, ann_docs, project_docs): - for project_doc in project_docs: - self.validate_project_file_counts(project_doc, file_docs) - case_sample = random.sample(case_docs, min(len(case_docs), 100)) - for case_doc in case_sample: - self.verify_data_category_count(case_doc) - self.validate_annotations(ann_docs) - - def validate_annotations(self, ann_docs): - for ann_doc in ann_docs: - if ann_doc["entity_type"] == "case": - if ann_doc["entity_id"] != ann_doc["case_id"]: - self.error( - "Annotation case_id does not match entity_id", - "case_id/entity_id mismatch: {} != {}".format( - ann_doc["entity_id"], ann_doc["case_id"] - ), - tags=[ - "annotation_id:{}".format(ann_doc.get("annotation_id", "?")) - ], - ) - - def verify_data_category_count(self, case): - for data_category in self.existing_data_types: - calc = len( - [f for f in case["files"] if f.get("data_category") == data_category] - ) - - act = ( - [ - d["file_count"] - for d in case["summary"]["data_categories"] - if d["data_category"] == data_category - ][:1] - or [0] - )[0] - - if act != calc: - self.error( - "Inconsistent data_category count", - f"{data_category}: {act} != {calc}", - tags=["case_id:{}".format(case.get("case_id", "?"))], - ) - def validate_against_mapping( self, doc: Union[dict, list], mapping: Mapping[str, Any] ) -> None: @@ -1942,7 +1848,7 @@ def validate_against_mapping( for list_entry in doc: self.validate_against_mapping(list_entry, mapping) - def validate_case(self, node, case): + def validate_case(self, node: models.Node, case: dict) -> None: # Check that file count = summary.file_count if len(case["files"]) != case["summary"]["file_count"]: self.error( @@ -1961,21 +1867,20 @@ def validate_case(self, node, case): ################################################################### @staticmethod - def is_harmonized_file(node): + def is_harmonized_file(node: models.Node) -> bool: return node.label == "file" and node._sysan.get("source", "").endswith( "_alignment" ) - def is_old_supplement_file(self, node): + def is_old_supplement_file(self, node: models.Node) -> bool: if node.label == "file": node = self.add_file_metadata_from_indexd(node) return any( - p.match(node._props.get("file_name", "")) - for p in self.supplement_regexes + p.match(node._props.get("file_name", "")) for p in SUPPLEMENT_PATTERNS ) return False - def is_file_indexed(self, node): + def is_file_indexed(self, node: models.Node) -> bool: """Return false if node is a file that is not supposed to be indexed.""" # This function should test only file nodes if node.label not in self.file_labels: @@ -2016,7 +1921,7 @@ def is_file_indexed(self, node): return True - def is_omitted_project_or_neighbor_case(self, node: Node) -> bool: + def is_omitted_project_or_neighbor_case(self, node: models.Node) -> bool: """Return false if the node is a project that is not supposed to be indexed.""" if node.label == "project": projects = [node] @@ -2042,7 +1947,7 @@ def is_omitted_project_or_neighbor_case(self, node: Node) -> bool: # Check project and program against omitted_projects for program_name in program_names: for project_code in project_codes: - if (program_name, project_code) in self.omitted_projects: + if (program_name, project_code) in OMITTED_PROJECTS: return True elif self.build_projects: if (program_name, project_code) in self.build_projects: @@ -2057,7 +1962,7 @@ def is_unindexed_case(self, node): self.neighbors_labeled(node, "project", 1) ) - def is_node_unindexed_by_property(self, node: Node) -> bool: + def is_node_unindexed_by_property(self, node: models.Node) -> bool: """Check if node properties specified in self.unindexed_by_property. Returns True if node should be removed because its properties are @@ -2075,7 +1980,7 @@ def is_node_unindexed_by_property(self, node: Node) -> bool: return False - def is_node_public(self, node: Node) -> bool: + def is_node_public(self, node: models.Node) -> bool: """Return whether a node is public. A node is public if: @@ -2124,17 +2029,17 @@ def is_node_public(self, node: Node) -> bool: return False - def cache_skipped_node(self, node, reason): + def cache_skipped_node( + self, node: Union[models.Node, tuple[models.Node, str]], reason: str + ) -> None: """Cache skipped node in self.skipped_nodes['{reason-for-skipping}'].""" - # TODO: Use defaultdict for skipped_nodes - self.skipped_nodes.setdefault(reason, []) self.skipped_nodes[reason].append(str(node)) - def is_node_indexed(self, node): + def is_node_indexed(self, node: models.Node) -> bool: """Return false if the node is not supposed to be indexed.""" # Is the node allowed to be displayed publicly if not self.is_node_public(node): - self.cache_skipped_node([node, node._props.get("state")], "not-public") + self.cache_skipped_node((node, node._props.get("state")), "not-public") return False if self.is_unindexed_case(node): @@ -2159,7 +2064,7 @@ def is_node_indexed(self, node): return True @staticmethod - def truncate_path(path: List[str], label: str) -> List[str]: + def truncate_path(path: Sequence[str], label: str) -> Sequence[str]: """Truncate a path, so it starts from next value of the given label. Given a path (a list of node labels), "truncate" it from the left @@ -2178,42 +2083,43 @@ def truncate_path(path: List[str], label: str) -> List[str]: for i, currlabel in enumerate(path): if currlabel == label: return path[i + 1 :] - return [] + return () - def get_suppressed_children(self, redacted): + def get_suppressed_children(self, redacted: models.Node) -> Iterator[models.Node]: """Get the children of a redacted node.""" - to_suppress = [] if redacted.label == "case": paths = self.case_to_file_paths else: - paths = [ + paths = ( self.truncate_path(p, redacted.label) for p in self.case_to_file_paths - ] + ) # filter empty paths - paths = [p for p in paths if p] + paths = tuple(filter(None, paths)) log.info("suppressing %s, which is redacted directly.", redacted) - to_suppress.append(redacted) + yield redacted + log.info("Walking down towards file with paths %s", paths) extra = self.walk_paths(redacted, paths, whole=True) + log.info( "Found %s other things to suppress by walking from %s", extra, redacted ) - to_suppress.extend(extra) - return to_suppress + yield from extra - def get_redaction_annotations(self): + def get_redaction_annotations(self) -> Iterator[models.Node]: """Return an iterator of annotations that should cause redactions.""" return ( annotation for annotation in self.nodes_labeled("annotation") if annotation.classification == "Redaction" and annotation.status != "Rescinded" - and annotation.category not in self.redacted_but_not_suppressed + # Suppress entities with redaction annotation if + # entity.annotation.category not in this list + and annotation.category not in ("Subject withdrew consent",) # ) - def suppressed_nodes(self): + def suppressed_nodes(self) -> Iterator[models.Node]: """Find all nodes that need to be suppressed due to redactions.""" - to_suppress = [] for redaction in self.get_redaction_annotations(): redacted_list = list(self.G.neighbors(redaction)) @@ -2240,9 +2146,7 @@ def suppressed_nodes(self): ) for redacted in redacted_list: - to_suppress += self.get_suppressed_children(redacted) - - return to_suppress + yield from self.get_suppressed_children(redacted) def remove_unindexed_nodes_from_graph(self) -> None: """Remove unindexed nodes and suppressed nodes from cached networkx graph. @@ -2252,17 +2156,17 @@ def remove_unindexed_nodes_from_graph(self) -> None: - suppressed_nodes() """ log.info("Selecting entities to be removed from cache...") - removed_nodes = [ + removed_nodes = tuple( node for node in self.G.nodes() if not self.is_node_indexed(node) - ] + ) log.info(f"Removing {len(removed_nodes)} nodes from cache") self.G.remove_nodes_from(removed_nodes) log.info("Finding and removing suppressed nodes") - suppressed = self.suppressed_nodes() + suppressed = tuple(self.suppressed_nodes()) log.info("Removing %s suppressed nodes", len(suppressed)) self.G.remove_nodes_from(suppressed) - def iter_database_edges(self) -> Iterator[Edge]: + def iter_database_edges(self) -> Iterator[models.Edge]: # type: ignore """Return an iterable of edges to load from the database. Eagerly (with join) loads the source and destination of the edge. @@ -2279,33 +2183,31 @@ def iter_database_edges(self) -> Iterator[Edge]: log.info(f"Getting {project_ids} from database") relevant_node_ids = { - nd.node_id for nd in self.g.nodes().prop_in("project_id", project_ids) + nd.node_id for nd in self.g.nodes().prop_in("project_id", project_ids) # type: ignore } # Add relevant Project nodes to relevant nodes set: projects = [p[1] for p in self.build_projects] - relevant_projects = self.g.nodes(md.Project).prop_in("code", projects) + relevant_projects = self.g.nodes(models.Project).prop_in("code", projects) # type: ignore - relevant_node_ids.update([p.node_id for p in relevant_projects]) + relevant_node_ids.update(p.node_id for p in relevant_projects) # Query only relevant edges - query = lambda node_type: self.g.edges(node_type).src(relevant_node_ids) + query = lambda node_type: self.g.edges(node_type).src(relevant_node_ids) # type: ignore else: # Query all edges query = lambda node_type: self.g.edges(node_type) - return itertools.chain( - *[ - query(subclass) - .options(joinedload(subclass.src)) - .options(joinedload(subclass.dst)) - .yield_per(int(1e5)) - for subclass in Edge.__subclasses__() - ] + return itertools.chain.from_iterable( + query(subclass) + .options(orm.joinedload(subclass.src)) + .options(orm.joinedload(subclass.dst)) # type: ignore + .yield_per(int(1e5)) + for subclass in models.Edge.get_subclasses() # type: ignore ) - def cache_database(self): + def cache_database(self) -> None: """Cache database psqlgraph into memory. Load the database into memory and remember only edge labels that we @@ -2320,7 +2222,7 @@ def cache_database(self): for e in self.iter_database_edges(): pbar.update(pbar.value + 1) triple = (e.src.label, e.label, e.dst.label) - needs_differentiation = triple in self.differentiated_edges + needs_differentiation = triple in DIFFERENTIATED_EDGES if triple == ("file", "data_from", "file"): # for files that are "data_from" other files, the # centers and aliquots of the source files count @@ -2369,14 +2271,14 @@ def _cache_cases(self) -> None: """Cache a list of all Case nodes.""" if not self.cases: log.info("Caching cases...") - self.cases = list(self.nodes_labeled("case")) + self.cases = tuple(self.nodes_labeled("case")) - def _cache_entity_cases(self): + def _cache_entity_cases(self) -> None: """Cache the related Case nodes for each file.""" if self.entity_cases: return - entities = list(self.nodes_labeled(self.possible_associated_entities)) + entities = list(self.nodes_labeled(POSSIBLE_ASSOCIATED_ENTITIES)) pbar = self.pbar("Caching entity cases: ", len(entities)) self.entity_cases = {} @@ -2404,12 +2306,12 @@ def _cache_entity_cases(self): continue if len(cases) != 0: - self.entity_cases[e] = cases.pop() + self.entity_cases[e] = more_itertools.first(cases) pbar.update(pbar.value + 1) pbar.finish() - def get_cls_file_to_case_paths(self, cls: Node) -> Iterable[List[str]]: + def get_cls_file_to_case_paths(self, cls: models.Node) -> Iterable[Sequence[str]]: """Given a node, return the paths the lead monotonically up to case. Args: @@ -2418,7 +2320,9 @@ def get_cls_file_to_case_paths(self, cls: Node) -> Iterable[List[str]]: Returns: generator of paths from node to case """ - parent_labels = {link["dst_type"].label for link in cls._pg_links.values()} + parent_labels = frozenset( + link["dst_type"].label for link in cls._pg_links.values() + ) return ( path for path in self.file_to_case_paths @@ -2450,10 +2354,10 @@ def _cache_relevant_nodes(self) -> None: pbar.finish() - def _cache_annotations(self): + def _cache_annotations(self) -> None: if not self.annotations: # cache what nodes are annotations - self.annotations = list(self.nodes_labeled("annotation")) + self.annotations = tuple(self.nodes_labeled("annotation")) if self.annotation_entities: # we've already cached the related entities return @@ -2473,13 +2377,18 @@ def _cache_annotations(self): pbar.update(pbar.value + 1) pbar.finish() - def _cache_popular_neighbor(self, node, neighbors, labels): - if node not in self.popular_nodes: - self.popular_nodes[node] = {} - self.popular_nodes[node][labels] = {n for n in neighbors if n.label in labels} + def _cache_popular_neighbor( + self, + node: models.Node, + neighbors: Iterable[models.Node], + labels: tuple[str, ...], + ) -> Set[models.Node]: + self.popular_nodes[node][labels] = frozenset( + n for n in neighbors if n.label in labels + ) return self.popular_nodes[node][labels] - def _cache_experimental_strategies(self): + def _cache_experimental_strategies(self) -> None: """Cache files classified in each experimental strategy. Looking up the files that are classified in each @@ -2498,7 +2407,7 @@ def _cache_experimental_strategies(self): self.walk_path(exp_strat, ["file"]) ) - def _cache_data_categories(self): + def _cache_data_categories(self) -> None: log.info("Caching data categories/types") if self.data_categories: @@ -2510,7 +2419,7 @@ def _cache_data_categories(self): set(self.walk_path(data_category, ["data_subtype", "file"])) ) - def _cache_file_properties(self): + def _cache_file_properties(self) -> None: """Cache file nodes by some properties. Cache file nodes by the following properties: experimental_strategy, @@ -2524,30 +2433,29 @@ def _cache_file_properties(self): if strategy: self.experimental_strategies[strategy].add(file_) - data_type = file_._props.get("data_type") - if data_type: - self.existing_data_types.add(data_type) - category = file_._props.get("data_category") if category: self.data_categories[category].add(file_) -def get_namespaced_uuid(ns, seed): +def get_namespaced_uuid(ns: str, seed: str) -> str: """Create a consistent UUID5 string using the provided args. Args: - ns (str): namespace - seed (str): seed value + ns: namespace + seed: seed value Returns: str: uuid5 string """ namespace = get_uuid_namespace(ns) - return str(uuid5(namespace, seed)) + return str(uuid.uuid5(namespace, seed)) -def get_uuid_namespace(label): +UUID_NAMESPACES: dict[str, uuid.UUID] = {} + + +def get_uuid_namespace(label: str) -> uuid.UUID: """Return a consistent uuid4 string for a given label. Args: @@ -2559,12 +2467,8 @@ def get_uuid_namespace(label): if label in UUID_NAMESPACES: return UUID_NAMESPACES[label] - namespace = hashlib.sha1(bytes(label, "utf-8")).hexdigest() + namespace = hashlib.sha1(bytes(label, "utf-8"), usedforsecurity=False).hexdigest() namespace = namespace[:32] - namespace_uuid = UUID(hex=namespace, version=4) - - UUID_NAMESPACES[label] = namespace_uuid - return namespace_uuid - + namespace_uuid = uuid.UUID(hex=namespace, version=4) -UUID_NAMESPACES = {} + return UUID_NAMESPACES.setdefault(label, namespace_uuid) diff --git a/esbuild/graph/common/mimic.py b/esbuild/graph/common/mimic.py deleted file mode 100644 index eb596d3c..00000000 --- a/esbuild/graph/common/mimic.py +++ /dev/null @@ -1,38 +0,0 @@ -"""esbuild.graph.common.mimic. - -Mimics the isolation of functionality such as filtering nodes from the -index. - -""" -import logging -from typing import Iterable, Union - -log = logging.getLogger(__name__) - - -class CommonMimic: - """Mixin for mimic classes.""" - - def neighbors_labeled( - self, node, labels: Union[str, Iterable[str]], *args, **kwargs - ): - """Get node neighbors whose label is in labels. - - For a given node, return an iterator with generates neighbors to - that node that are in a list of labels. `label` can be either a - string or list of strings. - - .. note:: - This is mocked to use association proxies as a workaround - to the assumption that the real builder will have the - entire graph cached. - - """ - if isinstance(labels, Iterable) and not isinstance(labels, str): - labels = set(labels) - else: - labels = {labels} - - return [edge.dst for edge in node.edges_out if edge.dst.label in labels] + [ - edge.src for edge in node.edges_in if edge.src.label in labels - ] diff --git a/esbuild/reports/__init__.py b/esbuild/reports/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/esbuild/reports/alignment_queries.py b/esbuild/reports/alignment_queries.py deleted file mode 100644 index 6d8b2ab4..00000000 --- a/esbuild/reports/alignment_queries.py +++ /dev/null @@ -1,96 +0,0 @@ -from gdcdatamodel2 import models -from sqlalchemy import BigInteger, Boolean, desc, func - -SORT_ORDER = ( - models.File._sysan["alignment_seen_docker_error"].cast(Boolean).nullsfirst(), - func.random(), -) - - -def exome(graph, source): - wxs = models.ExperimentalStrategy.name.astext == "WXS" - illumina = models.Platform.name.astext.contains("Illumina") - bam = models.DataFormat.name.astext == "BAM" - all_file_ids_sq = ( - graph.nodes(models.File.node_id) - .sysan(source=source) - .distinct(models.File._sysan["cghub_legacy_sample_id"].astext) - .filter(models.File.experimental_strategies.any(wxs)) - .filter(models.File.platforms.any(illumina)) - .filter(models.File.data_formats.any(bam)) - .order_by( - models.File._sysan["cghub_legacy_sample_id"].astext, - desc(models.File._sysan["cghub_upload_date"].cast(BigInteger)), - ) - .subquery() - ) - return graph.nodes(models.File).filter( - models.File.node_id == all_file_ids_sq.c.node_id - ) - - -def wgs(graph, source): - wgs = models.ExperimentalStrategy.name.astext == "WGS" - illumina_plus_hiseq_x_ten = models.Platform.name.astext.contains("Illumina") | ( - models.Platform.name.astext == "HiSeq X Ten" - ) - bam = models.DataFormat.name.astext == "BAM" - all_file_ids_sq = ( - graph.nodes(models.File.node_id) - .sysan(source=source) - .distinct(models.File._sysan["cghub_legacy_sample_id"].astext) - .filter(models.File.experimental_strategies.any(wgs)) - .filter(models.File.platforms.any(illumina_plus_hiseq_x_ten)) - .filter(models.File.data_formats.any(bam)) - .order_by( - models.File._sysan["cghub_legacy_sample_id"].astext, - desc(models.File._sysan["cghub_upload_date"].cast(BigInteger)), - ) - .subquery() - ) - return graph.nodes(models.File).filter( - models.File.node_id == all_file_ids_sq.c.node_id - ) - - -def mirnaseq(graph, source): - strategy = models.ExperimentalStrategy.name.astext == "miRNA-Seq" - platform = models.Platform.name.astext.contains("Illumina") - dataformat = models.DataFormat.name.astext == "BAM" - - subquery = ( - graph.nodes(models.File.node_id) - .sysan(source=source) - .distinct(models.File._sysan["cghub_legacy_sample_id"].astext) - .filter(models.File.experimental_strategies.any(strategy)) - .filter(models.File.platforms.any(platform)) - .filter(models.File.data_formats.any(dataformat)) - .order_by( - models.File._sysan["cghub_legacy_sample_id"].astext, - desc(models.File._sysan["cghub_upload_date"].cast(BigInteger)), - ) - .subquery() - ) - return graph.nodes(models.File).filter(models.File.node_id == subquery.c.node_id) - - -def rnaseq(graph, source): - strategy = models.ExperimentalStrategy.name.astext == "RNA-Seq" - platform = models.Platform.name.astext.contains("Illumina") - dataformat = models.DataFormat.name.astext.in_(["TAR", "TARGZ"]) - - subquery = ( - graph.nodes(models.File.node_id) - .sysan(source=source) - .distinct(models.File._sysan["cghub_legacy_sample_id"].astext) - .filter(models.File.experimental_strategies.any(strategy)) - .filter(models.File.platforms.any(platform)) - .filter(models.File.data_formats.any(dataformat)) - .order_by( - models.File._sysan["cghub_legacy_sample_id"].astext, - desc(models.File._sysan["cghub_upload_date"].cast(BigInteger)), - ) - .subquery() - ) - - return graph.nodes(models.File).filter(models.File.node_id == subquery.c.node_id) diff --git a/esbuild/reports/alignment_report.py b/esbuild/reports/alignment_report.py deleted file mode 100644 index 8586b01a..00000000 --- a/esbuild/reports/alignment_report.py +++ /dev/null @@ -1,354 +0,0 @@ -import logging -import os -import smtplib -from datetime import datetime, timedelta -from email import encoders -from email.MIMEBase import MIMEBase -from email.MIMEMultipart import MIMEMultipart -from email.MIMEText import MIMEText - -import salt.client -from consulate import Consul -from gdcdatamodel2 import models -from psqlgraph import PsqlGraphDriver -from sqlalchemy import create_engine, desc -from sqlalchemy.pool import NullPool - -from esbuild.reports.alignment_queries import exome, mirnaseq, rnaseq, wgs - -logger = logging.getLogger(__name__) - - -def with_derived(q): - return q.filter(File.derived_files.any()) - - -def alignment_time(file): - return ( - file._FileDataFromFile_out[0].sysan["alignment_finished"] - - file._FileDataFromFile_out[0].sysan["alignment_started"] - ) - - -ALIGNER_NAMES = { - "tcga_wgs_aligner": "TCGA WGS", - "tcga_exome_aligner": "TCGA Exome", - "tcga_rnaseq_aligner": "TCGA RNA-Seq", - "tcga_mirnaseq_aligner": "TCGA miRNA-Seq", - "target_exome_aligner": "TARGET Exome", - "target_rnaseq_aligner": "TARGET RNA-Seq", -} - - -class AlignmentReporter: - def __init__(self, graph=None, os_mysql=None, mailserver=None, toaddrs=None): - if graph: - self.graph = graph - else: - self.graph = PsqlGraphDriver( - os.environ["PG_HOST"], - os.environ["PG_USER"], - os.environ["PG_PASS"], - os.environ["PG_NAME"], - poolclass=NullPool, - ) - self.consul = Consul() - self.salt_caller = salt.client.Caller() - if os_mysql: - self.os_mysql = os_mysql - else: - conn_str = "mysql://{user}:{pw}@{host}/{db}".format( - user=os.environ["OS_MYSQL_USER"], - pw=os.environ["OS_MYSQL_PASS"], - host=os.environ["OS_MYSQL_HOST"], - db=os.environ["OS_MYSQL_NAME"], - ) - self.os_mysql = create_engine(conn_str) - self.mailserver = mailserver - self.toaddrs = toaddrs - self._aligned = None - - @property - def totals(self): - """Totals per zhenyu.""" - return { - "WGS (>= 320 GB)": 364, - "WGS (< 320 GB)": 4355, - "WXS (TCGA)": 22561, - "WXS (TARGET)": 1630, - "miRNA-Seq": 11914, - "RNA-Seq (TARGET)": 721, - "RNA-Seq (TCGA)": 11293, - } - - @property - def total_sizes(self): - """Total sizes (in bytes) per zhenyu.""" - return { - "WGS (>= 320 GB)": 161761091978632, - "WGS (< 320 GB)": 365747226114438, - "WXS (TCGA)": 315891670201207, - "WXS (TARGET)": 21382957729925, - "miRNA-Seq": 236939308278 + 2791773509555, - "RNA-Seq (TARGET)": 10099756185600, - "RNA-Seq (TCGA)": 74780058034896, - } - - @property - def aligned_files(self): - if not self._aligned: - logger.info("Querying for aligned files") - wgs_files = ( - with_derived(wgs(self.graph, "tcga_cghub")).all() - + with_derived(wgs(self.graph, "target_cghub")).all() - ) - self._aligned = { - "WGS (>= 320 GB)": [ - f for f in wgs_files if f.file_size >= 320000000000 - ], - "WGS (< 320 GB)": [f for f in wgs_files if f.file_size < 320000000000], - "WXS (TCGA)": with_derived(exome(self.graph, "tcga_cghub")).all(), - "WXS (TARGET)": with_derived(exome(self.graph, "target_cghub")).all(), - "miRNA-Seq": with_derived(mirnaseq(self.graph, "tcga_cghub")).all() - + with_derived(mirnaseq(self.graph, "target_cghub")).all(), - "RNA-Seq (TARGET)": with_derived( - rnaseq(self.graph, "target_cghub") - ).all(), - "RNA-Seq (TCGA)": with_derived(rnaseq(self.graph, "tcga_cghub")).all(), - } - return self._aligned - - def aligned_file_counts(self): - return {key: len(val) for key, val in self.aligned_files.items()} - - def aligned_file_sizes(self): - return { - key: sum(f.file_size for f in val) - for key, val in self.aligned_files.items() - } - - def generate_files_to_attach(self): - logger.info("Generating files to attach") - return { - "alignment_numbers.txt": self.generate_numbers_file(), - "analysis_ids_complete.txt": self.generate_aligned_analysis_ids_file(), - "analysis_ids_in_progres.txt": self.generate_in_progress_analysis_ids_file(), - "wgs_timings.txt": self.generate_timings_file(), - "analysis_ids_fixmate_problem.txt": self.generate_fixmate_problem_analysis_ids_file(), - "analysis_ids_markdups_failure.txt": self.generate_markdups_failure_analysis_ids_file(), - } - - def generate_numbers_file(self): - logger.info("Generating file with aligned numbers") - aligned_counts = self.aligned_file_counts() - aligned_sizes = self.aligned_file_sizes() - attachment = "Aligned Files (counts)\n" - attachment += "=============\n\n" - attachment += "\n".join( - [ - "{key}: {aligned} / {total} ({percent:.2f}%)".format( - key=key, - aligned=aligned_counts[key], - total=total, - percent=100 * (float(aligned_counts[key]) / total), - ) - for key, total in iter(sorted(self.totals.items())) - ] - ) - attachment += "\n\n" - attachment += "Total sizes aligned\n" - attachment += "=============\n\n" - attachment += "\n".join( - [ - "{key}: {aligned:.2f} TB / {total:.2f} TB ({percent:.2f}%)".format( - key=key, - aligned=float(aligned_sizes[key]) / 1e12, - total=float(total_size) / 1e12, - percent=100 * (float(aligned_sizes[key]) / total_size), - ) - for key, total_size in iter(sorted(self.total_sizes.items())) - ] - ) - attachment += "\n\n" - # breakdown WGS by step completed - attachment += "WGS Aligned files breakdown by step completed\n" - attachment += "=============\n\n" - for key in ["WGS (< 320 GB)", "WGS (>= 320 GB)", "WXS (TCGA)", "WXS (TARGET)"]: - all_of_key = self.aligned_files[key] - edges = [ - self.graph.edges(models.FileDataFromFile) - .src(f.node_id) - .order_by(desc(models.FileDataFromFile.created)) - .first() - for f in all_of_key - ] - fully_complete = [ - e for e in edges if e.sysan.get("alignment_last_step") in [None, "md"] - ] - fixmate_finished = [ - e for e in edges if e.sysan.get("alignment_last_step") == "fixmate" - ] - merge_finished = [ - e for e in edges if e.sysan.get("alignment_last_step") == "reheader" - ] - attachment += key + "\n" - attachment += ("=" * len(key)) + "\n" - attachment += ( - "Merge finished: {count} ({size:.2f} TB)".format( - count=len(merge_finished), - size=float(sum(e.src.file_size for e in merge_finished)) / 1e12, - ) - + "\n" - ) - attachment += ( - "Fixmate finished: {count} ({size:.2f} TB)".format( - count=len(fixmate_finished), - size=float(sum(e.src.file_size for e in fixmate_finished)) / 1e12, - ) - + "\n" - ) - attachment += ( - "Fully Complete: {count} ({size:.2f} TB)".format( - count=len(fully_complete), - size=float(sum(e.src.file_size for e in fully_complete)) / 1e12, - ) - + "\n" - ) - attachment += "\n" - # now add running alignment counts - attachment += "Currently running aligners\n" - attachment += "==========================\n" - consul_keys = list(self.consul.kv.keys()) - mine_results = self.salt_caller.sminion.functions["mine.get"]( - "service:aligner", "grains.items", "grain" - ) - for prefix, name in ALIGNER_NAMES.items(): - running = len([k for k in consul_keys if prefix in k]) - # this is true for now, let's keep it the case - alignment_type_grain = prefix.replace("_aligner", "") - allocated = len( - { - k: v - for k, v in mine_results.items() - if v.get("alignment_type") == alignment_type_grain - } - ) - attachment += ( - "{name}: {running} currently running / {allocated} allocated\n".format( - name=name, running=running, allocated=allocated - ) - ) - return attachment - - def generate_aligned_analysis_ids_file(self): - logger.info("Generating file with aligned analysis ids") - analysis_ids = [] - for _, files in self.aligned_files.items(): - analysis_ids.extend([f.sysan["analysis_id"] for f in files]) - attachment = "\n".join(analysis_ids) - return attachment - - def generate_in_progress_analysis_ids_file(self): - logger.info("Generating file with in progress analysis ids") - in_progres_gdc_ids = [ - k.split("/")[-1] - for k in self.consul.kv.keys() - if "align" in k and "current" in k - ] - logger.info("Querying for analysis ids of files currently being aligned") - analysis_ids = [ - res[0] - for res in self.graph.nodes(models.File._sysan["analysis_id"]) - .filter(models.File.node_id.in_(in_progres_gdc_ids)) - .all() - ] - attachment = "\n".join(analysis_ids) - return attachment - - def generate_timings_file(self): - logger.info("Generating wgs timings file") - aligned_wgs_files = [] - for key, files in self.aligned_files.items(): - if "WGS" in key: - aligned_wgs_files.extend(files) - logger.info("Getting uuid -> hostname mapping from gdc mysql") - uuid_to_host = dict( - self.os_mysql.execute("SELECT uuid, host from instances;").fetchall() - ) - attachment = ( - "analysis_id,alignment_time,input_file_size,aligner_uuid,aligner_host\n" - ) - logger.info("Generating rows") - for file in aligned_wgs_files: - edge = ( - self.graph.edges(models.FileDataFromFile) - .src(file.node_id) - .order_by(desc(models.FileDataFromFile.created)) - .first() - ) - os_uuid = edge.sysan.get("alignment_host_openstack_uuid") - row = [ - file.sysan["analysis_id"], - str(alignment_time(file)), - str(file.file_size), - str(os_uuid), - str(uuid_to_host.get(os_uuid)), - ] - attachment += ",".join(row) - attachment += "\n" - return attachment - - def generate_fixmate_problem_analysis_ids_file(self): - logger.info("Generating file with in FixMateInformation failure analysis ids") - problem_files = ( - self.graph.nodes(models.File).sysan(alignment_fixmate_failure=True).all() - ) - analysis_ids = [f.sysan["analysis_id"] for f in problem_files] - attachment = "\n".join(analysis_ids) - return attachment - - def generate_markdups_failure_analysis_ids_file(self): - logger.info("Generating file with in MarkDuplicates failure analysis ids") - problem_files = ( - self.graph.nodes(models.File).sysan(alignment_markdups_failure=True).all() - ) - analysis_ids = [f.sysan["analysis_id"] for f in problem_files] - attachment = "\n".join(analysis_ids) - return attachment - - def attach_files(self, msg, files): - for name, contents in files.items(): - part = MIMEBase("application", "octet-stream") - part.set_payload(contents) - encoders.encode_base64(part) - part.add_header("Content-Disposition", f"attachment; filename= {name}") - msg.attach(part) - - def send_email(self): - logger.info("Building email") - fromaddr = "alignmentreport@opensciencedatacloud.org" - toaddrs = self.toaddrs - msg = MIMEMultipart() - - msg["From"] = fromaddr - if len(toaddrs) == 1: - msg["To"] = toaddrs[0] - else: - msg["To"] = ", ".join(toaddrs) - msg["Subject"] = "Alignment tracking (as of {})".format( - (datetime.now() - timedelta(hours=5)).strftime("%Y-%m-%d %I:%M:%S %p") - ) - - body = "Attached is the latest alignment information." - - msg.attach(MIMEText(body, "plain")) - - with self.graph.session_scope(): - files = self.generate_files_to_attach() - self.attach_files(msg, files) - logger.info("Connecting and sending email") - server = smtplib.SMTP(self.mailserver, 25) - if len(toaddrs) == 1: - server.sendmail(fromaddr, toaddrs[0], msg.as_string()) - else: - server.sendmail(fromaddr, toaddrs, msg.as_string()) diff --git a/esbuild/utils.py b/esbuild/utils.py index 85dc22d8..1a5f4f68 100644 --- a/esbuild/utils.py +++ b/esbuild/utils.py @@ -28,12 +28,12 @@ def get_file_state(doc): state = None for url, meta in doc.urls_metadata.items(): - if meta.get("type") == GraphIndexBuilder.INDEXD_URL_TYPE: + if meta.get("type") == builder.INDEXD_URL_TYPE: state = state or meta.get("state") return state -INDEXD_METADATA_FIELDS = GraphIndexBuilder.data_file_indexd_fields + ["file_id"] +INDEXD_METADATA_FIELDS = (*builder.DATA_FILE_INDEXD_FIELDS, "file_id") INDEXD_METADATA_VALUE_GETTERS = { "file_id": lambda doc: doc.did, "md5sum": lambda doc: doc.hashes["md5"], diff --git a/tests/integration/aliquot_maf/conftest.py b/tests/integration/aliquot_maf/conftest.py index e8449be0..bfae7c0e 100644 --- a/tests/integration/aliquot_maf/conftest.py +++ b/tests/integration/aliquot_maf/conftest.py @@ -11,7 +11,16 @@ def maf_graph(generate_scenario): @pytest.fixture def maf_index(pg_driver, init_indexd, maf_graph): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() index = builder.denormalize_all() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 314a4b0a..1f1cd69d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -437,7 +437,16 @@ def scenario_index(pg_driver, init_indexd, generate_scenario): def make_graph(filename): generate_scenario(filename) - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ) with pg_driver.session_scope(): builder.cache_database() diff --git a/tests/integration/copy_number/conftest.py b/tests/integration/copy_number/conftest.py index 737a1327..77f389e6 100644 --- a/tests/integration/copy_number/conftest.py +++ b/tests/integration/copy_number/conftest.py @@ -19,7 +19,16 @@ def copy_number_estimate_index( init_indexd: client.IndexClient, copy_number_estimate_graph: None, ) -> conftest.Index: - active_builder = builder.ActiveGraphIndexBuilder(pg_driver, init_indexd) + active_builder = builder.ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): active_builder.cache_database() index = active_builder.denormalize_all() @@ -38,7 +47,16 @@ def copy_number_segment_index( init_indexd: client.IndexClient, copy_number_segment_graph: None, ) -> conftest.Index: - active_builder = builder.ActiveGraphIndexBuilder(pg_driver, init_indexd) + active_builder = builder.ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): active_builder.cache_database() index = active_builder.denormalize_all() diff --git a/tests/integration/data.py b/tests/integration/data.py index 22858061..879ac0f9 100644 --- a/tests/integration/data.py +++ b/tests/integration/data.py @@ -15,10 +15,9 @@ from gdcdictionary import gdcdictionary from psqlgraph.mocks import NodeFactory -from esbuild.graph.common.builder import GraphIndexBuilder +from esbuild.graph.common import builder -DATA_FILE_CATEGORIES = GraphIndexBuilder.data_file_categories -DATA_FILE_INDEXD_FIELDS = GraphIndexBuilder.data_file_indexd_fields +DATA_FILE_CATEGORIES = ("data_file", "metadata_file") # Populated each time get_node_id is called, Used for debugging missing ids NODE_ID_TO_STRING = {} # Defaults for the node factory @@ -76,7 +75,7 @@ def patch_test_data_get_indexd(nodes): "node_id": node.node_id, } - for key in DATA_FILE_INDEXD_FIELDS: + for key in builder.DATA_FILE_INDEXD_FIELDS: key_value = getattr(node, key, None) if key_value is None: continue diff --git a/tests/integration/genotyping_array/conftest.py b/tests/integration/genotyping_array/conftest.py index ac1d9d3a..71497760 100644 --- a/tests/integration/genotyping_array/conftest.py +++ b/tests/integration/genotyping_array/conftest.py @@ -19,7 +19,16 @@ def genotyping_array_index( init_indexd: client.IndexClient, genotyping_array_graph: None, ) -> conftest.Index: - active_builder = builder.ActiveGraphIndexBuilder(pg_driver, init_indexd) + active_builder = builder.ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): active_builder.cache_database() index = active_builder.denormalize_all() diff --git a/tests/integration/methylation/conftest.py b/tests/integration/methylation/conftest.py index 48be12e1..dd3c5266 100644 --- a/tests/integration/methylation/conftest.py +++ b/tests/integration/methylation/conftest.py @@ -17,7 +17,16 @@ def maf_graph(generate_scenario: Callable) -> None: def methylation_index( pg_driver: PsqlGraphDriver, init_indexd: IndexClient, maf_graph ) -> NamedTuple: - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() index = builder.denormalize_all() diff --git a/tests/integration/methylation/test_methylation_array.py b/tests/integration/methylation/test_methylation_array.py index 778eb167..9a77f4c0 100644 --- a/tests/integration/methylation/test_methylation_array.py +++ b/tests/integration/methylation/test_methylation_array.py @@ -57,4 +57,4 @@ def test_methylation_array_counts(methylation_index, index_type, path, expectati counts = Counter(results) for value, count in expectations.items(): - assert counts[value] == count, counts + assert counts[value] == count, f"{value}: {counts[value]} != {count}" diff --git a/tests/integration/molecular_test/conftest.py b/tests/integration/molecular_test/conftest.py index dd22f897..c1fd6c6a 100644 --- a/tests/integration/molecular_test/conftest.py +++ b/tests/integration/molecular_test/conftest.py @@ -11,7 +11,16 @@ def molecular_test_scenario(generate_scenario): @pytest.fixture def molecular_test_index(pg_driver, init_indexd, molecular_test_scenario): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() index = builder.denormalize_all() diff --git a/tests/integration/pathology_detail/conftest.py b/tests/integration/pathology_detail/conftest.py index dd5e9a11..cb322333 100644 --- a/tests/integration/pathology_detail/conftest.py +++ b/tests/integration/pathology_detail/conftest.py @@ -11,7 +11,16 @@ def pathology_detail_graph(generate_scenario): @pytest.fixture def pathology_detail_index(pg_driver, init_indexd, pathology_detail_graph): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() index = builder.denormalize_all() diff --git a/tests/integration/submittable/conftest.py b/tests/integration/submittable/conftest.py index c381a4bf..e2094d94 100644 --- a/tests/integration/submittable/conftest.py +++ b/tests/integration/submittable/conftest.py @@ -11,7 +11,16 @@ def pathology_report_graph(generate_scenario): @pytest.fixture def pathology_index(pg_driver, init_indexd, pathology_report_graph): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() index = builder.denormalize_all() diff --git a/tests/integration/test_graph_active_builder.py b/tests/integration/test_graph_active_builder.py index 7124e20d..e81804e8 100644 --- a/tests/integration/test_graph_active_builder.py +++ b/tests/integration/test_graph_active_builder.py @@ -6,25 +6,23 @@ """ -import operator -from functools import reduce +import collections +import itertools +from collections.abc import Collection, Iterable, Iterator, Set import jmespath +import more_itertools import psqlgraph import pytest from gdcdatamodel2 import models from indexclient import client from esbuild.graph.active.builder import ActiveGraphIndexBuilder -from esbuild.graph.common.builder import GraphIndexBuilder +from esbuild.graph.common.builder import POSSIBLE_ASSOCIATED_ENTITIES from tests.integration.conftest import Index, raise_test_error from tests.integration.data import get_node_id from tests.integration.test_utils import validate_file_metadata -DATA_FILE_CATEGORIES = GraphIndexBuilder.data_file_categories -DATA_FILE_INDEXD_FIELDS = GraphIndexBuilder.data_file_indexd_fields - - # Define the number of files that should be loaded as documents N_FILES = 15 N_OUTPUT_FILES = 6 @@ -34,13 +32,47 @@ # this needs to be updated N_FILES_UNDER_ALIQUOT_1 = 10 + +def validate_project_file_counts(project_doc: dict, file_docs: Iterable[dict]): + project_id = project_doc["project_id"] + file_docs = filter( + lambda f: any(c["project"]["project_id"] == project_id for c in f["cases"]), + file_docs, + ) + actual = more_itertools.ilen(file_docs) + expected = project_doc["summary"]["file_count"] + assert ( + actual == expected + ), f"File count mismatch {project_id} file count mismatch: {actual} != {expected}" + + +def verify_data_category_count(case): + actual_counts = collections.Counter(f["data_category"] for f in case["files"]) + expected_counts = { + s["data_category"]: s["file_count"] for s in case["summary"]["data_categories"] + } + + assert actual_counts == expected_counts + + # ====================================================================== # Fixtures @pytest.fixture -def index(init_indexd, pg_driver): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) +def index( + init_indexd: client.IndexClient, pg_driver: psqlgraph.PsqlGraphDriver +) -> Index: + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() index = builder.denormalize_all() @@ -48,26 +80,48 @@ def index(init_indexd, pg_driver): @pytest.fixture -def cached_builder(init_indexd, pg_driver): +def cached_builder( + init_indexd: client.IndexClient, pg_driver: psqlgraph.PsqlGraphDriver +) -> Iterator[ActiveGraphIndexBuilder]: with pg_driver.session_scope(): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) builder.cache_database() yield builder @pytest.fixture() -def builder(init_indexd, pg_driver): - return ActiveGraphIndexBuilder(pg_driver, init_indexd) +def builder( + init_indexd: client.IndexClient, pg_driver: psqlgraph.PsqlGraphDriver +) -> ActiveGraphIndexBuilder: + return ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) @pytest.fixture -def aligned_reads(index): - return [d for d in index.files if d["type"] == "aligned_reads"] +def aligned_reads(index: Index) -> Collection[dict]: + return tuple(d for d in index.files if d["type"] == "aligned_reads") @pytest.fixture -def simple_somatic_mutations(index): - return [d for d in index.files if d["type"] == "simple_somatic_mutation"] +def simple_somatic_mutations(index: Index) -> Collection[dict]: + return tuple(d for d in index.files if d["type"] == "simple_somatic_mutation") @pytest.fixture @@ -84,7 +138,7 @@ def diagnosis_annotations(generate_scenario): # Tests -def test_get_file_metadata_from_indexd(index): +def test_get_file_metadata_from_indexd(index: Index) -> None: """ Test that file metadata fields are taken from indexd (by checking that their value is not 'error' or -1 which are values in the graph) @@ -94,7 +148,9 @@ def test_get_file_metadata_from_indexd(index): validate_file_metadata(key, value) -def test_selective_caching(init_indexd, ro_pg_driver): +def test_selective_caching( + init_indexd: client.IndexClient, ro_pg_driver: psqlgraph.PsqlGraphDriver +) -> None: """ Tests that partial graph data caching is working in subset build scenario """ @@ -103,7 +159,11 @@ def test_selective_caching(init_indexd, ro_pg_driver): ro_pg_driver, init_indexd, build_projects=projects_subset, + index_prefix="", + build_awg=False, selective_caching=True, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), ) builder1.cache_database() @@ -113,7 +173,14 @@ def test_selective_caching(init_indexd, ro_pg_driver): assert built_projects == projects_subset builder2 = ActiveGraphIndexBuilder( - ro_pg_driver, init_indexd, selective_caching=True + ro_pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=True, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), ) builder2.cache_database() @@ -123,21 +190,29 @@ def test_selective_caching(init_indexd, ro_pg_driver): assert built_projects != all_projects -def test_awg_build(init_indexd, pg_driver): +def test_awg_build( + init_indexd: client.IndexClient, pg_driver: psqlgraph.PsqlGraphDriver +) -> None: """ Tests AWG build mode """ build_projects = {"TCGA-LUAD", "INTERNAL-AWG-ONE"} builder = ActiveGraphIndexBuilder( - pg_driver, init_indexd, build_awg=True, build_projects=build_projects + pg_driver, + init_indexd, + index_prefix="", + build_projects=build_projects, + build_awg=True, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), ) builder.cache_database() # Check that only AWG nodes were built - built_nodes = {} + built_nodes: dict[str, set] = collections.defaultdict(set) for node in builder.G.nodes(): - built_nodes.setdefault(node.label, set()) - built_nodes[node.label].update([node.node_id]) + built_nodes[node.label].add(node.node_id) assert built_nodes == { "case": {get_node_id("submitted-awg-case"), get_node_id("processed-awg-case")}, @@ -147,17 +222,26 @@ def test_awg_build(init_indexd, pg_driver): @pytest.mark.parametrize( - "gencode,expected_number", - [ - ["v22", 17], - ["v36", 18], - ], + ("gencode", "expected_number"), + ( + ("v22", 17), + ("v36", 18), + ), ) -def test_gencode_version(apply_gencode_to_indexd, pg_driver, gencode, expected_number): +def test_gencode_version( + apply_gencode_to_indexd: client.IndexClient, + pg_driver: psqlgraph.PsqlGraphDriver, + gencode: str, + expected_number: int, +) -> None: builder = ActiveGraphIndexBuilder( psqlgraph_driver=pg_driver, indexd_client=apply_gencode_to_indexd, + index_prefix="", build_projects={"TCGA-BRCA"}, + build_awg=False, + selective_caching=False, + versioned_files={}, allowed_gencode_versions=frozenset(["neutral", gencode]), ) builder.cache_database() @@ -172,21 +256,21 @@ def test_gencode_version(apply_gencode_to_indexd, pg_driver, gencode, expected_n @pytest.mark.parametrize( - "index_type,path", - [ + ("index_type", "path"), + ( ("cases", "[].clinical"), ("cases", "[].files[].file_state"), ("files", "[].file_state"), ("annotations", "[].creator"), - ], + ), ) -def test_path_is_absent(index, index_type, path): +def test_path_is_absent(index: Index, index_type: str, path: str) -> None: assert not jmespath.search(path, getattr(index, index_type)) @pytest.mark.parametrize( - "index_type,path,count", - [ + ("index_type", "path", "count"), + ( ("projects", "[].primary_site", 2), ("projects", "[].disease_type", 2), ("cases", "[].primary_site", 5), @@ -218,25 +302,25 @@ def test_path_is_absent(index, index_type, path): ("annotations", "[].project_id", 0), ("annotations", "[].annotation_id", 3), ("files", "[].associated_entities[].entity_type", N_FILES + 3), - ], + ), ) -def test_path_count(index, index_type, path, count): +def test_path_count(index: Index, index_type: str, path: str, count: int) -> None: results = jmespath.search(path, getattr(index, index_type)) assert len(results) == count @pytest.mark.parametrize( - "index_type, count", - [("annotations", 3), ("projects", 2), ("cases", 5), ("files", N_FILES)], + ("index_type", "count"), + (("annotations", 3), ("projects", 2), ("cases", 5), ("files", N_FILES)), ) -def test_basic_counts(index, index_type, count): +def test_basic_counts(index: Index, index_type: str, count: int) -> None: data = getattr(index, index_type) assert len(data) == count @pytest.mark.parametrize( - "index_type,path,count,expected", - [ + ("index_type", "path", "count", "expected"), + ( ( "projects", "[].name", @@ -363,9 +447,11 @@ def test_basic_counts(index, index_type, count): "structural_variation", }, ), - ], + ), ) -def test_path_value_set_equals(index, index_type, path, expected, count): +def test_path_value_set_equals( + index: Index, index_type: str, path: str, expected: Set[str], count: int +) -> None: results = jmespath.search(path, getattr(index, index_type)) actual = frozenset(results) assert actual == expected @@ -373,78 +459,94 @@ def test_path_value_set_equals(index, index_type, path, expected, count): @pytest.mark.parametrize( - "index_type,path,cls,node_ids", - [ + ("index_type", "path", "cls", "node_id"), + ( ( "files", "[].file_id", models.Aliquot, - [get_node_id("aliquot-derived-from-unreleased-sample")], + get_node_id("aliquot-derived-from-unreleased-sample"), ), ( "cases", "[].case_id", models.Case, - [get_node_id("released-case-in-unreleased-project")], + get_node_id("released-case-in-unreleased-project"), ), ( "cases", "[].samples[].sample_id", models.Sample, - [get_node_id("sample-unreleased")], + get_node_id("sample-unreleased"), ), ( "cases", "[].annotations[].annotation_id", models.Annotation, - [get_node_id("unreleased-annotation")], + get_node_id("unreleased-annotation"), ), - ], + ), ) def test_unreleased_nodes_not_indexed( - pg_driver, index, index_type, path, cls, node_ids -): + pg_driver: psqlgraph.PsqlGraphDriver, + index: Index, + index_type: str, + path: str, + cls: type[models.Node], + node_id: str, +) -> None: with pg_driver.session_scope(): - for node_id in node_ids: - node = pg_driver.nodes(cls).ids(node_id).one() - assert node.state in ["submitted", "released"] + node = pg_driver.nodes(cls).get(node_id) + assert node.state in ["submitted", "released"] results = frozenset(jmespath.search(path, getattr(index, index_type))) - assert len(results & frozenset(node_ids)) == 0 + assert node_id not in results @pytest.mark.parametrize( - "index_type,path,count,expected", - [ + ("index_type", "path", "count", "expected"), + ( ( "projects", "[].disease_type", 2, - {"Blood Vessel Tumors", "Adenomas and Adenocarcinomas"}, + frozenset(("Blood Vessel Tumors", "Adenomas and Adenocarcinomas")), ), - ("projects", "[].primary_site", 2, {"Breast", "Prostate gland", "Rectum"}), - ], + ( + "projects", + "[].primary_site", + 2, + frozenset(("Breast", "Prostate gland", "Rectum")), + ), + ), ) -def test_path_value_set_equals_set(index, index_type, path, expected, count): +def test_path_value_set_equals_set( + index: Index, index_type: str, path: str, expected: Set[str], count: int +) -> None: results = jmespath.search(path, getattr(index, index_type)) # reduce the dimensionality because we only really care about the # existing values here and the count - actual = reduce(operator.or_, map(lambda x: frozenset(x), results)) + actual = frozenset(itertools.chain.from_iterable(results)) assert actual == expected assert len(results) == count @pytest.mark.parametrize( - "node_cls", [(models.SubmittedAlignedReads), (models.SubmittedMethylationBetaValue)] + "node_cls", (models.SubmittedAlignedReads, models.SubmittedMethylationBetaValue) ) -def test_no_submitted_types(pg_driver, index, node_cls): +def test_no_submitted_types( + pg_driver: psqlgraph.PsqlGraphDriver, index: Index, node_cls: type[models.Node] +): with pg_driver.session_scope(): f_ids = {n.node_id for n in pg_driver.nodes(node_cls).all()} assert not [d for d in index.files if d["file_id"] in f_ids] -def test_aligned_reads_analysis_input_files(index, simple_somatic_mutations): +@pytest.mark.usefixtures("index") +def test_aligned_reads_analysis_input_files( + simple_somatic_mutations: Collection[dict], +) -> None: for doc in simple_somatic_mutations: assert doc["analysis"].get("input_files") assert len(doc["analysis"]["input_files"]) == 2 @@ -453,7 +555,8 @@ def test_aligned_reads_analysis_input_files(index, simple_somatic_mutations): assert f["data_format"] -def test_aligned_reads_analysis_read_group(index, aligned_reads): +@pytest.mark.usefixtures("index") +def test_aligned_reads_analysis_read_group(aligned_reads: Collection[dict]) -> None: for doc in aligned_reads: assert doc["analysis"].get("metadata") read_groups = doc["analysis"]["metadata"]["read_groups"] @@ -462,24 +565,19 @@ def test_aligned_reads_analysis_read_group(index, aligned_reads): assert rg["read_group_id"] -def test_project_file_counts(index, builder, monkeypatch): - monkeypatch.setattr(builder, "error", raise_test_error) +def test_project_file_counts(index: Index) -> None: for project in index.projects: - builder.validate_project_file_counts(project, index.files) + validate_project_file_counts(project, index.files) -def test_data_category_count(index, builder, monkeypatch): - monkeypatch.setattr(builder, "error", raise_test_error) +def test_data_category_count(index: Index) -> None: for case in index.cases: - builder.verify_data_category_count(case) + verify_data_category_count(case) -def test_case_summary_data_category_counts(index): +def test_case_summary_data_category_counts(index: Index) -> None: for case in index.cases: - actual_counts = {} - for f in case["files"]: - category = f["data_category"] - actual_counts[category] = actual_counts.get(category, 0) + 1 + actual_counts = collections.Counter(f["data_category"] for f in case["files"]) for entry in case["summary"]["data_categories"]: category, count = entry["data_category"], entry["file_count"] @@ -487,45 +585,39 @@ def test_case_summary_data_category_counts(index): assert actual_counts[category] == count, category -def test_case_summary_file_counts(index): +def test_case_summary_file_counts(index: Index) -> None: for case in index.cases: - actual_count = len( - [f for f in index.files if f["cases"][0]["case_id"] == case["case_id"]] + actual_count = more_itertools.ilen( + f for f in index.files if f["cases"][0]["case_id"] == case["case_id"] ) assert actual_count == case["summary"]["file_count"] -def test_get_file_read_groups(pg_driver, index): +def test_get_file_read_groups( + pg_driver: psqlgraph.PsqlGraphDriver, index: Index +) -> None: with pg_driver.session_scope(): - f_ids = {n.node_id for n in pg_driver.nodes(models.SubmittedAlignedReads).all()} - assert not [d for d in index.files if d["file_id"] in f_ids] - - -@pytest.mark.parametrize( - "cls,count", - [ - (models.AlignmentWorkflow, 2), - (models.SomaticMutationCallingWorkflow, 2), - ], -) -def test_get_analysis_read_groups(pg_driver, cached_builder, cls, count): - for workflow in pg_driver.nodes(cls).all(): - read_groups = list(cached_builder.get_analysis_read_groups(workflow)) - assert len(read_groups) == count - for read_group in read_groups: - assert read_group.label == "read_group" + f_ids = frozenset( + n.node_id for n in pg_driver.nodes(models.SubmittedAlignedReads).all() + ) + assert not any(d for d in index.files if d["file_id"] in f_ids) @pytest.mark.parametrize( - "cls,count", - [ + ("cls", "count"), + ( (models.AlignedReads, 1), (models.CopyNumberSegment, 1), (models.RunMetadata, 1), (models.ExperimentMetadata, 1), - ], + ), ) -def test_get_file_associated_entities(pg_driver, cached_builder, cls, count): +def test_get_file_associated_entities( + pg_driver: psqlgraph.PsqlGraphDriver, + cached_builder: ActiveGraphIndexBuilder, + cls: type[models.Node], + count: int, +) -> None: for node in pg_driver.nodes(cls).all(): if cached_builder.is_file_indexed(node): entities = list(cached_builder.get_file_associated_entities(node)) @@ -533,82 +625,98 @@ def test_get_file_associated_entities(pg_driver, cached_builder, cls, count): @pytest.mark.parametrize( - "cls,count", - [ + ("cls", "count"), + ( (models.BiospecimenSupplement, 0), (models.ClinicalSupplement, 0), - ], + ), scope="module", ) -def test_add_related_files(pg_driver, cached_builder, cls, count): +def test_add_related_files( + pg_driver: psqlgraph.PsqlGraphDriver, + cached_builder: ActiveGraphIndexBuilder, + cls: type[models.Node], + count: int, +) -> None: for node in pg_driver.nodes(cls).all(): if cached_builder.is_file_indexed(node): - doc = {} + doc: dict = {} cached_builder.add_related_files(node, doc) - assert len(doc.get("metadata_files", [])) == count + assert len(doc.get("metadata_files", ())) == count @pytest.mark.parametrize( - "cls,has_archive", - [ + ("cls", "has_archive"), + ( (models.BiospecimenSupplement, True), (models.ClinicalSupplement, True), (models.AlignedReads, False), (models.CopyNumberSegment, False), - ], + ), scope="module", ) -def test_add_archive(pg_driver, cached_builder, cls, has_archive): +def test_add_archive( + pg_driver: psqlgraph.PsqlGraphDriver, + cached_builder: ActiveGraphIndexBuilder, + cls: type[models.Node], + has_archive: bool, +) -> None: for node in pg_driver.nodes(cls).all(): if cached_builder.is_file_indexed(node): - doc = {} + doc: dict = {} cached_builder.add_archives(node, doc) assert ("archive" in doc) == has_archive -def test_aligned_reads_count(aligned_reads): +def test_aligned_reads_count(aligned_reads: Collection[dict]) -> None: assert len(aligned_reads) == 2 -def test_aligned_reads_associated_entities(index, aligned_reads): +def test_aligned_reads_associated_entities(aligned_reads: Collection[dict]) -> None: for f in aligned_reads: assert len(f["associated_entities"]) == 1 -def test_aligned_reads_ancestor_sample_types(index, aligned_reads): +def test_aligned_reads_ancestor_sample_types(aligned_reads: Collection[dict]) -> None: for f in aligned_reads: assert len(f["cases"]) == 1 assert len(f["cases"][0]["samples"]) == 1 -def test_no_duplicate_top_level_ids(index): +def test_no_duplicate_top_level_ids(index: Index) -> None: for case in index.cases: - aliquot_ids = case.get("aliquot_ids", []) + aliquot_ids = case.get("aliquot_ids", ()) assert len(aliquot_ids) == len(set(aliquot_ids)) -def test_somatic_aggregation_workflow_read_groups(index): - aggregated_somatic_mutations = [ +def test_somatic_aggregation_workflow_read_groups(index: Index): + aggregated_somatic_mutations = tuple( doc for doc in index.files if doc["data_type"] == "Aggregated Somatic Mutation" - ] + ) assert aggregated_somatic_mutations for asm in aggregated_somatic_mutations: - assert not asm["analysis"].get("metadata", {}).get("read_groups", []) + assert not asm["analysis"].get("metadata", {}).get("read_groups", ()) -def test_sample_analyte_indexed(index): +def test_sample_analyte_indexed(index: Index) -> None: """Tests to verify that aliquots under the subtree case.sample.analyte are indexed""" case_affected = None for case in index.cases: if case["submitter_id"] == "fake_submitter_2": case_affected = case break + + assert case_affected and "aliquot_ids" in case_affected assert len(case_affected["aliquot_ids"]) == 1 aliquot_ids = case_affected["aliquot_ids"] assert aliquot_ids[0] == get_node_id("tt-260-aliquot") -def test_inconsistent_slides_in_graph(pg_driver, init_indexd, inconsistent_slides): +@pytest.mark.usefixtures("inconsistent_slides") +def test_inconsistent_slides_in_graph( + pg_driver: psqlgraph.PsqlGraphDriver, + init_indexd: client.IndexClient, +) -> None: """ Make sure that regardless of the order in which we cache Slide node relations to cases, the caching still completes as expected. @@ -637,16 +745,34 @@ def nodes_labeled(self, labels): results = [n for n in results if n.label != "slide"] + slides return results - builderA = MyBuilderA(pg_driver, init_indexd) + builderA = MyBuilderA( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builderA.cache_database() - labeled = builderA.nodes_labeled(builderA.possible_associated_entities) + labeled = builderA.nodes_labeled(POSSIBLE_ASSOCIATED_ENTITIES) assert labeled and all(n.label == "slide" for n in labeled[:3]) - builderB = MyBuilderB(pg_driver, init_indexd) + builderB = MyBuilderB( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builderB.cache_database() - labeled = builderB.nodes_labeled(builderB.possible_associated_entities) + labeled = builderB.nodes_labeled(POSSIBLE_ASSOCIATED_ENTITIES) assert labeled and all(n.label == "slide" for n in labeled[-3:]) # Comparing that 2 maps are the same @@ -680,15 +806,28 @@ def nodes_labeled(self, labels): @pytest.mark.usefixtures("diagnosis_annotations") -def test_diagnosis_annotation_has_extra_data(pg_driver, init_indexd): - builder = ActiveGraphIndexBuilder(pg_driver, init_indexd) +def test_diagnosis_annotation_has_extra_data( + pg_driver: psqlgraph.PsqlGraphDriver, init_indexd: client.IndexClient +) -> None: + builder = ActiveGraphIndexBuilder( + pg_driver, + init_indexd, + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset({"neutral", "v36"}), + ) with pg_driver.session_scope(): builder.cache_database() cases, _, _, _ = builder.denormalize_all() - target_case = [c for c in cases if c["submitter_id"] == "da_case_1"][0] + target_case = more_itertools.first( + c for c in cases if c["submitter_id"] == "da_case_1" + ) assert len(target_case["diagnoses"]) == 1 assert len(target_case["diagnoses"][0]["annotations"]) == 1 diff --git a/tests/integration/test_test_data.py b/tests/integration/test_test_data.py index 92a4e3e6..b5cd7ace 100644 --- a/tests/integration/test_test_data.py +++ b/tests/integration/test_test_data.py @@ -1,16 +1,15 @@ import pytest -from esbuild.graph.common.builder import GraphIndexBuilder -from tests.integration.data import INDEXD, NODES - -DATA_FILE_CATEGORIES = GraphIndexBuilder.data_file_categories -DATA_FILE_INDEXD_FIELDS = GraphIndexBuilder.data_file_indexd_fields +from esbuild.graph.common import builder +from tests.integration import data @pytest.fixture(scope="session") def file_nodes(): return [ - nd for nd in NODES if nd._dictionary.get("category") in DATA_FILE_CATEGORIES + nd + for nd in data.NODES + if nd._dictionary.get("category") in data.DATA_FILE_CATEGORIES ] @@ -19,7 +18,7 @@ def test_file_nodes_patched(file_nodes): Checks that all file nodes' indexd fields are patched with error values """ for node in file_nodes: - for key in DATA_FILE_INDEXD_FIELDS: + for key in builder.DATA_FILE_INDEXD_FIELDS: key_value = getattr(node, key, None) if not key_value: continue @@ -39,14 +38,16 @@ def test_indexd_data(init_indexd, file_nodes): Test that indexd data corresponds to graph data """ # Check that indexd data count == number of file nodes - assert len(INDEXD) == len(file_nodes) + assert len(data.INDEXD) == len(file_nodes) # Check that indexd uuids == file nodes uuids - assert {r["node_id"] for r in INDEXD} == {getattr(n, "node_id") for n in file_nodes} + assert {r["node_id"] for r in data.INDEXD} == { + getattr(n, "node_id") for n in file_nodes + } # Check that all DATA_FILE_INDEXD_FIELDS are in all indexd records for node in file_nodes: record = init_indexd.get(node.node_id).to_json() - for key in DATA_FILE_INDEXD_FIELDS: + for key in builder.DATA_FILE_INDEXD_FIELDS: if key not in record and key in node.to_json(): assert key in record["metadata"] diff --git a/tests/integration/test_utils.py b/tests/integration/test_utils.py index c0c7599f..8b7567da 100644 --- a/tests/integration/test_utils.py +++ b/tests/integration/test_utils.py @@ -1,14 +1,13 @@ import pytest -from esbuild.graph.common.builder import get_namespaced_uuid, get_uuid_namespace -from esbuild.utils import ReleaseHelper, get_index_names +from esbuild import utils +from esbuild.graph.common import builder from tests.integration import es_data -from tests.integration.data import DATA_FILE_INDEXD_FIELDS def test_get_projects_list(test_index_data): es, index_name = test_index_data - helper = ReleaseHelper(es, "foo") + helper = utils.ReleaseHelper(es, "foo") projects = {d["project_id"] for d in es_data.DOCS["project"]} @@ -40,7 +39,7 @@ def validate_file_metadata(key, value): validate_file_metadata(subkey, output_files) return - if key in DATA_FILE_INDEXD_FIELDS: + if key in builder.DATA_FILE_INDEXD_FIELDS: error_msg = f'"{key}" is loaded from graph instead of indexd' if isinstance(value, str): assert value != "error", error_msg @@ -58,7 +57,7 @@ def validate_file_metadata(key, value): def test_projects_deleted(es_after_deletion): es, index_name, projects_before, deleted_projects = es_after_deletion - helper = ReleaseHelper(es, audit_index="build_metadata_test") + helper = utils.ReleaseHelper(es, audit_index="build_metadata_test") expected_projects = {p for p in projects_before if p not in deleted_projects} assert helper.get_project_ids(index_name) == expected_projects @@ -74,7 +73,7 @@ def test_delete_project_docs(es_after_deletion): "annotation": "project.project_id", } - index_names = get_index_names(index_prefix, path_to_id.keys()) + index_names = utils.get_index_names(index_prefix, path_to_id.keys()) # Check files projects = set() @@ -111,7 +110,7 @@ def get_value_at_path(tree, path): ], ) def test_get_uuid_namespace(namespace, expectation): - u = get_uuid_namespace(namespace) + u = builder.get_uuid_namespace(namespace) assert expectation == str(u) @@ -136,5 +135,5 @@ def test_get_uuid_namespace(namespace, expectation): ], ) def test_get_namespaced_uuid(namespace, seed, expectation): - u = get_namespaced_uuid(namespace, seed) + u = builder.get_namespaced_uuid(namespace, seed) assert expectation == str(u) diff --git a/tests/integration/versioning/test_versioning.py b/tests/integration/versioning/test_versioning.py index 6613b05a..90b2d54e 100644 --- a/tests/integration/versioning/test_versioning.py +++ b/tests/integration/versioning/test_versioning.py @@ -102,7 +102,7 @@ def assert_aligned_reads_documents(indexd, pg_driver, ar_node, es_response): ar_props = ( ar_ts.new_props if ar_doc.did == ar_node.node_id else ar_ts.old_props ) # noqa - assert_inclusion(ar_props, ar_hit, INDEXD_METADATA_FIELDS + ["updated_datetime"]) + assert_inclusion(ar_props, ar_hit, (*INDEXD_METADATA_FIELDS, "updated_datetime")) ari_props = ( ari_ts.new_props if ari_doc.did == ari_node.node_id else ari_ts.old_props @@ -110,7 +110,7 @@ def assert_aligned_reads_documents(indexd, pg_driver, ar_node, es_response): assert_inclusion( ari_props, ar_hit["index_files"][0], - INDEXD_METADATA_FIELDS + ["updated_datetime"], + (*INDEXD_METADATA_FIELDS, "updated_datetime"), ) @@ -137,7 +137,7 @@ def test_esbuild_versioning( es_expectations = versioned_reads_expectations nodes, versioned_nodes, versioned_docs, params = versioned_reads_setup - source = INDEXD_METADATA_FIELDS + ["index_files"] + source = (*INDEXD_METADATA_FIELDS, "index_files") res = es.search( index="gdc_es_test_file", body={ diff --git a/tests/unit/graph/common/test_builder.py b/tests/unit/graph/common/test_builder.py index 66d2bb64..efc67b07 100644 --- a/tests/unit/graph/common/test_builder.py +++ b/tests/unit/graph/common/test_builder.py @@ -8,26 +8,19 @@ from esbuild.graph.common import builder, mappings -class TestIndexBuilder(builder.GraphIndexBuilder): - def __init__( - self, - psqlgraph_driver: psqlgraph.PsqlGraphDriver, - indexd_client: client.IndexClient, - index_prefix: Optional[str] = "", - case_to_file_paths=(), - file_labels=frozenset(()), - **kwargs: Any - ) -> None: - self.case_to_file_paths = case_to_file_paths - self.file_labels = file_labels - - super().__init__(psqlgraph_driver, indexd_client, index_prefix, **kwargs) - - def test__denormalize_annotations__no_annotations() -> None: graph = mock.MagicMock() annotations = () - index_builder = TestIndexBuilder(graph, mock.MagicMock(), "") + index_builder = builder.GraphIndexBuilder( + psqlgraph_driver=graph, + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ) result = index_builder.denormalize_annotations(annotations=annotations) @@ -47,7 +40,16 @@ def test__denormalize_annotations__annotated_case_node() -> None: graph = mock.MagicMock() graph.edges.return_value = graph graph.filter.return_value = (annotation_edge,) - index_builder = TestIndexBuilder(graph, mock.MagicMock(), "") + index_builder = builder.GraphIndexBuilder( + psqlgraph_driver=graph, + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ) result = index_builder.denormalize_annotations(annotations=(annotation,)) @@ -95,7 +97,16 @@ def test__denormalize_annotations__annotated_workflow_with_linked_case() -> None graph = mock.MagicMock() graph.edges.return_value = graph graph.filter.return_value = (annotation_edge,) - index_builder = TestIndexBuilder(graph, mock.MagicMock(), "") + index_builder = builder.GraphIndexBuilder( + psqlgraph_driver=graph, + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ) result = index_builder.denormalize_annotations(annotations=(annotation,)) result_doc = more_itertools.one(result) @@ -139,7 +150,16 @@ def test__denormalize_annotations__annotated_workflow_without_linked_case() -> N graph = mock.MagicMock() graph.edges.return_value = graph graph.filter.return_value = (annotation_edge,) - index_builder = TestIndexBuilder(graph, mock.MagicMock(), "") + index_builder = builder.GraphIndexBuilder( + psqlgraph_driver=graph, + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ) result = index_builder.denormalize_annotations(annotations=(annotation,)) result_doc = more_itertools.one(result) diff --git a/tests/unit/test_graph_active_bulder.py b/tests/unit/test_graph_active_bulder.py index b151745d..97d5a41c 100644 --- a/tests/unit/test_graph_active_bulder.py +++ b/tests/unit/test_graph_active_bulder.py @@ -1,3 +1,6 @@ +from collections.abc import Iterable, Sequence +from unittest import mock + import pytest from gdcdatamodel2 import models @@ -6,77 +9,101 @@ @pytest.mark.parametrize( "prefix", - [ - ["sample", "aliquot", "read_group"], - ["sample", "portion", "analyte", "aliquot", "read_group"], - ], + ( + ("sample", "aliquot", "read_group"), + ("sample", "portion", "analyte", "aliquot", "read_group"), + ), ) -def test_get_case_to_file_paths_contains_expected_path(prefix): +def test_get_case_to_file_paths_contains_expected_path(prefix: Sequence[str]) -> None: assert ( - prefix - + [ - "submitted_aligned_reads", - "aligned_reads", - "somatic_mutation_calling_workflow", - "simple_somatic_mutation", - ] - in builder.ActiveGraphIndexBuilder.case_to_file_paths - ) + *prefix, + "submitted_aligned_reads", + "aligned_reads", + "somatic_mutation_calling_workflow", + "simple_somatic_mutation", + ) in builder.ActiveGraphIndexBuilder( + psqlgraph_driver=mock.MagicMock(), + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ).case_to_file_paths @pytest.mark.parametrize( ("a", "b", "expected"), ( ( - [["a", "b"], ["-", "#"]], - [[0, 1], [2, 3], [4, 5, 6, 7]], - [ - ["a", "b", 0, 1], - ["a", "b", 2, 3], - ["a", "b", 4, 5, 6, 7], - ["-", "#", 0, 1], - ["-", "#", 2, 3], - ["-", "#", 4, 5, 6, 7], - ], + (("a", "b"), ("-", "#")), + ((0, 1), (2, 3), (4, 5, 6, 7)), + ( + ("a", "b", 0, 1), + ("a", "b", 2, 3), + ("a", "b", 4, 5, 6, 7), + ("-", "#", 0, 1), + ("-", "#", 2, 3), + ("-", "#", 4, 5, 6, 7), + ), ), ), ) -def test_list_product(a, b, expected): - assert builder.list_product(a, b) == expected +def test_list_product( + a: Iterable[Sequence[str]], + b: Iterable[Sequence[str]], + expected: Iterable[Sequence[str]], +) -> None: + assert builder._path_product(a, b) == expected @pytest.mark.parametrize( ("node", "expected"), ( - (models.RnaExpressionWorkflow, ["gene_expression"]), + (models.RnaExpressionWorkflow, ("gene_expression",)), ( models.ReadGroup, - [ + ( "submitted_aligned_reads", "aligned_reads", "somatic_mutation_calling_workflow", "simple_somatic_mutation", - ], + ), ), ), ) -def test_subtree_paths_to_file_subset(node, expected): - assert expected in builder.subtree_paths_to_file(node) +def test_subtree_paths_to_file_subset( + node: type[models.Node], expected: Sequence[str] +) -> None: + assert expected in builder._subtree_paths_to_file(node) -def test_subtree_paths_to_file_expecting_empty(): - assert builder.subtree_paths_to_file(models.Annotation) == [] +def test_subtree_paths_to_file_expecting_empty() -> None: + assert tuple(builder._subtree_paths_to_file(models.Annotation)) == () @pytest.mark.parametrize( "path", ( - ["case", "sample", "portion", "analyte", "aliquot"], - ["case", "sample", "aliquot"], + ("case", "sample", "portion", "analyte", "aliquot"), + ("case", "sample", "aliquot"), ), ) -def test_case_to_file_paths_is_absent(path): - assert path not in builder.ActiveGraphIndexBuilder.case_to_file_paths +def test_case_to_file_paths_is_absent(path: tuple[str, ...]): + assert ( + path + not in builder.ActiveGraphIndexBuilder( + psqlgraph_driver=mock.MagicMock(), + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ).case_to_file_paths + ) @pytest.mark.parametrize( @@ -86,19 +113,17 @@ def test_case_to_file_paths_is_absent(path): "sample.portion.analyte.aliquot.read_group.submitted_unaligned_reads.alignment_workflow.aligned_reads", ), ) -def test_get_case_to_file_path_is_present(path): - assert path.split(".") in builder.ActiveGraphIndexBuilder.case_to_file_paths - - -@pytest.mark.parametrize( - ("label", "path"), - ( - ("submitted_aligned_reads", ["read_group"]), - ( - "aligned_reads", - ["alignment_workflow", "submitted_aligned_reads", "read_group"], - ), - ), -) -def test_file_to_read_group_paths(label, path): - assert path in builder.ActiveGraphIndexBuilder.file_to_read_group_paths[label] +def test_get_case_to_file_path_is_present(path: str) -> None: + assert ( + tuple(path.split(".")) + in builder.ActiveGraphIndexBuilder( + psqlgraph_driver=mock.MagicMock(), + indexd_client=mock.MagicMock(), + index_prefix="", + build_projects=(), + build_awg=False, + selective_caching=False, + versioned_files={}, + allowed_gencode_versions=frozenset(("neutral", "v36")), + ).case_to_file_paths + )