diff --git a/.github/workflows/sdkless_test.yml b/.github/workflows/sdkless_test.yml new file mode 100644 index 0000000..2500d25 --- /dev/null +++ b/.github/workflows/sdkless_test.yml @@ -0,0 +1,34 @@ +name: SDK-less unit tests + +on: + pull_request: + types: + - opened + - synchronize + - ready_for_review + +jobs: + sdkless_test: + runs-on: ubuntu-latest + + steps: + - name: Checkout GitHub repo + uses: actions/checkout@v4 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Cache pip install + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-gha.txt') }} + + - name: Install pip packages + run: pip install -r requirements-gha.txt + + - name: Run sdkless tests + run: make test-sdkless + diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index bc5b7ea..a7605b5 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,10 @@ # kb_gtdbtk release notes ========================= +1.4.1 +_____ +* Fixed report generation and file post-processing issues + 1.0.1 _____ * Reverting to most recent prior release to address the recurring bug from the new release 1.4.0 diff --git a/kbase.yml b/kbase.yml index 9c73688..19c3799 100644 --- a/kbase.yml +++ b/kbase.yml @@ -8,10 +8,10 @@ service-language: python module-version: - 1.4.0 + 1.4.1 owners: - [dylan, psdehal, donovan_parks, aaronmussig, pchaumeil, gaprice, zcrockett, tgu2] + [dylan, psdehal, donovan_parks, aaronmussig, pchaumeil, gaprice, zcrockett, tgu2, wjriehl] data-version: 0.1.1 diff --git a/lib/kb_gtdbtk/core/gtdbtk_runner.py b/lib/kb_gtdbtk/core/gtdbtk_runner.py index 1e49db0..3d3f6dd 100644 --- a/lib/kb_gtdbtk/core/gtdbtk_runner.py +++ b/lib/kb_gtdbtk/core/gtdbtk_runner.py @@ -5,10 +5,10 @@ import logging import json import os +import shutil import pandas as pd import tempfile -from datetime import datetime from pathlib import Path from shutil import ( copyfile, @@ -20,7 +20,8 @@ Dict, List, Optional, - Tuple + Tuple, + Union ) from kb_gtdbtk.core.string_util import now_ISOish @@ -99,6 +100,8 @@ def run_gtdbtk( os.environ['GTDBTK_DATA_PATH'] = str(data_root_dir / f"r{db_ver}") # set output dirs + # temp_output = main gtdbtk output directory + # temp_trees_output = only used if a second gtdbtk run is done with --skip_ani_screen temp_output = temp_dir / 'output' / timestamp temp_trees_output = temp_dir / 'output_trees' / timestamp temp_output.mkdir(parents=True, exist_ok=True) @@ -146,9 +149,16 @@ def run_gtdbtk( return _process_output_files(temp_output, temp_trees_output, output_dir, id_to_name) -# _all_ids_in_trees () -# def _all_ids_in_trees(temp_output: Path, id_to_name: Dict[str, str]) -> bool: + """ + Looks at the primary gtdbtk output file - either gtdbtk.ar53.summary.tsv or + gtdbtk.bac120.summary.tsv (under the temp_output directory). These files have + 20 columns. First is the user_genome id, and 13th is the taxonomy of the tree it was + placed in. This checks the following: + 1. if all user_genomes exist in the output. + 2. if they're all mapped to trees - that pplacer_taxonomy is not "N/A" + Returns True if all genomes submitted appear with trees, False otherwise. + """ all_ids_found = True ids_found = dict() @@ -173,28 +183,63 @@ def _all_ids_in_trees(temp_output: Path, id_to_name: Dict[str, str]) -> bool: return all_ids_found -# _process_output_files() -# def _process_output_files( temp_output: Path, temp_trees_output: Path, out_dir: Path, id_to_name: Dict[str, str]) -> Tuple[dict, dict]: + """ + Process GTDB-tk output files and consolidate results into a structured format. + + This function handles the post-processing of GTDB-tk classification workflow output. + It consolidates results from both a primary run and an optional secondary run + (performed with --skip_ani_screen), merges summary data, converts TSV files to JSON + format, and remaps internal sequence identifiers to their original assembly names. + + Key operations: + 1. Copies the entire temp_output directory structure to the output directory + 2. Saves the mapping between internal IDs and original assembly names + 3. Copies tree files, preferring those from the secondary run if available + 4. Merges summary TSV files from both runs, with secondary run data taking + precedence for N/A fields + 5. Converts summary TSV files to JSON format for web UI consumption + 6. Remaps internal IDs back to original assembly names in output data + 7. Filters out blank/null fields by replacing them with '-' for consistency + + :param temp_output: Path to the primary GTDB-tk output directory. Contains + results from the initial classify_wf run with all markers and ANI screening. + Expected subdirectories: 'classify' and 'identify' + :param temp_trees_output: Path to the secondary GTDB-tk output directory. Contains + results from the secondary classify_wf run (if performed) with --skip_ani_screen + flag. May be empty if all sequences were placed in trees during the primary run. + Expected subdirectories: 'classify' and 'identify' + :param out_dir: Path to the output directory where final results will be written. + Should be an existing, writable directory. Output includes: + - 'runtime_output/': Copy of temp_output directory structure + - 'id_to_name.map': Tab-separated file mapping internal IDs to assembly names + - Tree files (*.tree): Phylogenetic trees from both runs + - Summary TSV files: Classification and marker summary data + - Summary JSON files: JSON-converted versions of summary TSV files + :param id_to_name: Mapping from internal sequence identifiers (e.g., 'id0', 'id1') + to original assembly names. Used to remap internal IDs in output data back to + user-supplied assembly names. + + :return: A tuple of (classification, summary_tables) where: + - classification (dict): Mapping from assembly name to classification string. + Keys are original assembly names (from id_to_name values). + Values are classification strings from the 'classification' field in summary files. + - summary_tables (dict): Mapping from filename to parsed JSON data. + Keys are TSV filenames (e.g., 'gtdbtk.bac120.summary.tsv'). + Values are dictionaries with a 'data' key containing a list of record objects + with assembly names and classifications. + """ classification = dict() summary_tables = dict() # copy over all created output - """ - for file_ in os.listdir (temp_output): - tmppath = temp_output / file_ - if not tmppath.is_file(): - continue - path = out_dir / file_ - copyfile(tmppath, path) - """ sub_out_dir = Path(out_dir / 'runtime_output') - if os.path.isdir(sub_out_dir): # only occurs during unit tests + if os.path.isdir(sub_out_dir): # should only occur during unit tests rmtree(sub_out_dir) copytree(temp_output, sub_out_dir, symlinks=True) @@ -218,6 +263,9 @@ def _process_output_files( 'gtdbtk.bac120.markers_summary.tsv', 'gtdbtk.bac120.tree.mapping.tsv' ] + # These are the output files we care about - they're either in the classify/ or identify/ + # subdirectories. We'll update these below based on file structure (and whether or not + # there was a --skip_ani_screen run) file_folder = { 'gtdbtk.ar53.summary.tsv': 'classify', 'gtdbtk.bac120.summary.tsv': 'classify', @@ -239,6 +287,7 @@ def _process_output_files( # copy all tree files to the output directory # these may be in the temp_trees_output or the temp_output directory + # prefer the temp_trees_output directory for file_ in tree_files + bb_tree_file + extra_bac_tree_files: treepath = temp_trees_output / file_folder[file_] / file_ tmppath = temp_output / file_folder[file_] / file_ @@ -252,52 +301,8 @@ def _process_output_files( for file_ in base_files: treepath = temp_trees_output / file_folder[file_] / file_ tmppath = temp_output / file_folder[file_] / file_ - path = out_dir / file_ - found_file = False - num_cols = 0 - - id_order = [] - tmp_buf = dict() - # TODO: decompose, avoid duplications, overlap - num_tmp_cols = 0 - num_tree_cols = 0 - if tmppath.is_file(): - found_file = True - with open(tmppath, 'r') as tmppath_h: - for info_line in tmppath_h: - row = info_line.rstrip().split("\t") - tmp_buf[row[0]] = row - id_order.append(row[0]) - num_tmp_cols = max(len(row), num_tmp_cols) - tree_buf = dict() - if treepath.is_file(): - found_file = True - id_order = [] - with open (treepath, 'r') as treepath_h: - for info_line in treepath_h: - row = info_line.rstrip().split("\t") - tree_buf[row[0]] = row - id_order.append(row[0]) - num_tree_cols = max(len(row), num_tree_cols) - - num_cols = max(num_tmp_cols, num_tree_cols) - - if not found_file: - continue - out_buf = [] - for qid in id_order: - row = ["N/A"] * num_cols - if qid in tmp_buf: - row = tmp_buf[qid] - if qid in tree_buf: - for field_i, _ in enumerate(tree_buf[qid]): - if row[field_i] == 'N/A': - row[field_i] = tree_buf[qid][field_i] - out_buf.append("\t".join(row)) - - # write merged summaries - with open(path, 'w') as summary_h: - summary_h.write("\n".join(out_buf)+"\n") + out_path = out_dir / file_ + _merge_summary_tsv_files(tmppath, treepath, out_path) # load results for file_ in base_files: @@ -348,3 +353,89 @@ def _process_output_files( summary_tables[file_] = sj return (classification, summary_tables) + +def _merge_summary_tsv_files(std_path: Path, tree_path: Path, output_path: Path): + """ + Merges two TSV files from separate runs of GTDBtk. + The std_path represents the standard run, using a mash DB. + The tree_path is from a run using --skip_ani_screen, which generates file in a different path. + These two TSV files have slightly different outputs, with data in tree_path being generally more + informative. So fields for each genome in the summary file should have tree_path override any N/A values + found in std_path. E.g.: + + std_path: + user_genome field1 field2 field3 + some_genome N/A N/A X + + tree_path: + user_genome field1 field2 field3 + some_genome Y N/A X + + merged: + user_genome field1 field2 field3 + some_genome Y N/A X + """ + + std_file_exists = std_path.is_file() + tree_file_exists = tree_path.is_file() + if not std_file_exists and not tree_file_exists: + return + if std_file_exists and not tree_file_exists: + shutil.copy(std_path, output_path) + return + if tree_file_exists and not std_file_exists: + shutil.copy(tree_path, output_path) + return + std_file_data = _load_summary_tsv_file(std_path) + tree_file_data = _load_summary_tsv_file(tree_path) + if std_file_data["header"] != tree_file_data["header"]: + logging.warning(f"While merging summary files: header of {std_path} (from using a mash DB) does not match {tree_path} (skipping the ANI screen). These cannot be merged, saving only the ANI-skipped file.") + shutil.copy(tree_path, output_path) + return + out_buf = ["\t".join(std_file_data["header"])] + num_cols = len(std_file_data["header"]) + for qid in std_file_data["id_order"]: + row = ["N/A"] * num_cols + if qid in std_file_data["data"]: + row = std_file_data["data"][qid] + if qid in tree_file_data["data"]: + for field_i, value in enumerate(tree_file_data["data"][qid]): + if row[field_i] == "N/A": + row[field_i] = value + out_buf.append("\t".join(row)) + + # write merged summaries + with open(output_path, "w") as summary_file: + summary_file.write("\n".join(out_buf)+"\n") + + +def _load_summary_tsv_file(filepath: Path) -> Dict[str, Union[List, Dict]]: + """ + Loads a TSV file, includes the header (assumes that each summary file from GTDBtk has + a header) and all lines. + Summary files all have a unique identifier (generally the genome or assembly name) + as the first element of each line. This, then, creates the following structure: + { + "header": [], + "data": {"id": [row]}, + "id_order": [] + } + """ + header = [] + data = {} + id_order = [] + with open(filepath, "r") as infile: + for line_num, line in enumerate(infile): + row = line.rstrip("\n\r").split("\t") + if line_num == 0: + header = row + else: + identifier = row[0] + data[identifier] = row + id_order.append(identifier) + + return { + "header": header, + "data": data, + "id_order": id_order + } diff --git a/lib/kb_gtdbtk/kb_gtdbtkImpl.py b/lib/kb_gtdbtk/kb_gtdbtkImpl.py index c453ddb..0144cb5 100644 --- a/lib/kb_gtdbtk/kb_gtdbtkImpl.py +++ b/lib/kb_gtdbtk/kb_gtdbtkImpl.py @@ -46,7 +46,7 @@ class kb_gtdbtk: ### log() # def log(self, target, message): - message = '['+self.now_ISOish()+'] '+message + message = '['+now_ISOish()+'] '+message if target is not None: target.append(message) print(message) diff --git a/requirements-gha.txt b/requirements-gha.txt new file mode 100644 index 0000000..e097623 --- /dev/null +++ b/requirements-gha.txt @@ -0,0 +1,7 @@ +flake8 +jsonrpcbase +mypy +pandas +pytest +pytest-cov +requests diff --git a/test/core/gtdbtk_runner_test.py b/test/core/gtdbtk_runner_test.py index 6129ad2..cfd108b 100644 --- a/test/core/gtdbtk_runner_test.py +++ b/test/core/gtdbtk_runner_test.py @@ -7,7 +7,12 @@ from pathlib import Path -from kb_gtdbtk.core.gtdbtk_runner import run_gtdbtk +from kb_gtdbtk.core.gtdbtk_runner import ( + run_gtdbtk, + _load_summary_tsv_file, + _merge_summary_tsv_files, + _process_output_files +) logging.basicConfig(format='%(created)s %(levelname)s: %(message)s', level=logging.INFO) @@ -166,3 +171,320 @@ def test_gtdbtk_run_fail_no_default_refdata(tmp_path): 0, 16, ) + + +class TestLoadSummaryTsvFile: + """Tests for _load_summary_tsv_file function""" + + def test_load_summary_tsv_basic(self, tmp_path): + """Test loading a basic TSV file with header and data rows""" + tsv_file = tmp_path / "test.tsv" + tsv_file.write_text("user_genome\tfield1\tfield2\n" + "genome1\tvalue1\tvalue2\n" + "genome2\tvalue3\tvalue4\n") + + result = _load_summary_tsv_file(tsv_file) + + assert result["header"] == ["user_genome", "field1", "field2"] + assert result["data"]["genome1"] == ["genome1", "value1", "value2"] + assert result["data"]["genome2"] == ["genome2", "value3", "value4"] + assert result["id_order"] == ["genome1", "genome2"] + + def test_load_summary_tsv_with_na_values(self, tmp_path): + """Test loading TSV with N/A values""" + tsv_file = tmp_path / "test.tsv" + tsv_file.write_text("user_genome\tfield1\tfield2\n" + "genome1\tN/A\tvalue2\n" + "genome2\tvalue3\tN/A\n") + + result = _load_summary_tsv_file(tsv_file) + + assert result["data"]["genome1"][1] == "N/A" + assert result["data"]["genome2"][2] == "N/A" + + def test_load_summary_tsv_preserves_trailing_tabs(self, tmp_path): + """Test that trailing empty fields are preserved""" + tsv_file = tmp_path / "test.tsv" + tsv_file.write_text("user_genome\tfield1\tfield2\tfield3\tfield4\n" + "genome1\tvalue1\tvalue2\t\t\n") + + result = _load_summary_tsv_file(tsv_file) + + # The last element should be empty string, not removed + assert len(result["data"]["genome1"]) == 5 + assert result["data"]["genome1"] == [ + "genome1", "value1", "value2", "", "" + ] + +class TestMergeSummaryTsvFiles: + """Tests for _merge_summary_tsv_files function""" + + def test_merge_both_files_exist(self, tmp_path): + """Test merging when both std and tree files exist""" + std_file = tmp_path / "std.tsv" + tree_file = tmp_path / "tree.tsv" + out_file = tmp_path / "merged.tsv" + + # Standard file with some N/A values + std_file.write_text("user_genome\tfield1\tfield2\tfield3\n" + "genome1\tN/A\tvalue2\tvalue3\n" + "genome2\tvalue1\tN/A\tvalue3\n") + + # Tree file with values where std has N/A + tree_file.write_text("user_genome\tfield1\tfield2\tfield3\n" + "genome1\tvalue1_tree\tN/A\tvalue3\n" + "genome2\tvalue1_tree\tvalue2_tree\tN/A\n") + + _merge_summary_tsv_files(std_file, tree_file, out_file) + + result = _load_summary_tsv_file(out_file) + + # Tree values should override N/A in std + assert result["data"]["genome1"][1] == "value1_tree" # Was N/A in std + assert result["data"]["genome1"][2] == "value2" # From std, tree has N/A + assert result["data"]["genome2"][1] == "value1" # From std + assert result["data"]["genome2"][2] == "value2_tree" # Was N/A in std + + def test_merge_both_files_exist_whitespace(self, tmp_path): + """Test merging when both std and tree files exist""" + std_file = tmp_path / "std.tsv" + tree_file = tmp_path / "tree.tsv" + out_file = tmp_path / "merged.tsv" + + # Standard file with some N/A values + std_file.write_text("user_genome\tfield1\tfield2\tfield3\tfield4\n" + "genome1\tN/A\tvalue2\tvalue3\t\n" + "genome2\tvalue1\tN/A\tvalue3\tvalue4\n") + + # Tree file with values where std has N/A + tree_file.write_text("user_genome\tfield1\tfield2\tfield3\tfield4\n" + "genome1\tvalue1_tree\tN/A\tvalue3\t\n" + "genome2\tvalue1_tree\tvalue2_tree\tN/A\t\n") + + _merge_summary_tsv_files(std_file, tree_file, out_file) + + result = _load_summary_tsv_file(out_file) + + # Tree values should override N/A in std + assert result["data"]["genome1"] == [ + "genome1", "value1_tree", "value2", "value3", "" + ] + assert result["data"]["genome2"] == [ + "genome2", "value1", "value2_tree", "value3", "value4" + ] + + def test_merge_only_std_file(self, tmp_path): + """Test when only standard file exists""" + std_file = tmp_path / "std.tsv" + tree_file = tmp_path / "tree.tsv" + out_file = tmp_path / "merged.tsv" + + std_file.write_text("user_genome\tfield1\tfield2\n" + "genome1\tvalue1\tvalue2\n") + + _merge_summary_tsv_files(std_file, tree_file, out_file) + + assert out_file.read_text() == std_file.read_text() + + def test_merge_only_tree_file(self, tmp_path): + """Test when only tree file exists""" + std_file = tmp_path / "std.tsv" + tree_file = tmp_path / "tree.tsv" + out_file = tmp_path / "merged.tsv" + + tree_file.write_text("user_genome\tfield1\tfield2\n" + "genome1\tvalue1\tvalue2\n") + + _merge_summary_tsv_files(std_file, tree_file, out_file) + + assert out_file.read_text() == tree_file.read_text() + + def test_merge_no_files(self, tmp_path): + """Test when neither file exists""" + std_file = tmp_path / "std.tsv" + tree_file = tmp_path / "tree.tsv" + out_file = tmp_path / "merged.tsv" + + _merge_summary_tsv_files(std_file, tree_file, out_file) + + # Output file should not be created + assert not out_file.exists() + + def test_merge_mismatched_headers(self, tmp_path): + """Test merging files with different headers""" + std_file = tmp_path / "std.tsv" + tree_file = tmp_path / "tree.tsv" + out_file = tmp_path / "merged.tsv" + + std_file.write_text("user_genome\tfield1\tfield2\n" + "genome1\tvalue1\tvalue2\n") + + tree_file.write_text("user_genome\tfield1\tfield3\n" + "genome1\tvalue1_tree\tvalue3\n") + + _merge_summary_tsv_files(std_file, tree_file, out_file) + + # Should copy only the tree file when headers don't match + assert out_file.read_text() == tree_file.read_text() + + +class TestProcessOutputFiles: + """Tests for _process_output_files function""" + + def test_process_output_creates_json_files(self, tmp_path): + """Test that JSON versions of TSV files are created""" + temp_output = tmp_path / "temp_output" + temp_output.mkdir() + (temp_output / "classify").mkdir() + (temp_output / "identify").mkdir() + + temp_trees_output = tmp_path / "temp_trees_output" + temp_trees_output.mkdir() + + out_dir = tmp_path / "out" + out_dir.mkdir() + + # Create a summary TSV file + summary_file = temp_output / "classify" / "gtdbtk.bac120.summary.tsv" + summary_file.write_text("user_genome\tclassification\tfield2\n" + "id0\tclass_a\tvalue1\n" + "id1\tclass_b\tvalue2\n") + + id_to_name = { + "id0": "genome1.fasta", + "id1": "genome2.fasta" + } + + classification, summary_tables = _process_output_files( + temp_output, temp_trees_output, out_dir, id_to_name + ) + + # Check that JSON file was created + json_file = out_dir / "gtdbtk.bac120.summary.tsv.json" + assert json_file.exists() + + # Check JSON content + with open(json_file) as f: + json_data = json.load(f) + + assert "data" in json_data + assert len(json_data["data"]) == 2 + assert json_data["data"][0]["user_genome"] == "genome1.fasta" + assert json_data["data"][1]["user_genome"] == "genome2.fasta" + + def test_process_output_classification_mapping(self, tmp_path): + """Test that classification dict is populated correctly""" + temp_output = tmp_path / "temp_output" + temp_output.mkdir() + (temp_output / "classify").mkdir() + (temp_output / "identify").mkdir() + + temp_trees_output = tmp_path / "temp_trees_output" + temp_trees_output.mkdir() + + out_dir = tmp_path / "out" + out_dir.mkdir() + + summary_file = temp_output / "classify" / "gtdbtk.bac120.summary.tsv" + summary_file.write_text("user_genome\tclassification\n" + "id0\td__Bacteria;p__Test\n" + "id1\td__Bacteria;p__Other\n") + + id_to_name = { + "id0": "genome1.fasta", + "id1": "genome2.fasta" + } + + classification, _ = _process_output_files( + temp_output, temp_trees_output, out_dir, id_to_name + ) + + assert classification["genome1.fasta"] == "d__Bacteria;p__Test" + assert classification["genome2.fasta"] == "d__Bacteria;p__Other" + + def test_process_output_handles_null_values(self, tmp_path): + """Test that null/blank values are replaced with '-'""" + temp_output = tmp_path / "temp_output" + temp_output.mkdir() + (temp_output / "classify").mkdir() + (temp_output / "identify").mkdir() + + temp_trees_output = tmp_path / "temp_trees_output" + temp_trees_output.mkdir() + + out_dir = tmp_path / "out" + out_dir.mkdir() + + summary_file = temp_output / "classify" / "gtdbtk.bac120.summary.tsv" + summary_file.write_text("user_genome\tfield1\tfield2\n" + "id0\tvalue1\t\n" + "id1\t\tvalue2\n") + + id_to_name = { + "id0": "genome1.fasta", + "id1": "genome2.fasta" + } + + _, summary_tables = _process_output_files( + temp_output, temp_trees_output, out_dir, id_to_name + ) + + json_data = summary_tables["gtdbtk.bac120.summary.tsv"] + # Empty values should be replaced with '-' + assert json_data["data"][0]["field2"] == "-" + assert json_data["data"][1]["field1"] == "-" + + def test_process_output_creates_id_map_file(self, tmp_path): + """Test that id_to_name.map file is created""" + temp_output = tmp_path / "temp_output" + temp_output.mkdir() + (temp_output / "classify").mkdir() + (temp_output / "identify").mkdir() + + temp_trees_output = tmp_path / "temp_trees_output" + temp_trees_output.mkdir() + + out_dir = tmp_path / "out" + out_dir.mkdir() + + id_to_name = { + "id0": "genome1.fasta", + "id1": "genome2.fasta" + } + + _process_output_files( + temp_output, temp_trees_output, out_dir, id_to_name + ) + + map_file = out_dir / "id_to_name.map" + assert map_file.exists() + + content = map_file.read_text() + assert "id0\tgenome1.fasta\n" in content + assert "id1\tgenome2.fasta\n" in content + + def test_process_output_missing_id_in_mapping_raises_error(self, tmp_path): + """Test that missing ID in mapping raises ValueError""" + temp_output = tmp_path / "temp_output" + temp_output.mkdir() + (temp_output / "classify").mkdir() + (temp_output / "identify").mkdir() + + temp_trees_output = tmp_path / "temp_trees_output" + temp_trees_output.mkdir() + + out_dir = tmp_path / "out" + out_dir.mkdir() + + summary_file = temp_output / "classify" / "gtdbtk.bac120.summary.tsv" + summary_file.write_text("user_genome\tclassification\n" + "id_missing\tclass_a\n") + + id_to_name = { + "id0": "genome1.fasta" + } + + with pytest.raises(ValueError, match="missing id_missing"): + _process_output_files( + temp_output, temp_trees_output, out_dir, id_to_name + )