PTV-1904 add github actions

.github/workflows/sdkless_test.yml

@@ -0,0 +1,34 @@
+name: SDK-less unit tests
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - ready_for_review
+
+jobs:
+  sdkless_test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout GitHub repo
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Cache pip install
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-gha.txt') }}
+
+      - name: Install pip packages
+        run: pip install -r requirements-gha.txt
+
+      - name: Run sdkless tests
+        run: make test-sdkless
+

RELEASE_NOTES.md

@@ -1,6 +1,10 @@
 # kb_gtdbtk release notes
 =========================
 
+1.4.1
+_____
+* Fixed report generation and file post-processing issues
+
 1.0.1
 _____
 * Reverting to most recent prior release to address the recurring bug from the new release 1.4.0

kbase.yml

@@ -8,10 +8,10 @@ service-language:
     python
 
 module-version:
-    1.4.0
+    1.4.1
 
 owners:
-    [dylan, psdehal, donovan_parks, aaronmussig, pchaumeil, gaprice, zcrockett, tgu2]
+    [dylan, psdehal, donovan_parks, aaronmussig, pchaumeil, gaprice, zcrockett, tgu2, wjriehl]
 
 data-version:
     0.1.1

lib/kb_gtdbtk/core/gtdbtk_runner.py

@@ -5,10 +5,10 @@
 import logging
 import json
 import os
+import shutil
 import pandas as pd
 import tempfile
 
-from datetime import datetime
 from pathlib import Path
 from shutil import (
     copyfile,
@@ -20,7 +20,8 @@
     Dict,
     List,
     Optional,
-    Tuple
+    Tuple,
+    Union
 )
 from kb_gtdbtk.core.string_util import now_ISOish
 
@@ -99,6 +100,8 @@ def run_gtdbtk(
     os.environ['GTDBTK_DATA_PATH'] = str(data_root_dir / f"r{db_ver}")
 
     # set output dirs
+    # temp_output = main gtdbtk output directory
+    # temp_trees_output = only used if a second gtdbtk run is done with --skip_ani_screen
     temp_output = temp_dir / 'output' / timestamp
     temp_trees_output = temp_dir / 'output_trees' / timestamp
     temp_output.mkdir(parents=True, exist_ok=True)
@@ -146,9 +149,16 @@ def run_gtdbtk(
     return _process_output_files(temp_output, temp_trees_output, output_dir, id_to_name)
 
 
-# _all_ids_in_trees ()
-#
 def _all_ids_in_trees(temp_output: Path, id_to_name: Dict[str, str]) -> bool:
+    """
+    Looks at the primary gtdbtk output file - either gtdbtk.ar53.summary.tsv or
+    gtdbtk.bac120.summary.tsv (under the temp_output directory). These files have
+    20 columns. First is the user_genome id, and 13th is the taxonomy of the tree it was
+    placed in. This checks the following:
+    1. if all user_genomes exist in the output.
+    2. if they're all mapped to trees - that pplacer_taxonomy is not "N/A"
+    Returns True if all genomes submitted appear with trees, False otherwise.
+    """
     all_ids_found = True
 
     ids_found = dict()
@@ -173,28 +183,63 @@ def _all_ids_in_trees(temp_output: Path, id_to_name: Dict[str, str]) -> bool:
     return all_ids_found
 
 
-# _process_output_files()
-#
 def _process_output_files(
         temp_output: Path,
         temp_trees_output: Path,
         out_dir: Path,
         id_to_name: Dict[str, str]) -> Tuple[dict, dict]:
+    """
+    Process GTDB-tk output files and consolidate results into a structured format.
+
+    This function handles the post-processing of GTDB-tk classification workflow output.
+    It consolidates results from both a primary run and an optional secondary run
+    (performed with --skip_ani_screen), merges summary data, converts TSV files to JSON
+    format, and remaps internal sequence identifiers to their original assembly names.
+
+    Key operations:
+    1. Copies the entire temp_output directory structure to the output directory
+    2. Saves the mapping between internal IDs and original assembly names
+    3. Copies tree files, preferring those from the secondary run if available
+    4. Merges summary TSV files from both runs, with secondary run data taking
+       precedence for N/A fields
+    5. Converts summary TSV files to JSON format for web UI consumption
+    6. Remaps internal IDs back to original assembly names in output data
+    7. Filters out blank/null fields by replacing them with '-' for consistency
+
+    :param temp_output: Path to the primary GTDB-tk output directory. Contains
+        results from the initial classify_wf run with all markers and ANI screening.
+        Expected subdirectories: 'classify' and 'identify'
+    :param temp_trees_output: Path to the secondary GTDB-tk output directory. Contains
+        results from the secondary classify_wf run (if performed) with --skip_ani_screen
+        flag. May be empty if all sequences were placed in trees during the primary run.
+        Expected subdirectories: 'classify' and 'identify'
+    :param out_dir: Path to the output directory where final results will be written.
+        Should be an existing, writable directory. Output includes:
+        - 'runtime_output/': Copy of temp_output directory structure
+        - 'id_to_name.map': Tab-separated file mapping internal IDs to assembly names
+        - Tree files (*.tree): Phylogenetic trees from both runs
+        - Summary TSV files: Classification and marker summary data
+        - Summary JSON files: JSON-converted versions of summary TSV files
+    :param id_to_name: Mapping from internal sequence identifiers (e.g., 'id0', 'id1')
+        to original assembly names. Used to remap internal IDs in output data back to
+        user-supplied assembly names.
+
+    :return: A tuple of (classification, summary_tables) where:
+        - classification (dict): Mapping from assembly name to classification string.
+          Keys are original assembly names (from id_to_name values).
+          Values are classification strings from the 'classification' field in summary files.
+        - summary_tables (dict): Mapping from filename to parsed JSON data.
+          Keys are TSV filenames (e.g., 'gtdbtk.bac120.summary.tsv').
+          Values are dictionaries with a 'data' key containing a list of record objects
+          with assembly names and classifications.
+    """
 
     classification = dict()
     summary_tables = dict()
 
     # copy over all created output
-    """
-    for file_ in os.listdir (temp_output):
-        tmppath = temp_output / file_
-        if not tmppath.is_file():
-            continue
-        path = out_dir / file_
-        copyfile(tmppath, path)
-    """
     sub_out_dir = Path(out_dir / 'runtime_output')
-    if os.path.isdir(sub_out_dir):  # only occurs during unit tests
+    if os.path.isdir(sub_out_dir):  # should only occur during unit tests
         rmtree(sub_out_dir)
     copytree(temp_output, sub_out_dir, symlinks=True)
 
@@ -218,6 +263,9 @@ def _process_output_files(
         'gtdbtk.bac120.markers_summary.tsv',
         'gtdbtk.bac120.tree.mapping.tsv'
     ]
+    # These are the output files we care about - they're either in the classify/ or identify/
+    # subdirectories. We'll update these below based on file structure (and whether or not
+    # there was a --skip_ani_screen run)
     file_folder = {
         'gtdbtk.ar53.summary.tsv': 'classify',
         'gtdbtk.bac120.summary.tsv': 'classify',
@@ -239,6 +287,7 @@ def _process_output_files(
 
     # copy all tree files to the output directory
     # these may be in the temp_trees_output or the temp_output directory
+    # prefer the temp_trees_output directory
     for file_ in tree_files + bb_tree_file + extra_bac_tree_files:
         treepath = temp_trees_output / file_folder[file_] / file_
         tmppath = temp_output / file_folder[file_] / file_
@@ -252,52 +301,8 @@ def _process_output_files(
     for file_ in base_files:
         treepath = temp_trees_output / file_folder[file_] / file_
         tmppath = temp_output / file_folder[file_] / file_
-        path = out_dir / file_
-        found_file = False
-        num_cols = 0
-
-        id_order = []
-        tmp_buf = dict()
-        # TODO: decompose, avoid duplications, overlap
-        num_tmp_cols = 0
-        num_tree_cols = 0
-        if tmppath.is_file():
-            found_file = True
-            with open(tmppath, 'r') as tmppath_h:
-                for info_line in tmppath_h:
-                    row = info_line.rstrip().split("\t")
-                    tmp_buf[row[0]] = row
-                    id_order.append(row[0])
-                    num_tmp_cols = max(len(row), num_tmp_cols)
-        tree_buf = dict()
-        if treepath.is_file():
-            found_file = True
-            id_order = []
-            with open (treepath, 'r') as treepath_h:
-                for info_line in treepath_h:
-                    row = info_line.rstrip().split("\t")
-                    tree_buf[row[0]] = row
-                    id_order.append(row[0])
-                    num_tree_cols = max(len(row), num_tree_cols)
-
-        num_cols = max(num_tmp_cols, num_tree_cols)
-
-        if not found_file:
-            continue
-        out_buf = []
-        for qid in id_order:
-            row = ["N/A"] * num_cols
-            if qid in tmp_buf:
-                row = tmp_buf[qid]
-            if qid in tree_buf:
-                for field_i, _ in enumerate(tree_buf[qid]):
-                    if row[field_i] == 'N/A':
-                        row[field_i] = tree_buf[qid][field_i]
-            out_buf.append("\t".join(row))
-
-        # write merged summaries
-        with open(path, 'w') as summary_h:
-            summary_h.write("\n".join(out_buf)+"\n")
+        out_path = out_dir / file_
+        _merge_summary_tsv_files(tmppath, treepath, out_path)
 
     # load results
     for file_ in base_files:
@@ -348,3 +353,89 @@ def _process_output_files(
             summary_tables[file_] = sj
 
     return (classification, summary_tables)
+
+def _merge_summary_tsv_files(std_path: Path, tree_path: Path, output_path: Path):
+    """
+    Merges two TSV files from separate runs of GTDBtk.
+    The std_path represents the standard run, using a mash DB.
+    The tree_path is from a run using --skip_ani_screen, which generates file in a different path.
+    These two TSV files have slightly different outputs, with data in tree_path being generally more
+    informative. So fields for each genome in the summary file should have tree_path override any N/A values
+    found in std_path. E.g.:
+
+    std_path:
+    user_genome    field1    field2    field3
+    some_genome    N/A       N/A       X
+
+    tree_path:
+    user_genome    field1    field2    field3
+    some_genome    Y         N/A       X
+
+    merged:
+    user_genome    field1    field2    field3
+    some_genome    Y         N/A       X
+    """
+
+    std_file_exists = std_path.is_file()
+    tree_file_exists = tree_path.is_file()
+    if not std_file_exists and not tree_file_exists:
+        return
+    if std_file_exists and not tree_file_exists:
+        shutil.copy(std_path, output_path)
+        return
+    if tree_file_exists and not std_file_exists:
+        shutil.copy(tree_path, output_path)
+        return
+    std_file_data = _load_summary_tsv_file(std_path)
+    tree_file_data = _load_summary_tsv_file(tree_path)
+    if std_file_data["header"] != tree_file_data["header"]:
+        logging.warning(f"While merging summary files: header of {std_path} (from using a mash DB) does not match {tree_path} (skipping the ANI screen). These cannot be merged, saving only the ANI-skipped file.")
+        shutil.copy(tree_path, output_path)
+        return
+    out_buf = ["\t".join(std_file_data["header"])]
+    num_cols = len(std_file_data["header"])
+    for qid in std_file_data["id_order"]:
+        row = ["N/A"] * num_cols
+        if qid in std_file_data["data"]:
+            row = std_file_data["data"][qid]
+        if qid in tree_file_data["data"]:
+            for field_i, value in enumerate(tree_file_data["data"][qid]):
+                if row[field_i] == "N/A":
+                    row[field_i] = value
+        out_buf.append("\t".join(row))
+
+    # write merged summaries
+    with open(output_path, "w") as summary_file:
+        summary_file.write("\n".join(out_buf)+"\n")
+
+
+def _load_summary_tsv_file(filepath: Path) -> Dict[str, Union[List, Dict]]:
+    """
+    Loads a TSV file, includes the header (assumes that each summary file from GTDBtk has
+    a header) and all lines.
+    Summary files all have a unique identifier (generally the genome or assembly name)
+    as the first element of each line. This, then, creates the following structure:
+    {
+        "header": [],
+        "data": {"id": [row]},
+        "id_order": []
+    }
+    """
+    header = []
+    data = {}
+    id_order = []
+    with open(filepath, "r") as infile:
+        for line_num, line in enumerate(infile):
+            row = line.rstrip("\n\r").split("\t")
+            if line_num == 0:
+                header = row
+            else:
+                identifier = row[0]
+                data[identifier] = row
+                id_order.append(identifier)
+
+    return {
+        "header": header,
+        "data": data,
+        "id_order": id_order
+    }

lib/kb_gtdbtk/kb_gtdbtkImpl.py

@@ -46,7 +46,7 @@ class kb_gtdbtk:
     ### log()
     #
     def log(self, target, message):
-        message = '['+self.now_ISOish()+'] '+message
+        message = '['+now_ISOish()+'] '+message
         if target is not None:
             target.append(message)
         print(message)

requirements-gha.txt

@@ -0,0 +1,7 @@
+flake8
+jsonrpcbase
+mypy
+pandas
+pytest
+pytest-cov
+requests