diff --git a/.github/dependabot.yml b/.github/dependabot.yml index aa48ef8..ae1d942 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,11 +9,12 @@ updates: directory: "/" schedule: interval: "daily" - target-branch: "module-profile" + target-branch: "main" + - package-ecosystem: "docker" directories: - "/env/picard" - "/env/bases2fastq" schedule: interval: "daily" - target-branch: "module-profile" + target-branch: "main" diff --git a/.github/workflows/update-nextflow-on-dependabot.yml b/.github/workflows/update-nextflow-on-dependabot.yml new file mode 100644 index 0000000..adccbac --- /dev/null +++ b/.github/workflows/update-nextflow-on-dependabot.yml @@ -0,0 +1,103 @@ +name: Update nextflow.config on Dependabot Docker PR + +on: + pull_request_target: + types: [opened, reopened, synchronize] + paths: + - 'env/picard/Dockerfile' + +permissions: + contents: write + pull-requests: write + +jobs: + update-nextflow-config: + if: github.actor == 'dependabot[bot]' + runs-on: ubuntu-latest + + steps: + - name: Checkout PR HEAD safely + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false + + - name: Parse image & version from env/picard/Dockerfile + id: parse + shell: bash + run: | + set -euo pipefail + DOCKERFILE="env/picard/Dockerfile" + if [[ ! -f "$DOCKERFILE" ]]; then + echo "skip=true" >>"$GITHUB_OUTPUT" + exit 0 + fi + + FULL_IMAGE="$(awk '/^FROM[[:space:]]+/ {print $2; exit}' "$DOCKERFILE")" + if [[ -z "${FULL_IMAGE:-}" ]]; then + echo "Could not parse FROM line in $DOCKERFILE"; exit 1 + fi + + IMAGE_BASE="${FULL_IMAGE%%:*}" + NEW_TAG="${FULL_IMAGE#*:}" + + echo "image_base=$IMAGE_BASE" >>"$GITHUB_OUTPUT" + echo "new_tag=$NEW_TAG" >>"$GITHUB_OUTPUT" + echo "skip=false" >>"$GITHUB_OUTPUT" + + - name: Update nextflow.config + if: steps.parse.outputs.skip == 'false' + shell: bash + run: | + set -euo pipefail + CONFIG_FILE="nextflow.config" + IMAGE_BASE="${{ steps.parse.outputs.image_base }}" + NEW_TAG="${{ steps.parse.outputs.new_tag }}" + + if [[ ! -f "$CONFIG_FILE" ]]; then + echo "Missing $CONFIG_FILE"; exit 1 + fi + + BEFORE_HASH="$(sha256sum "$CONFIG_FILE" | awk '{print $1}')" + + # 1) container = 'base:anything' (single or double quotes) + sed -i -E "s|(container[[:space:]]*=[[:space:]]*['\"])${IMAGE_BASE}:[^'\"]*(['\"])|\1${IMAGE_BASE}:${NEW_TAG}\2|g" "$CONFIG_FILE" + + # 2) process.container = 'base:anything' + sed -i -E "s|(process\.container[[:space:]]*=[[:space:]]*['\"])${IMAGE_BASE}:[^'\"]*(['\"])|\1${IMAGE_BASE}:${NEW_TAG}\2|g" "$CONFIG_FILE" + + # 3) Any stray appearances of base:tag (quote-agnostic), last-resort generic swap + sed -i -E "s|(${IMAGE_BASE}:)[0-9A-Za-z._-]+|\1${NEW_TAG}|g" "$CONFIG_FILE" + + AFTER_HASH="$(sha256sum "$CONFIG_FILE" | awk '{print $1}')" + + if [[ "$BEFORE_HASH" == "$AFTER_HASH" ]]; then + echo "No container reference to update in $CONFIG_FILE for base ${IMAGE_BASE}." + echo "Likely the image is referenced elsewhere or via a param; skipping commit." + exit 0 + fi + + echo "=== Diff ===" + git --no-pager diff -- "$CONFIG_FILE" || true + + - name: Commit & push back to PR branch + if: steps.parse.outputs.skip == 'false' + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + if git diff --quiet -- nextflow.config; then + echo "No changes to commit." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add nextflow.config + git commit -m "chore(nextflow): sync container tag with Dockerfile (${GITHUB_SHA::7})" + + PR_HEAD="${{ github.event.pull_request.head.ref }}" + REPO="${{ github.repository }}" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${REPO}.git" "HEAD:${PR_HEAD}" diff --git a/env/picard/Dockerfile b/env/picard/Dockerfile new file mode 100644 index 0000000..5a785c5 --- /dev/null +++ b/env/picard/Dockerfile @@ -0,0 +1,3 @@ +FROM broadinstitute/picard:1.0 + +CMD ["/bin/bash"] diff --git a/modules/archive_run.nf b/modules/archive_run.nf new file mode 100644 index 0000000..e69de29 diff --git a/modules/basecall.nf b/modules/basecall.nf new file mode 100644 index 0000000..01d5c41 --- /dev/null +++ b/modules/basecall.nf @@ -0,0 +1,58 @@ +process BASECALL { + publishDir "${params.alpha}/logs/${params.fcid}/basecall/${lane}", mode:'copy', failOnError: true, pattern: '.command.*' + tag "${params.fcid}" + + module params.PICARD_MODULE + module params.JDK_MODULE + + input: + val lane + + output: + val(lane), emit: lane + path(".command.*") + + shell: + ''' + read_structure=$(python3 -c " + import xml.dom.minidom + + read_structure = '' + runinfo = xml.dom.minidom.parse('!{params.run_dir_path}/RunInfo.xml') + nibbles = runinfo.getElementsByTagName('Read') + + for nib in nibbles: + read_structure += nib.attributes['NumCycles'].value + 'T' + + print(read_structure) + ") + + run_barcode=$(python3 -c " + print('!{params.run_dir_path}'.split('_')[-2].lstrip('0')) + ") + + out_path="!{params.alpha}/lane/!{params.fcid}/!{lane}" + mkdir -p $out_path + + tmp_work_dir="!{params.tmp_dir}!{params.fcid}/!{lane}" + mkdir -p $tmp_work_dir + + java -jar -Xmx58g $PICARD_JAR IlluminaBasecallsToFastq \ + LANE=!{lane} \ + READ_STRUCTURE=${read_structure} \ + BASECALLS_DIR=!{params.run_dir_path}/Data/Intensities/BaseCalls \ + OUTPUT_PREFIX=${out_path}/!{params.fcid}_l0!{lane} \ + RUN_BARCODE=${run_barcode} \ + MACHINE_NAME=!{params.seq_id} \ + FLOWCELL_BARCODE=!{params.fcid} \ + NUM_PROCESSORS=!{task.cpus} \ + APPLY_EAMSS_FILTER=false \ + INCLUDE_NON_PF_READS=false \ + MAX_READS_IN_RAM_PER_TILE=200000 \ + MINIMUM_QUALITY=2 \ + COMPRESS_OUTPUTS=true \ + TMP_DIR=${tmp_work_dir} + + rm -rf ${tmp_work_dir} + ''' +} \ No newline at end of file diff --git a/modules/check_do_merge.nf b/modules/check_do_merge.nf new file mode 100644 index 0000000..e69de29 diff --git a/modules/check_no_demux.nf b/modules/check_no_demux.nf new file mode 100644 index 0000000..401675c --- /dev/null +++ b/modules/check_no_demux.nf @@ -0,0 +1,17 @@ +process CHECK_NO_DEMUX { + tag "${params.fcid}" + echo true + + input: + val lane + + output: + tuple val(lane), env(no_demux), emit: lane + + shell: + ''' + export PYTHONPATH=$PYTHONPATH:!{workflow.projectDir}/bin + no_demux=$(python3 -c "from slime import check_demux;r=check_demux('!{params.fcid}', !{lane});print(str(r).lower())") + echo "check_no_demux: lane: !{lane}, no_demux: $no_demux" + ''' +} \ No newline at end of file diff --git a/modules/initialize.nf b/modules/initialize.nf new file mode 100644 index 0000000..e69de29 diff --git a/modules/rsync_to_archive.nf b/modules/rsync_to_archive.nf new file mode 100644 index 0000000..791b8a4 --- /dev/null +++ b/modules/rsync_to_archive.nf @@ -0,0 +1,21 @@ +process RSYNC_TO_ARCHIVE { + publishDir "${params.alpha}/logs/${params.fcid}/archive/rsync/", mode:'copy', failOnError: true + tag "${params.fcid}" + echo true + + input: + path SRC + val destination + + output: + path(".command.*") + env(exit_code), emit: exit_code + + shell: + ''' + echo "rsyncToArchive: SRC: !{SRC}, destination: !{destination}" + ssh -i $HOME/.ssh/id_rsa core2 "mkdir -p !{destination}" + rsync --copy-links --progress -r -e "ssh -i ${HOME}/.ssh/id_rsa" !{SRC} core2:!{destination}/. + exit_code=$? + ''' +} \ No newline at end of file diff --git a/modules/tar.nf b/modules/tar.nf new file mode 100644 index 0000000..e69de29 diff --git a/nextflow.config.template b/nextflow.config similarity index 64% rename from nextflow.config.template rename to nextflow.config index fc5311b..c648b91 100644 --- a/nextflow.config.template +++ b/nextflow.config @@ -118,3 +118,61 @@ trace { env { SEQERA_ACCESS_TOKEN="" } + +profiles { + hpc-modules { + process { + withName: 'multiqc' { + module = params.MULTIQC_MODULE + } + withName: 'tar' { + module = params.PBZIP2_MODULE + } + withName: '_basecall_bases2fastq' { + module = params.BASES2FASTQ_MODULE + } + withName: '_basecall_picard' { + module = params.PICARD_MODULE + module = params.JDK_MODULE + } + } + } + + hpc-container { + singularity.enabled = true + + process { + container = /path/to/conda-container + + withName: '_basecall_picard' { + container = 'docker://broadinstitute/picard:2.23.8' + } + + withName: 'bases2fastq' { + container = 'docker://elembio/bases2fastq:2.2.0 + } + } + } + + default { + singularity.enabled = true + + conda.enabled = true + #conda.cacheDir = + #conda.useMamba = true + + + process { + conda = ${baseDir}/environment.yml + + withName: '_basecall_picard' { + container = 'docker://broadinstitute/picard:1.0' + } + + withName: 'bases2fastq' { + container = 'docker://elembio/bases2fastq:2.1.0' + } + + } + } +}