diff --git a/.Rbuildignore b/.Rbuildignore index fffd6b6..96dfc85 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -6,7 +6,14 @@ environment.yml run_pipeline.sh scripts snakemake -tests venv .idea -.env_example \ No newline at end of file +.env_example +.env +.cache +^\.github$ +^\.lintr$ +^PLATFORM_SETUP\.md$ +^\.git$ +^\.gitignore$ +^\.cursorrules$ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..681344f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,29 @@ +.git +.github +.cursorrules +.snakemake +.genehackman +.cache +pipeline_logs +.ipynb_checkpoints +.env +.env_example +*.sif +build +build.log +README.pdf +tests/testthat/data +venv +__pycache__ +*.pyc +.pytest_cache +.quarto +**/*.quarto_ipynb +*input.yaml +*input.json +.Rproj.user +.DS_Store +ieugwasr_oauth +PLATFORM_SETUP.md +extract_by_range.r +input_for_project.yaml diff --git a/.env_example b/.env_example index 2610fe7..936836f 100644 --- a/.env_example +++ b/.env_example @@ -1,8 +1,10 @@ -DATA_DIR= -RESULTS_DIR= -RDFS_DIR= -GENOMIC_DATA_DIR=/mnt/storage/private/mrcieu/data/genomic_data/ -THOUSAND_GENOMES_DIR=/mnt/storage/private/mrcieu/data/genomic_data/1000genomes/b37_dbsnp156/ -LDSC_DIR=/mnt/storage/private/mrcieu/data/LDSCORE/b37_dbsnp156/ -QTL_DIRECTORY=/mnt/storage/private/mrcieu/data/qtl_datasets/ -DOCKER_VERSION= +#Mandatory variables +PIPELINE_DATA_DIR= +PROJECT_DIR= +DOCKER_VERSION=1.1.0 +# Optional variables +QTL_DATA_DIR= +SNAKEMAKE_PROFILE=snakemake/profiles/local/ +APPTAINER_MODULE= +SLURM_PARTITION= +SLURM_ACCOUNT= diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..7a9c4b0 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,37 @@ +name: CI/CD + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, ready_for_review, synchronize] + +jobs: + devtools_check: + runs-on: ubuntu-latest + container: + image: mrcieu/genehackman:develop + env: + R_LIBS_SITE: /usr/lib/R/site-library + steps: + - uses: actions/checkout@v4 + + - name: Run devtools::check() + run: Rscript -e "devtools::check()" + end_to_end_tests_run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check that end-to-end tests have been run successfully + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: | + if [ ! -f tests/testing_complete.txt ]; then + echo "Tests have not been run successfully, please run ./tests/e2e_tests/run_test_pipelines.sh and commit the results" + exit 1 + fi + if [ "$BRANCH_NAME" != "main" ] && ! grep -q "$BRANCH_NAME" tests/testing_complete.txt; then + echo "Tests have not been run successfully for branch: $BRANCH_NAME, please run Rscript tests/testthat/run_tests.R and commit the results" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 5a054fb..76f4721 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,17 @@ .Rproj.user +.cursorrules .snakemake/ +.genehackman/ +pipeline_logs/ .cache/ .ipynb_checkpoints/ +*input.yaml *input.json .env .bashrc .DS_Store ieugwasr_oauth +__pycache__/ + +/.quarto/ +**/*.quarto_ipynb diff --git a/.lintr b/.lintr new file mode 100644 index 0000000..968f89c --- /dev/null +++ b/.lintr @@ -0,0 +1,5 @@ +linters: with_defaults( + line_length_linter = line_length_linter(120), + quotes_linter = NULL, + object_name_linter = NULL +) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..42c9662 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,332 @@ +# Contributing to GeneHackman + +This guide covers how to change code, update the Docker image, and run the test suite before opening a pull request. + +For running pipelines in production, see [README.md](README.md) and [PLATFORM_SETUP.md](PLATFORM_SETUP.md). For pipeline inputs and parameters, see [snakemake/PIPELINES.md](snakemake/PIPELINES.md). + +## Development setup + +1. Clone the repository and create the conda environment: + + ```bash + git clone git@github.com:MRCIEU/GeneHackman.git + cd GeneHackman + conda env create -f environment.yml + conda activate genehackman + ``` + +2. Copy and edit `.env` (see [.env_example](.env_example)): + + ```bash + cp .env_example .env + ``` + + For development you need at least: + + - **`PROJECT_DIR`** — absolute path where test outputs go (e.g. a scratch folder; the pipeline writes to `PROJECT_DIR/data/` and `PROJECT_DIR/results/`). + - **`PIPELINE_DATA_DIR`** — absolute path to the reference data bundle from `gs://genehackman` (1000 Genomes LD panels, LDSC assets, etc.). + - **`DOCKER_VERSION`** — image tag to run (e.g. `1.1.0` or `develop`). + + Use **absolute paths** in `.env`. Relative paths (e.g. `QTL_DATA_DIR=hi`) break Apptainer bind mounts with errors like `destination must be an absolute path`. + +3. Install the R package locally (for unit tests outside Docker): + + ```bash + Rscript -e "devtools::install()" + ``` + +## Repository layout + +| Path | Role | +|------|------| +| `R/` | R package functions used by pipeline steps | +| `scripts/` | CLI entry points called from Snakemake (`Rscript …`, `python …`) | +| `snakemake/` | Workflow `.smk` files, `profiles/`, `input_templates/`, shared `util/` | +| `docker/` | `Dockerfile`, `requirements.R`, `requirements.txt` | +| `tests/testthat/` | Unit tests and small test GWAS files | +| `tests/e2e_tests/` | End-to-end Snakemake test runner | +| `inst/` | Package data (e.g. column maps) | + +Snakemake profiles bind-mount `R/`, `scripts/`, and `inst/` from the repo into the container, so **changes to R and script code take effect without rebuilding the image** on the next pipeline run. New R or Python **dependencies** still require a Docker rebuild. + +## Making code changes + +### R package (`R/`) + +- Follow existing style: **data.table** / **dplyr** patterns already in the file, **roxygen2** docs for exported functions. +- Regenerate documentation when you change exports: + + ```bash + Rscript -e "devtools::document()" + ``` + +- Snakemake rules typically call thin wrappers in `scripts/` that load the package and parse CLI args. + +### Scripts (`scripts/`) + +- Keep scripts as CLI wrappers; put reusable logic in `R/`. +- Python scripts (e.g. `run_multisusie.py`) should stay compatible with packages in `docker/requirements.txt`. + +### Snakemake (`snakemake/`) + +- Shared helpers live in `snakemake/util/` (`common.smk`, `constants.smk`, rules under `snakemake/rules/`). +- Add or update an example input under `snakemake/input_templates/` when you change required YAML fields. +- Site-specific cluster settings belong in new profiles under `snakemake/profiles/` (copy `local/` or `slurm/` as a template). + +### Conventions + +- **Ancestry** codes must be one of: `EUR`, `EAS`, `AFR`, `AMR`, `SAS`. +- **Finemap** (`finemap.smk`): ancestries must be either all the same (single-ancestry SuSiE) or all distinct (multi-ancestry MultiSuSiE). Mixed duplicates fail at startup. +- **Coloc** (`coloc.smk`): all GWAS inputs must share the same ancestry. + +## Docker changes + +The pipeline runs inside **`mrcieu/genehackman`** (Apptainer/Singularity on HPC, Docker locally). + +| File | Purpose | +|------|---------| +| `docker/Dockerfile` | Base OS, R, PLINK, liftOver, LDSC, PHESANT, bcftools | +| `docker/requirements.R` | CRAN/Bioconductor R dependencies | +| `docker/requirements.txt` | Python dependencies (Snakemake, MultiSuSiE, …) | + +The Dockerfile copies only `DESCRIPTION`, `docker/requirements.R`, and `docker/requirements.txt` before installing dependencies, so **edits to those files invalidate the dependency layer** and rebuild quickly without copying the whole repo first. + +### Build locally + +From the **repository root**: + +```bash +docker build --platform linux/amd64 -f docker/Dockerfile \ + -t mrcieu/genehackman:$(grep '^Version:' DESCRIPTION | awk '{print $2}') . +``` + +The image is **linux/amd64 only**. Use `--platform linux/amd64` on Apple Silicon. + +After changing `DESCRIPTION` (new R package in `Imports:`), update `docker/requirements.R` or rely on `remotes::install_deps("docker", …)` picking up new imports. + +After changing Python deps, edit `docker/requirements.txt` and rebuild. + +### Publish (maintainers) + +```bash +docker push mrcieu/genehackman: +``` + +Bump **`Version:`** in `DESCRIPTION` and **`DOCKER_VERSION`** in `.env` together so the SIF name (`genehackman_.sif`) matches the image tag. + +HPC users without Docker pull the same image via `run_pipeline.sh`, which builds or uses `$PIPELINE_DATA_DIR/genomic_data/pipeline/genehackman_.sif`. + +## Unit tests + +Unit tests use **testthat** and live in `tests/testthat/`. + +### Run all package tests + +```bash +# Inside the conda env, with the package installed: +Rscript -e "devtools::test()" + +# Or: +Rscript tests/testthat.R +``` + +### Full package check (what CI runs) + +Runs `R CMD check`–style validation (examples, vignettes, namespace, etc.): + +```bash +Rscript -e "devtools::check()" +``` + +CI runs this inside `mrcieu/genehackman:develop` (see [.github/workflows/main.yml](.github/workflows/main.yml)). + +### Writing tests + +- Add new test files as `tests/testthat/test_.R`. +- Use `testthat::local_mocked_bindings()` to mock external tools (PLINK, SuSiE, liftover) where the existing tests do. +- Small GWAS fixtures are under `tests/testthat/data/`. + +## End-to-end pipeline tests + +End-to-end tests run real Snakemake workflows against tiny test GWAS files via Apptainer. + +### Prerequisites + +- `.env` configured with valid **`PROJECT_DIR`** and **`PIPELINE_DATA_DIR`** (reference data required for LD, liftover, etc.). +- Apptainer/Singularity available (see [PLATFORM_SETUP.md](PLATFORM_SETUP.md)). +- Conda env activated. + +### Run all e2e tests + +```bash +./tests/e2e_tests/run_test_pipelines.sh +``` + +This script runs `run_pipeline.sh` with `-F` (force rerun) for: + +| Pipeline | Test input | +|----------|------------| +| `standardise_gwas.smk` | `tests/testthat/data/snakemake_inputs/standardise_gwas.yaml` | +| `disease_progression.smk` | `tests/testthat/data/snakemake_inputs/disease_progression.yaml` | +| `compare_gwases.smk` | `tests/testthat/data/snakemake_inputs/compare_gwases.yaml` | +| `finemap.smk` | `finemap.yaml` and `finemap_multi_ancestry.yaml` | +| `coloc.smk` | `tests/testthat/data/snakemake_inputs/coloc.yaml` | +| `qtl_mr.smk` | `qtl_mr_eqtlgen.yaml` (only if **`QTL_DATA_DIR`** is set in `.env`) | + +On success it writes **`tests/testing_complete.txt`** with a line like: + +```text +SUCCESS: All tests passed on branch: your-branch-name +``` + +### CI requirement + +Pull requests must include an updated **`tests/testing_complete.txt`** from a successful run on **your branch**. GitHub Actions checks that: + +1. The file exists. +2. On non-`main` branches, the file contains the branch name. + +Run the e2e script on your feature branch, then commit `tests/testing_complete.txt` with your other changes. + +### Run a single pipeline test + +```bash +./run_pipeline.sh snakemake/finemap.smk \ + tests/testthat/data/snakemake_inputs/finemap.yaml -F +``` + +Useful flags: `--dry-run`, `--unlock`, `-n` (dry run), `-R ` (rerun specific rule). + +## Suggested workflow before a PR + +1. Make changes on a feature branch. +2. Run unit tests: `Rscript -e "devtools::test()"` (or `devtools::check()` for a fuller pass). +3. Run e2e tests: `./tests/e2e_tests/run_test_pipelines.sh`. +4. Commit code changes **and** `tests/testing_complete.txt`. +5. Open a pull request against `main`. + +If you change Docker dependencies, note the new image tag in the PR description and confirm you have rebuilt (or that maintainers will publish) the matching `mrcieu/genehackman` image. + +## Cutting a release (maintainers) + +Releases tie together three versioned artefacts: + +| Artefact | Where | Format | +|----------|--------|--------| +| R package | `DESCRIPTION` → `Version:` | `1.2.0` (no `v` prefix) | +| Docker / Apptainer image | Docker Hub `mrcieu/genehackman` | tag `1.2.0` (matches `Version:`) | +| Git tag | GitHub | `v1.2.0` (`v` + same semver) | + +Users set **`DOCKER_VERSION=1.2.0`** in `.env`; Snakemake looks for `genehackman_1.2.0.sif` under `PIPELINE_DATA_DIR/genomic_data/pipeline/`. + +### Before you release + +1. Merge all intended changes to **`main`**. +2. Confirm CI is green on `main` ([Actions](https://github.com/MRCIEU/GeneHackman/actions)). +3. Run the full test suite on `main`: + + ```bash + git checkout main && git pull + Rscript -e "devtools::check()" + ./tests/e2e_tests/run_test_pipelines.sh + ``` + +4. Commit `tests/testing_complete.txt` on `main` if the e2e run updated it. + +### 1. Bump the version + +Edit **`Version:`** in [`DESCRIPTION`](DESCRIPTION) to the new semver (e.g. `1.2.0`). + +Update the example default in [`.env_example`](.env_example): + +```bash +DOCKER_VERSION=1.2.0 +``` + +Regenerate R docs if exports changed: + +```bash +Rscript -e "devtools::document()" +``` + +Commit on `main` (or via PR): + +```bash +git add DESCRIPTION .env_example +git commit -m "Bump version to 1.2.0" +git push origin main +``` + +### 2. Build and publish the Docker image + +From the repository root, on a machine with Docker Hub access to **`mrcieu`**: + +```bash +VERSION=1.2.0 + +docker build --platform linux/amd64 -f docker/Dockerfile \ + -t mrcieu/genehackman:${VERSION} . + +docker push mrcieu/genehackman:${VERSION} +``` + +Optional: refresh the rolling **`develop`** tag used by CI (`mrcieu/genehackman:develop` in [`.github/workflows/main.yml`](.github/workflows/main.yml)): + +```bash +docker tag mrcieu/genehackman:${VERSION} mrcieu/genehackman:develop +docker push mrcieu/genehackman:develop +``` + +### 3. Tag the release in Git + +Create an annotated tag on `main` pointing at the version bump commit: + +```bash +git checkout main && git pull +git tag -a v1.2.0 -m "Release 1.2.0" +git push origin v1.2.0 +``` + +Tags use a **`v` prefix** (e.g. `v1.0.0`); Docker tags do **not** (`1.2.0`). + +### 4. Create the GitHub release + +Using the GitHub CLI: + +```bash +gh release create v1.2.0 \ + --title "1.2.0" \ + --notes "$(cat <<'EOF' +## Summary +- … + +## Docker +`docker pull mrcieu/genehackman:1.2.0` + +## Citation +https://doi.org/10.5281/zenodo.10624713 +EOF +)" +``` + +Or in the browser: **GitHub → Releases → Draft a new release** → choose tag `v1.2.0`, title `1.2.0`, and add release notes (changes since the previous tag, Docker pull command, any breaking changes). + +### 5. Zenodo archive + +The project is archived on Zenodo ([10.5281/zenodo.10624713](https://doi.org/10.5281/zenodo.10624713)). If the [Zenodo–GitHub integration](https://docs.github.com/en/repositories/archiving-a-github-repository/referencing-and-citing-content) is enabled for this repository, publishing the GitHub release should trigger a new Zenodo version automatically. Otherwise, upload the release manually on Zenodo and note the new version DOI in the GitHub release. + +### After release + +Tell users to: + +1. Set **`DOCKER_VERSION`** in `.env` to the new version. +2. Pull or build the SIF, e.g. delete an old `genehackman_*.sif` and re-run `run_pipeline.sh` (it builds from `docker://mrcieu/genehackman:` if the file is missing), or on HPC: + + ```bash + singularity build genehackman_1.2.0.sif docker://mrcieu/genehackman:1.2.0 + ``` + +## Getting help + +- Open a [GitHub issue](https://github.com/MRCIEU/GeneHackman/issues) for bugs or feature requests. +- Contact **andrew.elmore at bristol dot ac uk** for Bristol-internal coordination. diff --git a/DESCRIPTION b/DESCRIPTION index 7a6ae8e..757f75c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,14 +1,16 @@ Package: GeneHackman Title: R package to support pipelines to run analyses on genetic summary statistics on the University of Bristol HPC -Version: 1.0.0 +Version: 1.1.0 Authors@R: person("Andrew", "Elmore", , "andrew.elmore@bristol.ac.uk", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-0292-447X")) Description: What the package does (one paragraph). License: GPL (>= 3) +Depends: + R (>= 4.1.0) Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.2 Suggests: testthat (>= 3.0.0) Config/testthat/edition: 3 @@ -16,13 +18,17 @@ Imports: coloc, data.table, dplyr, + fst, future, - genepi.utils, + furrr, ggplot2, ggrepel, httr, + locuszoomr, MendelianRandomization, + patchwork, prettydoc, + progressr, qqman, rlang, R.utils, @@ -30,6 +36,7 @@ Imports: shiny, SlopeHunter, stringr, + susieR, tibble, tidyr, TwoSampleMR, @@ -38,5 +45,4 @@ Remotes: github::MRCIEU/TwoSampleMR, github::MRCIEU/ieugwasr, github::Osmahmoud/SlopeHunter, - github::nicksunderland/genepi.utils@v0.0.5, github::yixuan/prettydoc diff --git a/NAMESPACE b/NAMESPACE index 6e0a3bf..304930c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,47 +1,62 @@ # Generated by roxygen2: do not edit by hand export(calculate_f_statistic) +export(change_snp_identifiers) +export(chrpos_to_rsid) export(coloc_analysis) +export(coloc_bf_bf_qtl_analysis) export(conduct_collider_bias_analysis) export(convert_beta_to_or) export(convert_or_to_beta) export(convert_reference_build_via_liftover) -export(convert_z_score_to_beta) +export(finemap_gwas) export(forest_plot) export(get_file_or_dataframe) export(grouped_forest_plot) export(gwas_region) export(harmonise_gwases) +export(locus_zoom_coloc) export(manhattan_and_qq) export(miami_plot) +export(populate_eaf_from_reference_panel) export(populate_gene_names) export(populate_rsid) +export(run_bf_bf_coloc) export(run_coloc_on_list_of_datasets) +export(run_coloc_on_qtl_mr_results) export(standardise_gwas) export(volcano_plot) export(vroom_snps) -import(MendelianRandomization) -import(R.utils) -import(SlopeHunter) +import(MendelianRandomization, except = c(mr_ivw, mr_median)) +import(R.utils, except = c(env, extract, reset, setProgress, timestamp, validate)) +import(SlopeHunter, except = c(format_data)) import(TwoSampleMR) import(coloc) -import(data.table) -import(dplyr) -import(future) -import(genepi.utils) +import(data.table, except = c(between, first, last)) +import(dplyr, except = c(between, first, last, filter, lag)) +import(future, except = c(run)) import(ggplot2) import(ggrepel) import(grDevices) import(graphics) import(httr) +import(patchwork) import(prettydoc) import(qqman) -import(rlang) +import(rlang, except = c(`:=`)) import(rmarkdown) -import(shiny) -import(stats) +import(shiny, except = c(setProgress, validate)) +import(stats, except = c(filter, lag)) import(stringr) +import(susieR) import(tibble) -import(tidyr) -import(utils) +import(tidyr, except = c(extract)) +import(utils, except = c(timestamp)) import(vroom) +importFrom(data.table,setkey) +importFrom(data.table,setkeyv) +importFrom(fst,fst) +importFrom(fst,read_fst) +importFrom(furrr,future_map) +importFrom(progressr,progressor) +importFrom(tibble,as_tibble) diff --git a/PLATFORM_SETUP.md b/PLATFORM_SETUP.md new file mode 100644 index 0000000..3d69dd2 --- /dev/null +++ b/PLATFORM_SETUP.md @@ -0,0 +1,138 @@ +# Running GeneHackman by platform + +Snakemake orchestrates steps; each step usually runs inside an **Apptainer/Singularity** image (`docker://mrcieu/genehackman` or a local `.sif`). Snakemake **profiles** live under **`snakemake/profiles/`** (each profile is a directory containing `config.yaml`). They choose how jobs run (local cores vs cluster scheduler) and which host paths are **bind-mounted** into the container. + +General prerequisites: + +1. **Conda env:** `conda env create -f environment.yml` then `conda activate genehackman` +2. **`.env`:** copy from `.env_example` and set **`PROJECT_DIR`** (the pipeline uses **`PROJECT_DIR/data/`** and **`PROJECT_DIR/results/`**) and **`PIPELINE_DATA_DIR`** (and paths your pipeline needs for genomic / 1000G data). Optionally set **`QTL_DATA_DIR`** when large QTL mirrors live on another volume or object-store mount; if omitted, `run_pipeline.sh` defaults it to `PIPELINE_DATA_DIR/qtl_datasets`. +3. **Input YAML:** see `snakemake/input_templates/` and `snakemake/PIPELINES.md`. + +**`./run_pipeline.sh`** loads `.env`, picks a **profile** (**`SNAKEMAKE_PROFILE`**, default `snakemake/profiles/local/`), then runs Snakemake. Pass the **`.smk` workflow first**, then optional input YAML, for example: + +```bash +./run_pipeline.sh snakemake/standardise_gwas.smk path/to/input.yaml +``` + +--- + +## macOS + +### Apptainer and Snakemake + +Snakemake looks for an executable named **`singularity`** on `PATH`. Many installs only provide **`apptainer`**. Shell **aliases** in `~/.zshrc` (e.g. `alias singularity='limactl shell apptainer'`) are **not** visible to `run_pipeline.sh` because it uses **bash** and does not load your zsh aliases. + +Typical setups: + +- **Lima VM** with Apptainer (e.g. `limactl shell apptainer`) — run the pipeline from an environment where **`apptainer`** or **`singularity`** is a real binary on `PATH`, or use a small **`~/bin/singularity`** wrapper that calls `limactl shell -- apptainer "$@"`. +- **Homebrew Apptainer** (if available for your OS version). + +Customising Lima VM for macOS +``` +mountType: virtiofs +mounts: +- location: ~ + mountPoint: ~ +-location /tmp/lima + mountPOint: /tmp/lima +``` + + +### SIF cache and `/Users` mounts + +If the workflow pulls a `docker://` image, Snakemake caches the `.sif` under **`--singularity-prefix`** (default: `.snakemake/singularity` in the repo). On **macOS + Lima**, paths under **`/Users/...`** are often VM mounts where **creating large SIF files fails** with errors like **read-only file system**. + +- Set **`SINGULARITY_DIR`** in `.env` to a **writable** location **inside the Linux VM** (commonly a path under that VM’s **`/tmp`**, e.g. `/tmp/genehackman_snakemake_singularity`), or to native VM disk — not only a bind-mounted folder that cannot handle the build. +- Alternatively, place a pre-built **`$PIPELINE_DATA_DIR/genehackman_.sif`** (underscore; see `DESCRIPTION` / `DOCKER_VERSION`) so Snakemake does not need to build on a problematic mount. +- Avoid **colons** in host-side SIF filenames (`genehackman:1.1.0.sif`); macOS can reject them. The repo’s Snakemake helper uses **`genehackman_.sif`**. + +### Apple Silicon (arm64) + +The published image may be **linux/amd64**. Use Lima with **Rosetta / x86 Linux** (or run on an amd64 Linux machine / HPC) if pulls or runs fail with architecture warnings. + +--- + +## Linux (workstation or single server) + +1. Install **Apptainer** or **SingularityCE** (distribution packages or upstream instructions). +2. Ensure **`singularity`** or **`apptainer`** is on `PATH` (some sites symlink `singularity` → `apptainer`). +3. Use **`snakemake/profiles/local/`** for local execution (same as default in `run_pipeline.sh`): + + ```bash + export SNAKEMAKE_PROFILE=snakemake/profiles/local/ + ./run_pipeline.sh snakemake/compare_gwases.smk + ``` + +4. Adjust **`snakemake/profiles/local/config.yaml`** `singularity-args` if your data live outside **`PROJECT_DIR`** (or **`PIPELINE_DATA_DIR`** / **`QTL_DATA_DIR`**); profiles bind **`DATA_DIR`** and **`RESULTS_DIR`** (derived from **`PROJECT_DIR`**). Add more `-B host:host` pairs if needed. + +--- + +## Slurm (HPC) + +The repo ships Slurm-oriented profiles (paths and partitions are **Bristol / MRC IEU-oriented**; **edit** them for your site): + +| Profile directory | Notes | +|-------------------|--------| +| `snakemake/profiles/slurm/` | Slurm + Apptainer: Make sure that slurm `account` and `partition` are set correctly for your HPC | +| `snakemake/profiles/local/` | Runs apptainer locally, no job invocation | +| `snakemake/profiles/uob-bp1/` | Specific example for UoB BP1 HPC | + +**Typical usage:** + +```bash +export SNAKEMAKE_PROFILE=snakemake/profiles/slurm/ +./run_pipeline.sh snakemake/compare_gwases.smk +``` + +`run_pipeline.sh` runs **`module load ${APPTAINER_MODULE}`** when the profile is **not** `snakemake/profiles/local/*` (or legacy `snakemake/local/*`) — set **`APPTAINER_MODULE`** in `.env` to match your cluster. + +**Customize** each profile under **`snakemake/profiles/`** (each directory has a **`config.yaml`**) as needed: + +- **`Slurm account / partition`:** In **`.env`**, optional **`SLURM_ACCOUNT`** overrides **`sbatch --account`**; if unset, the profile uses **`sacctmgr … | grep … | head -n1`**. **`SLURM_PARTITION`**, if unset, is inferred by **`run_pipeline.sh`** from **`sinfo -h -o '%P'`** (the partition marked with **`*`**, e.g. **`compute*`** → **`compute`**); if **`sinfo`** is unavailable or returns nothing, the fallback is **`compute`**. Passed to Snakemake as **`--default-resources partition=…`**. +- **`cluster:`** block: `sbatch` options (`partition`, `account`, walltime, memory, `--output` log directory). +- **`singularity-args`:** The generic profile binds repo code, **`$HOME`**, **`DATA_DIR`** and **`RESULTS_DIR`** (paths under **`PROJECT_DIR`**, set by `run_pipeline.sh`), **`PIPELINE_DATA_DIR`**, **`/tmp`**, and adds **`QTL_DATA_DIR`** only when non-empty (see **`GENEHACKMAN_EXTRA_SINGULARITY_BINDS`** in `run_pipeline.sh`). Add more `-B host:host` pairs for shared reference data on your site. + +**Snakemake version:** `environment.yml` pins **Snakemake 7.x**. Migrating to **Snakemake 8+** changes cluster syntax (`cluster-generic` executor + plugins); this repository’s profiles target the v7 style unless you have already updated them. + +--- + +## PBS Pro / OpenPBS / Torque + +There is **no** checked-in PBS profile. You can add one by mirroring the Slurm profiles with a **`cluster:`** line that calls your scheduler, for example (Snakemake 7 — adjust directives to your site): + +```yaml +cluster: >- + qsub -N {rule}-{wildcards} + -l walltime={resources.time} + -l select=1:ncpus={threads}:mem={resources.mem} + -o /path/to/your/pbs_logs/{rule}_%I.out + -j oe +``` + +Requirements: + +- **`qsub`** must return the job ID on stdout in a form Snakemake can track. +- For large workflows, set **`--cluster-status`** (or Snakemake’s DRMAA integration) if your site documents it; otherwise use Snakemake’s polling defaults where supported. + +Copy `snakemake/slurm_singularity/config.yaml` as a template, replace the **`cluster:`** section with **`qsub`**, and add **`singularity-args`** appropriate for PBS compute nodes (shared filesystem mounts). + +## Environment variables (quick reference) + +| Variable | Purpose | +|----------|---------| +| **`SNAKEMAKE_PROFILE`** | Snakemake profile path (default `snakemake/profiles/local/`). Example HPC: `snakemake/profiles/slurm/` | +| **`SLURM_ACCOUNT`** | Optional (**`profiles/slurm`**). Sets **`sbatch --account`**; omitted ⇒ existing **`sacctmgr`** derivation in `config.yaml` | +| **`SLURM_PARTITION`** | Optional (**`profiles/slurm`**). Overrides **`sinfo`** default-partition detection in **`run_pipeline.sh`** (see **`sinfo`** `*` suffix); ultimate fallback **`compute`** | + +The **pipeline YAML path** is not configured via `.env`. Use **`./run_pipeline.sh .smk [path/to/input.yaml]`** (defaults to **`input.yaml`**) or run **`snakemake`** with **`--config genehackman_input=path/to/input.yaml`** (see `snakemake/PIPELINES.md`). + +Use **`.env`** for values you want loaded every time `./run_pipeline.sh` runs (`export $(cat .env | xargs)`). + +--- + +## Further reading + +- Pipeline input schema: [`snakemake/PIPELINES.md`](snakemake/PIPELINES.md) +- Main readme: [`README.md`](README.md) +- Snakemake profiles: [documentation](https://snakemake.readthedocs.io/en/stable/executing/cli.html#profiles) +- Snakemake cluster execution: [docs](https://snakemake.readthedocs.io/en/stable/executing/cluster.html) diff --git a/R/chrpos_to_rsid.R b/R/chrpos_to_rsid.R new file mode 100644 index 0000000..8839eb2 --- /dev/null +++ b/R/chrpos_to_rsid.R @@ -0,0 +1,307 @@ +#' @title Chromosome & position data to variant RSID +#' +#' @param dt a data.frame like object, or file path, with at least columns (chrom, pos, ea, nea) +#' @param chr_col a string column name; chromosome position +#' @param pos_col a string column name; base position +#' @param ea_col a string column name; effect allele +#' @param nea_col a string column name; non effect allele +#' @param build a string, options: "b37_dbsnp156", "b38_dbsnp156" (corresponds to the appropriate data directory) +#' @param flip a string, options: "report", "allow", "no_flip" +#' @param alt_rsids a logical, whether to return additional alternate RSIDs +#' @param verbose a logical, runtime reporting +#' @param dbsnp_dir a string file path to the dbSNP .fst file directory - see setup documentation +#' @param parallel_cores an integer, the number of cores/workers to set up the `future::multisession` with +#' +#' @return a tibble with an RSID column, or (if `alt_rsids` is `TRUE`) a list with +#' tibbles: `data` and `alt_rsids` +#' @export +#' @importFrom data.table setkey setkeyv +#' @importFrom progressr progressor +#' @importFrom furrr future_map +#' @importFrom fst fst read_fst +#' @importFrom tibble as_tibble +#' +chrpos_to_rsid <- function(dt, + chr_col, + pos_col, + ea_col=NULL, + nea_col=NULL, + flip="allow", + alt_rsids=FALSE, + build="b37_dbsnp156", + dbsnp_dir = NA_character_, + parallel_cores = parallel::detectCores(), + verbose = TRUE) { + + #-- Checks ----------------------------------------------------------- + + # import / convert input + dt <- data.table::as.data.table(dt) + + stopifnot("`chr_col` must be a column name in `dt`" = chr_col %in% colnames(dt)) + stopifnot("`pos_col` must be a column name in `dt`" = pos_col %in% colnames(dt)) + stopifnot("`pos_col` must be numeric/integer type" = is.numeric(dt[,get(pos_col)])) + stopifnot("`ea_col` and `nea_col` must be either both provided or both NULL" = sum(c(is.null(nea_col),is.null(ea_col)))%in%c(0,2)) + if(all(!c(is.null(ea_col), is.null(nea_col)))) { + stopifnot("`ea_col` must be a column name in `dt`" = ea_col %in% colnames(dt)) + stopifnot("`nea_col` must be a column name in `dt`" = nea_col %in% colnames(dt)) + stopifnot("`ea_col` column must be character type" = is.character(dt[,get(ea_col)])) + stopifnot("`nea_col` column must be character type" = is.character(dt[,get(nea_col)])) + } + stopifnot("`chr_col` column must be character, integer, or numeric type" = is.character(dt[,get(chr_col)]) | + is.integer(dt[,get(chr_col)]) | + is.numeric(dt[,get(chr_col)])) + if(is.numeric(dt[,get(chr_col)])) { + stopifnot("`chr_col`, if numeric, must be whole numbers" = all(dt[,get(chr_col)] %% 1 == 0)) + } + stopifnot("`dbsnp_dir` must be a valid directory path" = dir.exists(dbsnp_dir)) + available_builds <- list.dirs(dbsnp_dir, full.names=FALSE)[-1] + build <- match.arg(build, available_builds) + flip <- match.arg(flip, c("report", "allow", "no_flip")) + + + #-- Processing --------------------------------------------------------- + + # data adjustments + # 1) remove chromosome 'Chr6' --> '6' and recode X/Y + # (?i) case-insensitive + # ^ at the beginning of the string + # (?:chr)? non-capture group, matches "chr" 0-1 times + # ([0-9XYMT]+) capture group //1, matches 1 or more occurances of group [0-9XYMT] + # \\1 replace with string found in capture group 1 + chr_col_orig = "chr_col_orig" + dt[, (chr_col_orig) := get(chr_col)] + dt[, (chr_col) := sub("(?i)^(?:chr)?([0-9XY]+)", toupper("\\1"), toupper(get(chr_col)))] + # 2) replace X[23,25] and Y[24] + dt[, (chr_col) := ifelse(get(chr_col) %in% c("X","25") ,"23",ifelse(get(chr_col)=="Y","24",get(chr_col)))] + # 3) delete the RSID column ahead of re-adding it, if it exists + if("RSID" %in% names(dt)) { + dt[, "RSID" := NULL] + } + + # split the dt into a list of data.tables, by chromosome + dts <- split(dt, by=chr_col) + + if(verbose) { + message(paste("RSID mapping...\nChromosomes to process: ", paste0(names(dts), collapse=", "))) + } + + # set up parallel processing and process each chromosome independently + future::plan(future::multisession, workers=parallel_cores) + progressr::with_progress({ + + # set up progress bar + p <- progressr::progressor(steps = 7*length(dts)) # going to update 7 times in the function + + # run each chromosome in parallel + # out[[1]] is the RSID data + # out[[2]] is the alternative RSID data (other RSIDs for the same CHR:BP:A1:A2) + out <- furrr::future_map(.x = dts, + .f = process_chromosome, + chr_col=chr_col, pos_col=pos_col, nea_col=nea_col, ea_col=ea_col, build=build, dbsnp_dir=dbsnp_dir, flip=flip, alt_rsids=alt_rsids, p=p, + .options=furrr::furrr_options(seed=TRUE)) + + }) + + future::plan(future::sequential) + rm(dts) + gc(verbose = FALSE) + + # deal with alternative RSIDs if requested + if(alt_rsids) { + + out_data <- data.table::rbindlist( lapply(out, `[[`, 1) ) + out_alts <- data.table::rbindlist( lapply(out, `[[`, 2) ) + + } else { + out_data <- data.table::rbindlist(out) + } + rm(out) + gc(verbose = FALSE) + + # put back the original chromosome column and remove the temporary one + out_data[, (chr_col) := chr_col_orig] + out_data[, chr_col_orig := NULL] + + # RSID to the front + data.table::setcolorder(out_data, c("RSID", names(out_data)[names(out_data) != "RSID"])) + + # report + msg <- paste0("RSID coverage ", round(100*sum(!is.na(out_data[["RSID"]]))/nrow(out_data), digits=2), "% (", sum(!is.na(out_data[["RSID"]])), "/", nrow(out_data), ")") + if(flip!="no_flip") { + msg <- paste0(msg, ", of which ", round(100*sum(out_data[["rsid_flip_match"]], na.rm=TRUE)/nrow(out_data), digits=2),"% (", sum(out_data[["rsid_flip_match"]], na.rm=TRUE), "/", nrow(out_data), ") where found flipping alleles") + } + message(msg) + + # remove flipping flag from output if not wanted + if(flip=="allow") { + out_data[, "rsid_flip_match" := NULL] + } + + # return as same type as input + # data.table::setattr(out_data, "orig_type", attr(dt,"orig_type")) + if (alt_rsids) { + return(list( + "data" = tibble::as_tibble(out_data), + "alt_rsids" = tibble::as_tibble(out_alts) + )) + } + return(tibble::as_tibble(out_data)) +} + + +# this function could be defined in the above function. However I took it out to avoid +# the gwas data.table being captured in each futures environment. See this discussion +# for the details: https://furrr.futureverse.org/articles/gotchas.html#function-environments-and-large-objects +process_chromosome <- function(chrom_dt, chr_col, pos_col, build, dbsnp_dir, flip, alt_rsids, p, nea_col=NULL, ea_col=NULL) { + message(paste0("Processing chromosome ", chrom_dt[[1, chr_col]])) + + # silence RMDcheck warning + RSID = i.RSID = baseRSID = rsid_flip_match = REF = ALT = NULL + + # increment progress bar #1 + p() + + # the chromosome to process, a string value e.g. "1" + chrom <- chrom_dt[[1, chr_col]] + + # the dbSNP `.fst` file for this chromosome + dbSNP_build_dir <- file.path(dbsnp_dir, build) + dbSNP_path <- file.path(dbSNP_build_dir, paste0("chr", chrom, ".fst")) + stopifnot("Missing .fst reference file - is the dbSNP directory set correctly?" = file.exists(dbSNP_path)) + + # read just the dbSNP position data (an integer vector) (reduce amount of data read in) + dbSNP_pos <- fst::read_fst(dbSNP_path, "BP", as.data.table=TRUE) + + # increment progress bar #2 + p() + + # set as keys + data.table::setkey(dbSNP_pos, "BP") + data.table::setkeyv(chrom_dt, pos_col) + + # get the indices of the positions that are needed / also found in the input data + dbSNP_pos[chrom_dt, "found" := get(pos_col)] + row_idxs <- which(!is.na(dbSNP_pos[["found"]])) + rm(dbSNP_pos) + + # define whether to work with alleles, or just CHR:POS + alleles <- !(is.null(nea_col) & is.null(ea_col)) + + # define the key depending on which columns are provided + if(alleles) { + dbSNP_key <- c("CHR","BP","REF","ALT") + dbSNP_keyflip <- c("CHR","BP","ALT","REF") + data_key <- c(chr_col, pos_col, nea_col, ea_col) + } else { + dbSNP_key <- c("CHR","BP") + data_key <- c(chr_col, pos_col) + } + + # if matches found + if(length(row_idxs)>0) { + + # increment progress bar #3 + p() + + # create a fst object which allows row access without reading the whole file + dbSNP_fst <- fst::fst(dbSNP_path) + + # read the needed rows + dbSNP_data <- dbSNP_fst[row_idxs, c("RSID", dbSNP_key)] |> data.table::as.data.table() + rm(dbSNP_fst) + + # increment progress bar #4 + p() + + if(alleles) { + # split the ALT column which can be a comma separate vector of alternative alleles + dbSNP_data[, "ALT" := lapply(.SD, strsplit, split=','), .SDcols="ALT"] + + # make data.table longer, one allele combination per row + dbSNP_data <- dbSNP_data[, lapply(.SD, unlist), by=1:nrow(dbSNP_data)] + dbSNP_data[, nrow := NULL] + + # recode as D/I if input data has D/I coding + if(any(grepl("^(D|I)$", chrom_dt[[nea_col]]))) { + dbSNP_data[, "ALT" := data.table::fcase(nchar(ALT)< nchar(REF), "D", + nchar(ALT)> nchar(REF), "I", + nchar(ALT)==nchar(REF), ALT)] + dbSNP_data[, "REF" := data.table::fcase(ALT=="D", "I", + ALT=="I", "D", + rep(TRUE, nrow(dbSNP_data)), REF)] + } + } + + # increment progress bar #5 + p() + + if(alt_rsids) { + # get the alt rsids (some rsids code for the same position, chr, alt, and ref...) + alt_rsid_data <- dbSNP_data[duplicated(dbSNP_data[, dbSNP_key, with = FALSE]),] + } + + # take unique (first rsid occurance) + dbSNP_data <- dbSNP_data[!duplicated(dbSNP_data[, dbSNP_key, with = FALSE]),] + + # set the keys to match expected way round + data.table::setkeyv(dbSNP_data, dbSNP_key) + data.table::setkeyv(chrom_dt, data_key) + chrom_dt[dbSNP_data, "RSID" := i.RSID] + + if(alt_rsids) { + # set key + data.table::setkeyv(alt_rsid_data, dbSNP_key) + + # map the chosen RSID to the alternative RSIDS + alt_rsid_data[chrom_dt, "baseRSID" := i.RSID] + } + + # increment progress bar #6 + p() + + # flip the alleles + if(flip!="no_flip" & alleles) { + + # set the keys to flipped alleles + data.table::setkeyv(dbSNP_data, dbSNP_keyflip) + + # add in flipped matches and a logical flag + chrom_dt[dbSNP_data, c("RSID", "rsid_flip_match") := list(i.RSID, TRUE)] + chrom_dt[!is.na(RSID) & is.na(rsid_flip_match), rsid_flip_match := FALSE] + + if(alt_rsids) { + # get any alt rsids that match flipped + data.table::setkeyv(alt_rsid_data, dbSNP_keyflip) + alt_rsid_data[chrom_dt, "baseRSID" := i.RSID] + } + + } + + # no matches found. skip all the processing but add the columns + } else { + p() + p() + p() + p() + chrom_dt[["RSID"]] <- NA_character_ + if(alt_rsids) alt_rsid_data <- data.table::copy(chrom_dt) + if(flip!="no_flip" & alleles) chrom_dt[, rsid_flip_match := NA] + if(alt_rsids) { + alt_rsid_data[, "baseRSID" := NA_character_] + alt_rsid_data <- alt_rsid_data[!is.na(baseRSID), ] + } + } + + # increment progress bar #7 + p() + + # return + if(alt_rsids) { + alt_rsid_data <- alt_rsid_data[is.na(baseRSID), ] + return(list("data"=chrom_dt, "alt_rsids"=alt_rsid_data)) + } else { + return(chrom_dt) + } + +} \ No newline at end of file diff --git a/R/collider_bias.R b/R/collider_bias.R index 33b4065..cba848f 100644 --- a/R/collider_bias.R +++ b/R/collider_bias.R @@ -20,13 +20,22 @@ collider_bias_results <- data.frame( #' * "Corrected Weighted Least Squares" (CWLS, or Dudbridge) Correction #' * MR IVW #' +#' @param incidence_gwas incidence GWAS +#' @param subsequent_gwas subsequent GWAS +#' @param clumped_snps_file clumped SNP list to run collider bias corrections against +#' @param adjustment_type adjustment type to save the adjusted GWAS +#' @param adjustment_pval adjustment pval to save the adjusted GWAS +#' @param collider_bias_results_file file to save the collider bias results +#' @param harmonised_effects_result_file file to save the harmonised effects +#' @param adjusted_output_file file to save the adjusted GWAS +#' @param p_value_thresholds p value thresholds to run corrections #' @return 2 plots: one manhattan plot and one QQ plot (with lambda included) -#' @import dplyr -#' @import SlopeHunter -#' @import TwoSampleMR -#' @import MendelianRandomization -#' @import data.table -#' @import vroom + + + + + + #' @export conduct_collider_bias_analysis <- function(incidence_gwas, subsequent_gwas, @@ -239,16 +248,16 @@ conduct_collider_bias_analysis <- function(incidence_gwas, #' This can be used in conjuction with weights calculated to account for collider bias. #' Currently used to work on a GWAS result of Slopehunter. #' -#' @param gwas: gwas that the new collider bias correction will be saved to. Swaps out BETA, SE, and P -#' @param harmonised_effects: a dataframe that includes BETA.incidence, BETA.prognosis, SE.incidence, SE.prognosis -#' @param collider_bias_type: string name of adjustment (eg. slopehunter) -#' @param beta: number, slope of the correction -#' @param se: number, SE of the corrected slope -#' @param output_file: name of file that the corrected GWAS will be saved to -#' @import dplyr -#' @import vroom -#' @import stats -#' @import R.utils +#' @param gwas gwas that the new collider bias correction will be saved to. Swaps out BETA, SE, and P +#' @param harmonised_effects a dataframe that includes BETA.incidence, BETA.prognosis, SE.incidence, SE.prognosis +#' @param collider_bias_type string name of adjustment (eg. slopehunter) +#' @param beta number, slope of the correction +#' @param se number, SE of the corrected slope +#' @param output_file name of file that the corrected GWAS will be saved to + + + + #' adjust_gwas_data_from_weights_and_save <- function(gwas, harmonised_effects, diff --git a/R/coloc.R b/R/coloc.R index fa841bf..e6bac85 100644 --- a/R/coloc.R +++ b/R/coloc.R @@ -1,7 +1,16 @@ -#' -#' @import dplyr -#' @import vroom -#' @import tibble +#' run_coloc_on_list_of_datasets: run coloc on a list of datasets +#' @param first_gwas_list list of first GWAS files +#' @param second_gwas_list list of second GWAS files +#' @param exposure_name_list list of exposure names +#' @param chr_list list of chromosomes +#' @param bp_list list of base pair positions +#' @param range range to filter in base pairs +#' @param default_n default sample size +#' @param output_file file to save the results +#' @return tibble of coloc results + + + #' @export run_coloc_on_list_of_datasets <- function(first_gwas_list=list(), second_gwas_list=list(), @@ -38,18 +47,31 @@ run_coloc_on_list_of_datasets <- function(first_gwas_list=list(), return(coloc_results) } -#' @import dplyr -#' @import tidyr -#' @import vroom -#' @import utils +#' Run BF-BF colocalization on significant QTL MR results using finemapped data +#' +#' For each significant MR result, loads the finemapped GWAS locus that covers +#' the MR hit, computes LBF scores for the QTL data via \code{convert_z_to_lbf}, +#' and runs \code{coloc::coloc.bf_bf}. +#' +#' @param mr_results_file MR results file +#' @param finemap_dir directory containing finemapped GWAS locus files +#' @param qtl_dataset QTL dataset name (metabrain or eqtlgen) +#' @param study_type study type for LBF conversion ("continuous" or "categorical") +#' @param exposures optional character vector of exposures to filter +#' @param default_n default sample size for QTL data +#' @param output_file output file path + + + + +#' @export run_coloc_on_qtl_mr_results <- function(mr_results_file, - gwas_file, + finemap_dir, qtl_dataset, - exposures=c(), - default_n=NA, + study_type = "continuous", + exposures = c(), + default_n = NA, output_file) { - gwas <- get_file_or_dataframe(gwas_file) - range <- 500000 mr_results <- get_file_or_dataframe(mr_results_file) |> dplyr::filter(p.adjusted < 0.05) @@ -58,44 +80,287 @@ run_coloc_on_qtl_mr_results <- function(mr_results_file, mr_results <- subset(mr_results, exposure %in% exposures) } + empty_result <- tibble::tibble( + exposure = character(), + locus = character(), + hit1 = character(), + hit2 = character(), + n_snps = integer(), + PP.H0.abf = numeric(), + PP.H1.abf = numeric(), + PP.H2.abf = numeric(), + PP.H3.abf = numeric(), + PP.H4.abf = numeric() + ) + if (nrow(mr_results) == 0) { - coloc_results <- tibble::tribble(~exposure, ~h0, ~h1, ~h2, ~h3, ~h4) + vroom::vroom_write(empty_result, output_file) + return(invisible(empty_result)) + } + + coloc_results <- apply(mr_results, 1, function(mr_result) { + qtl_gwas <- load_qtl_gwas_for_mr_result(mr_result, qtl_dataset) + if (is.null(qtl_gwas)) return(NULL) + + chr <- as.numeric(mr_result[["CHR"]]) + bp <- as.numeric(mr_result[["BP"]]) + exposure_name <- mr_result[["EXPOSURE"]] + + coloc_bf_bf_qtl_analysis( + finemap_dir = finemap_dir, + qtl_gwas = qtl_gwas, + exposure_name = exposure_name, + chr = chr, + bp = bp, + study_type = study_type, + default_n = default_n + ) + }) |> dplyr::bind_rows() + + if (nrow(coloc_results) == 0) coloc_results <- empty_result + + vroom::vroom_write(coloc_results, output_file) + return(invisible(coloc_results)) +} + + +#' Load the QTL GWAS file for an MR result row +#' @keywords internal +load_qtl_gwas_for_mr_result <- function(mr_result, qtl_dataset) { + if (qtl_dataset == qtl_datasets$metabrain) { + outcome <- unlist(strsplit(mr_result[["outcome"]], "_")) + brain_region <- outcome[1] + ancestry <- toupper(outcome[2]) + qtl_gwas_file <- paste0(metabrain_gwas_dir, "/", brain_region, "/", + mr_result[["EXPOSURE"]], "_", ancestry, ".tsv.gz") + } else if (qtl_dataset == qtl_datasets$eqtlgen) { + qtl_gwas_file <- paste0(eqtlgen_gwas_dir, "/", mr_result[["outcome"]], "/", + mr_result[["EXPOSURE"]], ".tsv.gz") } else { - coloc_results <- apply(mr_results, 1, function(mr_result) { - if (qtl_dataset == qtl_datasets$metabrain) { - outcome <- unlist(strsplit(mr_result[["outcome"]], "_")) - brain_region <- outcome[1] - ancestry <- toupper(outcome[2]) - - qtl_gwas_file <- paste0(metabrain_gwas_dir, "/", brain_region, "/", mr_result[["EXPOSURE"]], "_", ancestry, ".tsv.gz") - qtl_gwas <- get_file_or_dataframe(qtl_gwas_file) - } else if (qtl_dataset == qtl_datasets$eqtlgen) { - qtl_gwas_file <- paste0(eqtlgen_gwas_dir, "/", mr_result[["outcome"]], "/", mr_result[["EXPOSURE"]], ".tsv.gz") - qtl_gwas <- get_file_or_dataframe(qtl_gwas_file) - } else { - stop("Error: qtl dataset not supported for coloc right now") - } - chr <- as.numeric(mr_result[["CHR"]]) - bp <- as.numeric(mr_result[["BP"]]) - result <- coloc_analysis(gwas, qtl_gwas, mr_result[["EXPOSURE"]], chr, bp, range, default_n=default_n) + stop("Error: qtl dataset not supported for coloc right now") + } + + if (!file.exists(qtl_gwas_file)) { + message(paste("QTL GWAS file not found:", qtl_gwas_file)) + return(NULL) + } + get_file_or_dataframe(qtl_gwas_file) +} + + +#' Run BF-BF coloc between a finemapped GWAS locus and QTL data +#' +#' Finds the finemapped locus file covering the region around chr:bp, +#' extracts the GWAS LBF matrix, computes QTL LBF via \code{convert_z_to_lbf}, +#' and runs \code{coloc::coloc.bf_bf}. +#' +#' @param finemap_dir finemap output directory containing locus TSV files +#' @param qtl_gwas QTL GWAS data frame +#' @param exposure_name exposure label +#' @param chr chromosome of the MR hit +#' @param bp base position of the MR hit +#' @param study_type study type for LBF conversion +#' @param default_n default sample size +#' @param range_bp overlap window in bp (default 1e6 = ±1Mb) +#' @return tibble with coloc results or NULL + + + +#' @export +coloc_bf_bf_qtl_analysis <- function(finemap_dir, + qtl_gwas, + exposure_name, + chr, + bp, + study_type = "continuous", + default_n = NA, + range_bp = 1e6) { - #TODO: should we create Miami Plots for coloc results? - #miami_filename <- paste0(user_results_dir, "plots/mr_metabrain_coloc_", file_prefix(gwas_file), "_", mr_result[["EXPOSURE"]], ".png") - #miami_plot(qtl_gwas_file, gwas_file, miami_filename, paste0("Miami Plot of", mr_result[["EXPOSURE"]]), chr, bp, range) + finemap_locus <- find_finemap_locus_for_region(finemap_dir, chr, bp, range_bp) + if (is.null(finemap_locus)) { + message(paste("No finemapped locus found near", chr, ":", bp)) + return(NULL) + } - return(result) - }) |> dplyr::bind_rows() + lbf_cols <- grep("^LBF_[0-9]+$", colnames(finemap_locus), value = TRUE) + if (length(lbf_cols) == 0) { + message(paste("No LBF columns in finemapped locus for", chr, ":", bp)) + return(NULL) } - vroom::vroom_write(coloc_results, output_file) + numeric_cols <- c("CHR", "BP", "BETA", "SE", "P", "EAF", "N") + for (col in intersect(numeric_cols, colnames(qtl_gwas))) { + qtl_gwas[[col]] <- as.numeric(qtl_gwas[[col]]) + } + + qtl_gwas <- dplyr::filter(qtl_gwas, + !is.na(BETA) & !is.na(SE) & SE > 0 & + !is.na(EAF) & EAF > 0 & EAF < 1 + ) + + qtl_region <- gwas_region(qtl_gwas, chr, bp, range_bp) + if (nrow(qtl_region) < 2) { + message(paste("Too few QTL SNPs in region around", chr, ":", bp)) + return(NULL) + } + + shared_snps <- intersect(finemap_locus$SNP, qtl_region$SNP) + if (length(shared_snps) < 2) { + message(paste("Too few shared SNPs for", exposure_name, "at", chr, ":", bp)) + return(NULL) + } + + gwas_sub <- dplyr::filter(finemap_locus, SNP %in% shared_snps) |> + dplyr::filter(!duplicated(SNP)) |> + dplyr::arrange(SNP) + qtl_sub <- dplyr::filter(qtl_region, SNP %in% shared_snps) |> + dplyr::filter(!duplicated(SNP)) |> + dplyr::arrange(SNP) + + shared_snps <- intersect(gwas_sub$SNP, qtl_sub$SNP) + gwas_sub <- dplyr::filter(gwas_sub, SNP %in% shared_snps) + qtl_sub <- dplyr::filter(qtl_sub, SNP %in% shared_snps) + + gwas_bf <- build_bf_matrix(gwas_sub, lbf_cols) + if (is.null(gwas_bf)) return(NULL) + + qtl_n <- `if`("N" %in% colnames(qtl_sub) && !is.na(as.numeric(qtl_sub$N[1])), + as.numeric(qtl_sub$N[1]), default_n) + if (is.na(qtl_n)) { + message(paste("Cannot determine N for QTL data for", exposure_name)) + return(NULL) + } + + z_scores <- qtl_sub$BETA / qtl_sub$SE + qtl_lbf <- convert_z_to_lbf( + z = z_scores, + se = qtl_sub$SE, + eaf = qtl_sub$EAF, + sample_size = qtl_n, + study_type = study_type + ) + names(qtl_lbf) <- qtl_sub$SNP + + result <- tryCatch( + coloc::coloc.bf_bf(bf1 = gwas_bf, bf2 = qtl_lbf), + error = function(e) { + message(paste("coloc.bf_bf failed for", exposure_name, "at", chr, ":", bp, + ":", conditionMessage(e))) + return(NULL) + } + ) + + if (is.null(result)) return(NULL) + + locus_name <- paste0(chr, "_", bp) + parsed <- parse_bf_bf_result(result, exposure_name, locus_name, length(shared_snps)) + return(parsed) +} + + +#' Build a named BF matrix from a finemapped locus data frame +#' @keywords internal +build_bf_matrix <- function(locus_df, lbf_cols) { + lbf_cols <- intersect(lbf_cols, colnames(locus_df)) + if (length(lbf_cols) == 0) return(NULL) + + has_cs <- "CS" %in% colnames(locus_df) + if (has_cs) { + has_signal <- !all(is.na(locus_df$CS)) + } else { + has_signal <- TRUE + } + if (!has_signal) return(NULL) + + bf_mat <- t(as.matrix(locus_df[, lbf_cols, drop = FALSE])) + bf_mat[is.na(bf_mat)] <- 0 + colnames(bf_mat) <- locus_df$SNP + rownames(bf_mat) <- lbf_cols + return(bf_mat) +} + + +#' Find the finemapped locus file that covers a genomic region +#' @keywords internal +find_finemap_locus_for_region <- function(finemap_dir, chr, bp, range_bp = 1e6) { + if (!dir.exists(finemap_dir)) return(NULL) + + files <- list.files(finemap_dir, pattern = "_finemap\\.tsv\\.gz$", full.names = TRUE) + if (length(files) == 0) return(NULL) + + best_file <- NULL + best_dist <- Inf + + for (f in files) { + locus_name <- sub("_finemap\\.tsv\\.gz$", "", basename(f)) + parts <- strsplit(locus_name, "_")[[1]] + locus_chr <- as.numeric(parts[1]) + locus_bp <- as.numeric(parts[2]) + + if (locus_chr != chr) next + dist <- abs(locus_bp - bp) + if (dist <= range_bp && dist < best_dist) { + best_dist <- dist + best_file <- f + } + } + + if (is.null(best_file)) return(NULL) + vroom::vroom(best_file, show_col_types = FALSE) +} + + +#' Parse coloc.bf_bf result for QTL analysis +#' @keywords internal +parse_bf_bf_result <- function(result, exposure_name, locus_name, n_snps) { + if (!is.null(result$summary)) { + summary_df <- result$summary + if (is.data.frame(summary_df) || is.matrix(summary_df)) { + rows <- list() + for (i in seq_len(nrow(summary_df))) { + row <- summary_df[i, ] + rows <- c(rows, list(tibble::tibble( + exposure = exposure_name, + locus = locus_name, + hit1 = as.character(row[["hit1"]]), + hit2 = as.character(row[["hit2"]]), + n_snps = as.integer(row[["nsnps"]]), + PP.H0.abf = as.numeric(row[["PP.H0.abf"]]), + PP.H1.abf = as.numeric(row[["PP.H1.abf"]]), + PP.H2.abf = as.numeric(row[["PP.H2.abf"]]), + PP.H3.abf = as.numeric(row[["PP.H3.abf"]]), + PP.H4.abf = as.numeric(row[["PP.H4.abf"]]) + ))) + } + return(dplyr::bind_rows(rows)) + } + } + + tibble::tibble( + exposure = exposure_name, + locus = locus_name, + hit1 = NA_character_, + hit2 = NA_character_, + n_snps = n_snps, + PP.H0.abf = NA_real_, + PP.H1.abf = NA_real_, + PP.H2.abf = NA_real_, + PP.H3.abf = NA_real_, + PP.H4.abf = NA_real_ + ) } #' run_coloc_analysis takes two already harmonised gwases, and runs coloc on the results -#' @param first_gwas: first gwas to be run through coloc. This is the gwas that results will be based off -#' @param second_gwas: second gwas to be run through coloc -#' @returns tibble of coloc results (h0 - h4) -#' @import coloc -#' @import tibble +#' @param first_gwas first gwas to be run through coloc. This is the gwas that results will be based off +#' @param second_gwas second gwas to be run through coloc +#' @param exposure_name name of exposure to perform coloc on +#' @param chr chromosome to perform coloc on +#' @param bp base pair position to perform coloc on +#' @param range range to filter in base pairs +#' @param default_n default sample size +#' @return tibble of coloc results (h0 - h4) + + #' @export coloc_analysis <- function(first_gwas, second_gwas, exposure_name, chr=NA, bp=NA, range=NA, default_n=NA) { numeric_columns <- c("P", "SE", "EAF") diff --git a/R/coloc_bf.R b/R/coloc_bf.R new file mode 100644 index 0000000..4865587 --- /dev/null +++ b/R/coloc_bf.R @@ -0,0 +1,244 @@ +#' Run pairwise BF-BF colocalization on finemapped GWAS results +#' +#' For each pair of GWAS datasets, identifies overlapping finemapped signals +#' (lead SNPs within ±\code{overlap_kb} kb), then runs \code{coloc::coloc.bf_bf} +#' on shared SNPs using the per-signal LBF columns produced by the fine-mapping +#' pipeline. +#' +#' @param finemap_dirs named character vector of finemap output directories, +#' one per GWAS. Names are used as trait labels in the output. +#' @param overlap_kb distance in kb to define overlapping signals (default 1000 = ±1 Mb) +#' @param p1 prior probability a SNP is associated with trait 1 +#' @param p2 prior probability a SNP is associated with trait 2 +#' @param p12 prior probability a SNP is associated with both traits +#' @param output_file path to write the combined coloc results TSV +#' @return tibble of coloc results for every overlapping signal pair + + + +#' @export +run_bf_bf_coloc <- function(finemap_dirs, + overlap_kb = 1000, + p1 = 1e-4, + p2 = 1e-4, + p12 = 5e-6, + output_file = NULL) { + + trait_names <- names(finemap_dirs) + if (is.null(trait_names) || any(trait_names == "")) { + trait_names <- basename(finemap_dirs) + names(finemap_dirs) <- trait_names + } + + locus_data <- load_all_finemap_loci(finemap_dirs) + + if (length(locus_data) < 2) { + message("Need at least 2 traits with finemapped loci for colocalization.") + empty <- tibble::tibble( + trait1 = character(), trait2 = character(), + locus1 = character(), locus2 = character(), + n_snps = integer(), + PP.H0.abf = numeric(), PP.H1.abf = numeric(), + PP.H2.abf = numeric(), PP.H3.abf = numeric(), PP.H4.abf = numeric() + ) + if (!is.null(output_file)) vroom::vroom_write(empty, output_file) + return(invisible(empty)) + } + + pairs <- utils::combn(trait_names, 2, simplify = FALSE) + overlap_bp <- overlap_kb * 1000 + all_results <- list() + + for (pair in pairs) { + t1 <- pair[1] + t2 <- pair[2] + loci1 <- locus_data[[t1]] + loci2 <- locus_data[[t2]] + + for (l1 in loci1) { + for (l2 in loci2) { + if (l1$chr != l2$chr) next + if (abs(l1$lead_bp - l2$lead_bp) > overlap_bp) next + + result <- coloc_bf_bf_for_locus_pair(l1, l2, t1, t2, p1, p2, p12) + if (!is.null(result)) { + all_results <- c(all_results, list(result)) + } + } + } + } + + combined <- dplyr::bind_rows(all_results) + + if (nrow(combined) == 0) { + message("No overlapping finemapped signals found across trait pairs.") + } else { + message(paste("BF-BF colocalization complete:", nrow(combined), "signal pair(s) tested.")) + } + + if (!is.null(output_file)) { + vroom::vroom_write(combined, output_file) + } + + return(invisible(combined)) +} + + +#' Load all finemapped locus files from a set of directories +#' @keywords internal +load_all_finemap_loci <- function(finemap_dirs) { + locus_data <- list() + + for (trait in names(finemap_dirs)) { + dir_path <- finemap_dirs[[trait]] + if (!dir.exists(dir_path)) { + message(paste("Finemap directory not found for", trait, ":", dir_path)) + next + } + + files <- list.files(dir_path, pattern = "_finemap\\.tsv\\.gz$", full.names = TRUE) + if (length(files) == 0) { + message(paste("No finemap locus files found in", dir_path)) + next + } + + trait_loci <- list() + for (f in files) { + locus <- vroom::vroom(f, show_col_types = FALSE) + lbf_cols <- grep("^LBF_[0-9]+$", colnames(locus), value = TRUE) + if (length(lbf_cols) == 0) next + + locus_name <- sub("_finemap\\.tsv\\.gz$", "", basename(f)) + parts <- strsplit(locus_name, "_")[[1]] + chr <- as.numeric(parts[1]) + bp <- as.numeric(parts[2]) + + trait_loci <- c(trait_loci, list(list( + data = locus, + lbf_cols = lbf_cols, + chr = chr, + lead_bp = bp, + locus_name = locus_name + ))) + } + + if (length(trait_loci) > 0) { + locus_data[[trait]] <- trait_loci + } + } + + return(locus_data) +} + + +#' Run coloc::coloc.bf_bf for one pair of overlapping loci +#' @keywords internal +coloc_bf_bf_for_locus_pair <- function(l1, l2, trait1_name, trait2_name, + p1, p2, p12) { + d1 <- l1$data + d2 <- l2$data + + if (!"SNP" %in% colnames(d1) || !"SNP" %in% colnames(d2)) { + return(NULL) + } + + shared_snps <- intersect(d1$SNP, d2$SNP) + if (length(shared_snps) < 2) return(NULL) + + d1 <- dplyr::filter(d1, SNP %in% shared_snps) |> + dplyr::filter(!duplicated(SNP)) |> + dplyr::arrange(SNP) + d2 <- dplyr::filter(d2, SNP %in% shared_snps) |> + dplyr::filter(!duplicated(SNP)) |> + dplyr::arrange(SNP) + + shared_snps <- intersect(d1$SNP, d2$SNP) + d1 <- dplyr::filter(d1, SNP %in% shared_snps) + d2 <- dplyr::filter(d2, SNP %in% shared_snps) + + bf1 <- build_lbf_matrix(d1, l1$lbf_cols) + bf2 <- build_lbf_matrix(d2, l2$lbf_cols) + + if (is.null(bf1) || is.null(bf2)) return(NULL) + + result <- tryCatch( + coloc::coloc.bf_bf(bf1 = bf1, bf2 = bf2, p1 = p1, p2 = p2, p12 = p12), + error = function(e) { + message(paste("coloc.bf_bf failed for", trait1_name, l1$locus_name, + "vs", trait2_name, l2$locus_name, ":", conditionMessage(e))) + return(NULL) + } + ) + + if (is.null(result)) return(NULL) + + parse_pairwise_bf_bf_result(result, trait1_name, trait2_name, + l1$locus_name, l2$locus_name, length(shared_snps)) +} + + +#' Build a named BF matrix from locus data +#' @keywords internal +build_lbf_matrix <- function(locus_df, lbf_cols) { + lbf_cols <- intersect(lbf_cols, colnames(locus_df)) + if (length(lbf_cols) == 0) return(NULL) + + has_cs <- "CS" %in% colnames(locus_df) + if (has_cs) { + has_signal <- !all(is.na(locus_df$CS)) + } else { + has_signal <- TRUE + } + if (!has_signal) return(NULL) + + bf_mat <- t(as.matrix(locus_df[, lbf_cols, drop = FALSE])) + bf_mat[is.na(bf_mat)] <- 0 + colnames(bf_mat) <- locus_df$SNP + rownames(bf_mat) <- lbf_cols + return(bf_mat) +} + + +#' Parse the result of coloc.bf_bf for a pairwise GWAS comparison +#' @keywords internal +parse_pairwise_bf_bf_result <- function(result, trait1, trait2, locus1, locus2, n_snps) { + if (!is.null(result$summary)) { + summary_df <- result$summary + if (is.data.frame(summary_df) || is.matrix(summary_df)) { + rows <- list() + for (i in seq_len(nrow(summary_df))) { + row <- summary_df[i, ] + rows <- c(rows, list(tibble::tibble( + trait1 = trait1, + trait2 = trait2, + locus1 = locus1, + locus2 = locus2, + hit1 = as.character(row[["hit1"]]), + hit2 = as.character(row[["hit2"]]), + n_snps = as.integer(row[["nsnps"]]), + PP.H0.abf = as.numeric(row[["PP.H0.abf"]]), + PP.H1.abf = as.numeric(row[["PP.H1.abf"]]), + PP.H2.abf = as.numeric(row[["PP.H2.abf"]]), + PP.H3.abf = as.numeric(row[["PP.H3.abf"]]), + PP.H4.abf = as.numeric(row[["PP.H4.abf"]]) + ))) + } + return(dplyr::bind_rows(rows)) + } + } + + tibble::tibble( + trait1 = trait1, + trait2 = trait2, + locus1 = locus1, + locus2 = locus2, + hit1 = NA_character_, + hit2 = NA_character_, + n_snps = n_snps, + PP.H0.abf = NA_real_, + PP.H1.abf = NA_real_, + PP.H2.abf = NA_real_, + PP.H3.abf = NA_real_, + PP.H4.abf = NA_real_ + ) +} diff --git a/R/constants.R b/R/constants.R index f4eee34..c81014d 100644 --- a/R/constants.R +++ b/R/constants.R @@ -6,21 +6,90 @@ get_env_var <- function(env_var_name, default_value=NULL) { } } -user_data_dir <- get_env_var("DATA_DIR", "") -user_results_dir <- get_env_var("RESULTS_DIR", "") +# Slurm allocated memory (MB) when running under Slurm, else total system memory. +available_memory <- function() { + slurm <- Sys.getenv("SLURM_MEM_PER_NODE", "") + if (nzchar(slurm)) { + v <- suppressWarnings(as.numeric(slurm)) + if (!is.na(v) && v > 0) return(v) + } + os <- Sys.info()[["sysname"]] + if (os == "Linux") { + meminfo <- tryCatch(readLines("/proc/meminfo", n = 1), error = function(e) "") + kb <- suppressWarnings(as.numeric(gsub("[^0-9]", "", meminfo))) + if (!is.na(kb) && kb > 0) return(kb / 1024) + } else if (os == "Darwin") { + raw <- tryCatch(system2("sysctl", "-n hw.memsize", stdout = TRUE, stderr = FALSE), + error = function(e) "") + bytes <- suppressWarnings(as.numeric(raw)) + if (!is.na(bytes) && bytes > 0) return(bytes / (1024^2)) + } + return(NA) +} + +# Slurm node CPUs when SLURM_* env is set, else parallel::detectCores(). +available_cpus <- function() { + slurm <- Sys.getenv("SLURM_CPUS_ON_NODE", "") + if (nzchar(slurm)) { + v <- suppressWarnings(as.integer(slurm)) + if (!is.na(v)) { + return(max(1L, v)) + } + } + dc <- suppressWarnings(parallel::detectCores(logical = TRUE)) + if (is.na(dc) || dc < 1L) { + return(1L) + } else { + return(as.integer(dc)) + } +} + +# Shared by RSID population, finemap, etc. (load.R sources all R/*.R; keep only this definition). +calculate_parallelism <- function(max_workers = 10L, memory_per_worker_mb = 8000L) { + cpus <- available_cpus() + mem <- available_memory() + by_mem <- if (!is.na(mem) && mem > 0) { + round(floor(mem / memory_per_worker_mb)) + } else { + return(cpus) + } + return(max(1L, min(max_workers, by_mem, cpus))) +} -number_of_cpus_available <- as.numeric(get_env_var("SLURM_CPUS_ON_NODE", 1)) -genomic_data_dir <- get_env_var("GENOMIC_DATA_DIR", "") -thousand_genomes_dir <- get_env_var("THOUSAND_GENOMES_DIR", "") -qtl_directory <- get_env_var("QTL_DIRECTORY", "") +.strip_path <- function(x) { + return(sub("/+$", "", trimws(x))) +} -liftover_dir <- paste0(genomic_data_dir, '/liftover') -pqtl_top_hits_dir <- paste0(qtl_directory, "/pqtl") -metabrain_top_hits_dir <- paste0(qtl_directory, "/metabrain/top_hits") -metabrain_gwas_dir <- paste0(qtl_directory, "/metabrain/gwas") -eqtlgen_top_hits_dir <- paste0(qtl_directory, "/eqtlgen/top_hits") -eqtlgen_gwas_dir <- paste0(qtl_directory, "/eqtlgen/gwas") +project_dir <- .strip_path(get_env_var("PROJECT_DIR", "")) +if (nzchar(project_dir)) { + user_data_dir <- file.path(project_dir, "data") + user_results_dir <- file.path(project_dir, "results") +} else { + user_data_dir <- "" + user_results_dir <- "" +} + +pipeline_data_dir <- sub("/+$", "", get_env_var("PIPELINE_DATA_DIR", "")) +qtl_directory <- sub("/+$", "", get_env_var("QTL_DATA_DIR", "")) +is_on_cluster <- nzchar(Sys.getenv("SLURM_JOB_ID")) || nzchar(Sys.getenv("PBS_JOBID")) + +genomic_data_dir <- file.path(pipeline_data_dir, "genomic_data") +# Must match snakemake/util/constants.smk THOUSAND_GENOMES_DIR (used by clumping + LD). +thousand_genomes_dir <- file.path(genomic_data_dir, "1000genomes", "b37_dbsnp156") + +liftover_chain_dir <- file.path(genomic_data_dir, "liftover") +liftover_binary <- Sys.which("liftOver") +if (!nzchar(liftover_binary)) { + liftover_binary <- "/usr/local/bin/liftOver" +} +pqtl_top_hits_dir <- file.path(qtl_directory, "pqtl") +metabrain_top_hits_dir <- file.path(qtl_directory, "metabrain", "top_hits") +metabrain_gwas_dir <- file.path(qtl_directory, "metabrain", "gwas") +eqtlgen_top_hits_dir <- file.path(qtl_directory, "eqtlgen", "top_hits") +eqtlgen_gwas_dir <- file.path(qtl_directory, "eqtlgen", "gwas") qtl_datasets <- list(metabrain="metabrain", eqtlgen="eqtlgen") -populate_rsid_options <- list(full="full", partial="partial", none="none") +populate_rsid_options <- list(none="none", partial="partial", full="full") +populate_chr_bp_options <- list(none="none", partial="partial", full="full") rsid_builds <- list(GRCh37="b37_dbsnp156", GRCh38="b38_dbsnp156") +study_types <- list(continuous="continuous", categorical="categorical") \ No newline at end of file diff --git a/R/data_conversions.R b/R/data_conversions.R index cc059aa..8f0bff5 100644 --- a/R/data_conversions.R +++ b/R/data_conversions.R @@ -1,3 +1,8 @@ +#' populate_gene_names: populate the gene names from the ENSEMBL_ID column +#' @param gwas dataframe with the following columns: ENSEMBL_ID +#' @return gwas with new column GENE_NAME + + #' @export populate_gene_names <- function(gwas) { if ("ENSEMBL_ID" %in% colnames(gwas) && !"GENE_NAME" %in% colnames(gwas)) { @@ -9,23 +14,116 @@ populate_gene_names <- function(gwas) { } ensembl_id_to_gene_name <- function(gwas) { - gene_map <- vroom::vroom(paste0(genomic_data_dir, "gene_name_map.tsv")) + gene_map <- vroom::vroom(file.path(genomic_data_dir, "gene_name_map.tsv")) gwas$GENE_NAME <- gene_map$GENE_NAME[match(gwas$ENSEMBL_ID, gene_map$ENSEMBL_ID)] return(gwas) } gene_name_to_ensembl_id <- function(gwas) { - gene_map <- vroom::vroom(paste0(genomic_data_dir, "gene_name_map.tsv")) + gene_map <- vroom::vroom(file.path(genomic_data_dir, "gene_name_map.tsv")) gwas$ENSEMBL_ID <- gene_map$ENSEMBL_ID[match(gwas$GENE_NAME, gene_map$GENE_NAME)] return(gwas) } +#' Populate missing EAF from the LD reference panel \code{.frq} file via RSID matching. +#' +#' Expects the GWAS to have an \code{RSID} column (populated by \code{populate_rsid}) +#' and alleles in canonical (alphabetically sorted) order from \code{standardise_alleles}. +#' Only reads the lightweight \code{.frq} file; the \code{.bim} is not needed. +#' +#' @param gwas Standardised GWAS tibble with RSID, EA, OA columns. +#' @param ancestry One of EUR, EAS, AFR, AMR, SAS matching the reference panel prefix. +#' @export +populate_eaf_from_reference_panel <- function(gwas, ancestry) { + if ("EAF" %in% colnames(gwas) && !all(is.na(gwas$EAF))) { + if (sum(is.na(gwas$EAF)) == 0) { + message("EAF column already fully populated; skipping reference panel lookup.") + return(gwas) + } + } + + allowed <- c("EUR", "EAS", "AFR", "AMR", "SAS") + if (!ancestry %in% allowed) { + stop("ancestry must be one of: ", paste(allowed, collapse = ", ")) + } + frq_path <- paste0(file.path(thousand_genomes_dir, ancestry), ".frq") + if (!file.exists(frq_path)) { + stop("LD reference .frq not found: ", frq_path, + "\nGenerate with: plink --bfile ", + file.path(thousand_genomes_dir, ancestry), " --freq --out ", + file.path(thousand_genomes_dir, ancestry)) + } + + if (!"RSID" %in% colnames(gwas) || all(is.na(gwas$RSID))) { + warning("No RSIDs available for EAF lookup; skipping EAF population.") + return(gwas) + } + + if (!"EAF" %in% names(gwas)) { + gwas$EAF <- NA_real_ + } + need <- which(is.na(gwas$EAF) & !is.na(gwas$RSID)) + if (length(need) == 0L) { + message("EAF population: no rows need filling (all have EAF or lack RSID).") + return(gwas) + } + + rsids_needed <- unique(gwas$RSID[need]) + message("Reading reference .frq for ", length(rsids_needed), " RSIDs...") + + ref <- data.table::fread(frq_path, header = TRUE, showProgress = FALSE, data.table = TRUE) + names(ref) <- stringr::str_trim(names(ref)) + req <- c("SNP", "A1", "A2", "MAF") + if (!all(req %in% names(ref))) { + stop(".frq file must contain columns SNP, A1, A2, MAF (plink 1.9 --freq)") + } + ref <- ref[, .(SNP, A1 = toupper(A1), A2 = toupper(A2), MAF = as.numeric(MAF))] + ref <- ref[SNP %in% rsids_needed] + ref <- unique(ref, by = "SNP") + + if (nrow(ref) == 0L) { + message("EAF population (", ancestry, " reference): 0 RSIDs matched in .frq; ", + length(need), " still missing.") + return(gwas) + } + + idx <- match(gwas$RSID[need], ref$SNP) + matched <- !is.na(idx) + + ea <- toupper(gwas$EA[need[matched]]) + oa <- toupper(gwas$OA[need[matched]]) + ra1 <- ref$A1[idx[matched]] + ra2 <- ref$A2[idx[matched]] + maf <- ref$MAF[idx[matched]] + + eaf <- data.table::fcase( + ea == ra1 & oa == ra2, maf, + ea == ra2 & oa == ra1, 1 - maf, + default = NA_real_ + ) + + gwas$EAF[need[matched]] <- eaf + + n_fill <- sum(!is.na(eaf)) + n_total_need <- sum(is.na(gwas$EAF)) + n_fill + message( + "EAF population (", ancestry, " reference): filled ", n_fill, + " of ", n_total_need, " missing values; ", n_total_need - n_fill, " still missing." + ) + gwas +} + +#' populate_rsid: populate the RSID column from the SNP column +#' @param gwas dataframe with the following columns: SNP +#' @param option option to populate the RSID column +#' @return gwas with new column RSID + + #' @export populate_rsid <- function(gwas, option = populate_rsid_options$none) { gc() start_time <- Sys.time() if (option == populate_rsid_options$none || "RSID" %in% colnames(gwas)) { - message("Skipping RSID population for GWAS") return(gwas) } else if (option == populate_rsid_options$partial) { gwas <- populate_partial_rsids(gwas) @@ -43,23 +141,35 @@ populate_rsid <- function(gwas, option = populate_rsid_options$none) { populate_partial_rsids <- function(gwas) { message("populating RSIDs based on 1000genomes...") - marker_to_rsid_file <- paste0(thousand_genomes_dir, "marker_to_rsid.tsv.gz") + marker_to_rsid_file <- file.path(thousand_genomes_dir, "marker_to_rsid.tsv.gz") chrpos_to_rsid <- vroom::vroom(marker_to_rsid_file, col_select = c("HG37", "RSID"), show_col_types=F) gwas$RSID <- chrpos_to_rsid$RSID[match(gwas$SNP, chrpos_to_rsid$HG37)] return(gwas) } -#' @import data.table -#' @import tibble -#' @import genepi.utils -#' @import future -populate_full_rsids <- function(gwas, build = rsid_builds$GRCh37) { - dbsnp_dir <- paste0(genomic_data_dir, "dbsnp") + +populate_full_rsids <- function(gwas) { + build <- rsid_builds$GRCh37 + dbsnp_dir <- file.path(genomic_data_dir, "dbsnp") if (!build %in% rsid_builds) stop(paste("Error: invalid rsid build option:", build)) + parallel_chr_population <- calculate_parallelism() + message(paste0("Using ", parallel_chr_population, " cores for full RSID population")) + gwas <- data.table::as.data.table(gwas) - gwas <- genepi.utils::chrpos_to_rsid(gwas, "CHR", "BP", "EA", "OA", flip = "allow", dbsnp_dir=dbsnp_dir, build=build, alt_rsids = F, parallel_cores=number_of_cpus_available) + gwas <- chrpos_to_rsid( + gwas, + "CHR", + "BP", + "EA", + "OA", + flip = "allow", + dbsnp_dir = dbsnp_dir, + build = build, + alt_rsids = FALSE, + parallel_cores=parallel_chr_population + ) gwas <- tibble::as_tibble(gwas) return(gwas) -} +} \ No newline at end of file diff --git a/R/finemap.R b/R/finemap.R new file mode 100644 index 0000000..e1289be --- /dev/null +++ b/R/finemap.R @@ -0,0 +1,410 @@ +#' Run SuSiE fine-mapping across all clumped loci in a GWAS +#' +#' For each lead SNP from plink --clump output, extracts a window, computes an +#' LD matrix from the 1000 Genomes reference panel via plink, and runs +#' susieR::susie_rss. Writes one TSV per locus: all GWAS variants within the +#' genomic window around the lead SNP, with SuSiE Z-scores, credible-set +#' membership, and per-credible-set (independent signal) columns \code{LBF_1}, \code{LBF_2}, ... +#' +#' @param gwas filename or dataframe of a standardised GWAS +#' @param clumped_file plink --clump output file +#' @param ancestry ancestry code matching 1000 Genomes panel prefix (EUR, EAS, AFR, AMR, SAS) +#' @param default_n GWAS sample size when not inferrable from \code{gwas}; see +#' @param output_finemap_dir directory to write one +#' \verb{__finemap.tsv.gz} per clumped locus. +#' @param completion_file path to a sentinel file written on successful completion +#' (one line: expected lead count). If NULL, defaults to +#' \verb{/finemap_complete.txt}. +#' @param window_kb half-width of the fine-mapping window in kb (default 1000 = ±1 Mb) +#' @param max_causal maximum number of causal signals per locus (SuSiE L, default 10) +#' @param coverage credible set coverage (default 0.95) +#' @param min_abs_corr minimum absolute correlation for credible sets (default 0.5) +#' @return invisibly, the combined per-SNP finemap data.table (LD-matched SNPs only) +#' @export +finemap_gwas <- function(gwas, + clumped_file, + ancestry, + default_n, + output_finemap_dir, + completion_file = NULL, + window_kb = 1000, + max_causal = 10, + coverage = 0.95, + min_abs_corr = 0.5) { + + gwas <- get_file_or_dataframe(gwas) + data.table::setDT(gwas) + + numeric_cols <- c("CHR", "BP", "BETA", "SE", "P", "EAF", "N", "N_CASE", "N_CONTROL") + for (col in intersect(numeric_cols, colnames(gwas))) { + data.table::set(gwas, j = col, value = as.numeric(gwas[[col]])) + } + + data.table::setkey(gwas, CHR, BP) + + has_n <- "N" %in% colnames(gwas) + if (!has_n && (length(default_n) != 1L || is.na(default_n))) { + stop("Fine-mapping requires GWAS sample size") + } + + if (!dir.exists(output_finemap_dir)) { + dir.create(output_finemap_dir, recursive = TRUE) + } + + if (is.null(completion_file)) { + completion_file <- file.path(output_finemap_dir, "finemap_complete.txt") + } + + lead_snps <- data.table::fread(clumped_file, select = c("SNP", "CHR", "BP")) + if (nrow(lead_snps) == 0) { + message("No clumped SNPs found; no per-locus finemap files written.") + write_finemap_complete_marker(completion_file, 0L, output_finemap_dir) + empty <- data.table::data.table( + SNP = character(), CHR = numeric(), BP = numeric(), RSID = character(), + Z = numeric(), CS = integer() + ) + return(invisible(empty)) + } + + window_bp <- window_kb * 1000 + has_rsid <- "RSID" %in% colnames(gwas) + + n_loci <- nrow(lead_snps) + + # Pre-extract per-locus subsets so workers don't need the full GWAS + locus_subsets <- vector("list", n_loci) + window_subsets <- vector("list", n_loci) + for (i in seq_len(n_loci)) { + lead_chr <- as.numeric(lead_snps$CHR[i]) + lead_bp <- as.numeric(lead_snps$BP[i]) + bp_lo <- lead_bp - window_bp + bp_hi <- lead_bp + window_bp + + chr_rows <- gwas[.(lead_chr), nomatch = NULL] + window_dt <- chr_rows[BP >= bp_lo & BP <= bp_hi] + window_subsets[[i]] <- window_dt + locus_subsets[[i]] <- window_dt[!is.na(BETA) & !is.na(SE) & SE > 0] + } + rm(gwas); gc(verbose = FALSE) + + process_one_locus <- function(i) { + lead_rsid <- lead_snps$SNP[i] + lead_chr <- as.numeric(lead_snps$CHR[i]) + lead_bp <- as.numeric(lead_snps$BP[i]) + + tryCatch({ + locus_gwas <- locus_subsets[[i]] + window_gwas <- window_subsets[[i]] + + if (nrow(locus_gwas) < 2L) { + message(paste("Skipping locus", lead_rsid, "- fewer than 2 SNPs in window")) + return(NULL) + } + + if (!has_rsid || all(is.na(locus_gwas$RSID))) { + message(paste("Skipping locus", lead_rsid, "- no RSIDs available for LD computation")) + return(NULL) + } + locus_gwas <- locus_gwas[!is.na(RSID) & nchar(RSID) > 0L] + + ld_result <- compute_ld_matrix(locus_gwas$RSID, lead_chr, ancestry) + if (is.null(ld_result)) { + message(paste("Skipping locus", lead_rsid, "- LD matrix computation failed")) + return(NULL) + } + + shared_rsids <- intersect(locus_gwas$RSID, ld_result$snps) + if (length(shared_rsids) < 2L) { + rm(ld_result); gc(verbose = FALSE) + message(paste("Skipping locus", lead_rsid, "- too few shared SNPs between GWAS and LD panel")) + return(NULL) + } + + locus_gwas <- locus_gwas[match(shared_rsids, RSID)] + ld_idx <- match(shared_rsids, ld_result$snps) + R <- ld_result$matrix[ld_idx, ld_idx] + rm(ld_result); gc(verbose = FALSE) + + z_scores <- locus_gwas$BETA / locus_gwas$SE + n <- if (has_n && !is.na(as.numeric(locus_gwas$N[1L]))) as.numeric(locus_gwas$N[1L]) else default_n + if (is.na(n)) { + stop("Fine-mapping requires GWAS sample size") + } + + locus_lbf <- run_susie_for_locus( + z_scores = z_scores, + ld_matrix = R, + snp_info = locus_gwas, + n = n, + lead_snp = lead_rsid, + max_causal = max_causal, + coverage = coverage, + min_abs_corr = min_abs_corr + ) + rm(R, locus_gwas); gc(verbose = FALSE) + + if (nrow(locus_lbf) == 0L) return(NULL) + + lbf_nm <- grep("^LBF_[0-9]+$", names(locus_lbf), value = TRUE) + finemap_cols <- intersect(c("RSID", "Z", "CS", lbf_nm), names(locus_lbf)) + finemap_join <- locus_lbf[, ..finemap_cols] + + out_gwas <- merge(window_gwas, finemap_join, by = "RSID", all.x = TRUE, sort = FALSE) + rm(window_gwas, finemap_join) + + lead_chr_bp <- paste0( + lead_chr, "_", + format(as.numeric(lead_bp), scientific = FALSE, trim = TRUE, digits = 20) + ) + safe_locus <- gsub("[^A-Za-z0-9._-]+", "_", lead_chr_bp) + out_file <- file.path(output_finemap_dir, paste0(safe_locus, "_finemap.tsv.gz")) + data.table::fwrite(out_gwas, out_file, sep = "\t", compress = "gzip") + rm(out_gwas) + + locus_lbf + }, error = function(e) { + message(paste("Error processing locus", lead_rsid, ":", conditionMessage(e))) + return(NULL) + }) + } + + finemap_workers <- min( + calculate_parallelism(memory_per_worker_mb = 6000L), + n_loci + ) + locus_results <- parallel::mclapply( + seq_len(n_loci), + process_one_locus, + mc.cores = finemap_workers + ) + rm(locus_subsets, window_subsets); gc(verbose = FALSE) + + is_valid <- vapply(locus_results, function(x) { + is.data.frame(x) || data.table::is.data.table(x) + }, FUN.VALUE = logical(1)) + + n_errors <- sum(vapply(locus_results, inherits, "try-error", FUN.VALUE = logical(1))) + if (n_errors > 0L) { + message(paste(n_errors, "locus worker(s) returned errors")) + } + + all_lbf <- locus_results[is_valid] + rm(locus_results) + + if (length(all_lbf) == 0L) { + combined_lbf <- data.table::data.table( + SNP = character(), CHR = numeric(), BP = numeric(), RSID = character(), + Z = numeric(), CS = integer() + ) + } else { + combined_lbf <- data.table::rbindlist(all_lbf, fill = TRUE) + } + rm(all_lbf); gc(verbose = FALSE) + + if (nrow(combined_lbf) == 0L) { + message("No loci produced SuSiE results.") + } + + message(paste("Fine-mapping complete:", + nrow(lead_snps), "clumped loci,", + nrow(combined_lbf), "LD-matched SNPs with finemap stats,", + "outputs in", output_finemap_dir)) + + write_finemap_complete_marker(completion_file, nrow(lead_snps), output_finemap_dir) + + return(invisible(combined_lbf)) +} + + +#' Write a completion sentinel file (one line: expected lead count) for Snakemake. +#' @keywords internal +write_finemap_complete_marker <- function(completion_file, n_loci, output_finemap_dir = NULL) { + if (is.null(completion_file)) { + if (is.null(output_finemap_dir) || length(output_finemap_dir) != 1L || !nzchar(output_finemap_dir)) { + stop("completion_file is NULL; provide output_finemap_dir or an explicit completion_file path") + } + completion_file <- file.path(output_finemap_dir, "finemap_complete.txt") + } + if (!is.character(completion_file) || length(completion_file) != 1L || !nzchar(completion_file)) { + stop("completion_file must be a non-empty character path") + } + dir_path <- dirname(completion_file) + if (!dir.exists(dir_path)) dir.create(dir_path, recursive = TRUE) + writeLines(as.character(as.integer(n_loci)), completion_file) +} + + +#' Compute LD correlation matrix from 1000 Genomes via plink +#' @param rsids character vector of RSIDs +#' @param chr chromosome number +#' @param ancestry ancestry code (EUR, EAS, etc.) +#' @return list with components \code{matrix} (numeric LD matrix) and +#' \code{snps} (character vector of SNP IDs in matrix order), or NULL on failure +compute_ld_matrix <- function(rsids, chr, ancestry) { + bfile <- file.path(thousand_genomes_dir, ancestry) + tmpdir <- tempdir() + snp_file <- tempfile(tmpdir = tmpdir, fileext = ".snps") + out_prefix <- tempfile(tmpdir = tmpdir, pattern = "ld_") + + writeLines(rsids, snp_file) + + cmd <- paste( + "plink1.9", + "--threads", as.character(available_cpus()), + "--bfile", bfile, + "--chr", chr, + "--extract", snp_file, + "--r square", + "--out", out_prefix + ) + exit_code <- run_system(cmd, wait = TRUE, ignore.stdout = TRUE, ignore.stderr = TRUE) + + ld_file <- paste0(out_prefix, ".ld") + snp_order_file <- paste0(out_prefix, ".nosex") + + if (exit_code != 0 || !file.exists(ld_file)) { + unlink(c(snp_file, paste0(out_prefix, c(".ld", ".nosex", ".log", ".bim", ".bed", ".fam"))), + force = TRUE) + return(NULL) + } + + bim_file <- paste0(out_prefix, ".bim") + if (file.exists(bim_file)) { + snp_order <- data.table::fread(bim_file, header = FALSE, select = 2)$V2 + } else { + bim_ref <- paste0(bfile, ".bim") + all_bim <- data.table::fread(bim_ref, header = FALSE, select = c(1, 2, 4)) + colnames(all_bim) <- c("CHR", "SNP", "BP") + all_bim <- all_bim[CHR == chr & SNP %in% rsids] + data.table::setorder(all_bim, BP) + snp_order <- all_bim$SNP + rm(all_bim) + } + + ld_raw <- data.table::fread(ld_file, header = FALSE) + R <- as.matrix(ld_raw) + rm(ld_raw) + + if (nrow(R) != length(snp_order) || ncol(R) != length(snp_order)) { + rm(R) + unlink(c(snp_file, paste0(out_prefix, c(".ld", ".nosex", ".log", ".bim", ".bed", ".fam"))), + force = TRUE) + return(NULL) + } + + rownames(R) <- snp_order + colnames(R) <- snp_order + + unlink(c(snp_file, paste0(out_prefix, c(".ld", ".nosex", ".log", ".bim", ".bed", ".fam"))), + force = TRUE) + + return(list(matrix = R, snps = snp_order)) +} + + +#' Wrapper so tests can mock SuSiE with \code{testthat::local_mocked_bindings()}. +#' @keywords internal +run_susie_rss_impl <- function(z, R, n, L, coverage, min_abs_corr, verbose = FALSE) { + susieR::susie_rss( + z = z, R = R, n = n, L = L, coverage = coverage, min_abs_corr = min_abs_corr, verbose = verbose + ) +} + + +#' Build per-credible-set LBF columns (\code{LBF_1}, \code{LBF_2}, ...) from a SuSiE fit. +#' @keywords internal +susie_lbf_columns <- function(fitted, p) { + lv <- fitted$lbf_variable + if (is.null(lv) || !is.matrix(lv) || nrow(lv) < 1L || ncol(lv) != p) { + return(data.table::data.table()) + } + + n_row <- nrow(lv) + cs_list <- fitted$sets$cs + cs_idx <- fitted$sets$cs_index + col_vecs <- vector("list", 0L) + + if (!is.null(cs_list) && length(cs_list) > 0L) { + n_cs <- length(cs_list) + use_idx <- !is.null(cs_idx) && length(cs_idx) == n_cs + for (j in seq_len(n_cs)) { + row_i <- if (use_idx) cs_idx[j] else j + if (is.finite(row_i) && row_i >= 1L && row_i <= n_row) { + col_vecs[[paste0("LBF_", j)]] <- lv[row_i, ] + } else { + col_vecs[[paste0("LBF_", j)]] <- rep(NA_real_, p) + } + } + } else { + for (j in seq_len(n_row)) { + col_vecs[[paste0("LBF_", j)]] <- lv[j, ] + } + } + + data.table::as.data.table(col_vecs) +} + + +#' Run SuSiE on a single locus +#' @param z_scores numeric vector of z-scores +#' @param ld_matrix square LD correlation matrix +#' @param snp_info data frame with SNP, CHR, BP, RSID columns (same order as z_scores) +#' @param n sample size +#' @param lead_snp RSID of the lead (clumped) SNP +#' @param max_causal max causal signals (L) +#' @param coverage credible set coverage +#' @param min_abs_corr minimum absolute correlation for CS purity +#' @return data.table with SNP, CHR, BP, RSID, Z, CS, and \code{LBF_1}, \code{LBF_2}, ... per signal +run_susie_for_locus <- function(z_scores, ld_matrix, snp_info, n, lead_snp, + max_causal = 10, coverage = 0.95, + min_abs_corr = 0.5) { + + p <- length(z_scores) + fitted <- tryCatch( + run_susie_rss_impl( + z = z_scores, + R = ld_matrix, + n = n, + L = max_causal, + coverage = coverage, + min_abs_corr = min_abs_corr, + verbose = FALSE + ), + error = function(e) { + message(paste("SuSiE failed for locus", lead_snp, ":", conditionMessage(e))) + return(NULL) + } + ) + + if (is.null(fitted)) { + return(data.table::data.table( + SNP = character(), CHR = numeric(), BP = numeric(), RSID = character(), + Z = numeric(), CS = integer() + )) + } + + lbf_wide <- susie_lbf_columns(fitted, p) + + cs_membership <- rep(NA_integer_, p) + if (!is.null(fitted$sets) && !is.null(fitted$sets$cs)) { + for (cs_i in seq_along(fitted$sets$cs)) { + snp_indices <- fitted$sets$cs[[cs_i]] + cs_membership[snp_indices] <- cs_i + } + } + rm(fitted); gc(verbose = FALSE) + + result <- data.table::data.table( + SNP = snp_info$SNP, + CHR = snp_info$CHR, + BP = snp_info$BP, + RSID = snp_info$RSID, + Z = z_scores, + CS = cs_membership + ) + + if (ncol(lbf_wide) > 0L) { + result <- cbind(result, lbf_wide) + } + result +} \ No newline at end of file diff --git a/R/graphs.R b/R/graphs.R index 52b37bc..3919494 100644 --- a/R/graphs.R +++ b/R/graphs.R @@ -1,5 +1,27 @@ -#' @import ggplot2 -#' @import shiny +# Compute dynamic plot dimensions (px) given the number of items on the y-axis. +# Returns list(width, height) in pixels capped at ggplot2's max (49 in * dpi). +dynamic_plot_dims <- function(n_items, dpi = 300, + px_per_item = 35L, + min_height_px = 800L, + base_width_px = 2400L) { + max_dim_px <- floor(49 * dpi) + effective_px <- if (n_items <= 30) max(px_per_item, 80L) + else if (n_items <= 60) max(px_per_item, 55L) + else px_per_item + height <- min(max(min_height_px, as.integer(n_items) * effective_px), max_dim_px) + width <- min(base_width_px, max_dim_px) + list(width = width, height = height) +} + + +#' forest_plot: produce a forest plot from a GWAS file +#' @param table dataframe with the following columns: BETA, SE +#' @param title title of the plot +#' @param output_file file to save the plot +#' @param y_column column to use for the y-axis +#' @return ggplot2 object + + #' @export forest_plot <- function(table, title, output_file, y_column=NA) { if (!all(c("BETA", "SE") %in% names(table))) { @@ -27,7 +49,15 @@ forest_plot <- function(table, title, output_file, y_column=NA) { ggplot2::ggsave(output_file, limitsize = F) } -#' @import ggplot2 +#' grouped_forest_plot: produce a grouped forest plot from a GWAS file +#' @param table dataframe with the following columns: BETA, SE +#' @param title title of the plot +#' @param group_column column to use for the group +#' @param output_file file to save the plot +#' @param p_value_column column to use for the p-value +#' @param q_stat_column column to use for the Q-statistic +#' @return ggplot2 object + #' @export grouped_forest_plot <- function(table, title, group_column, output_file, p_value_column = NA, q_stat_column = NA) { if (!("BETA" %in% names(table)) || !("SE" %in% names(table))) { @@ -41,6 +71,8 @@ grouped_forest_plot <- function(table, title, group_column, output_file, p_value table$LL <- table$BETA - (1.96 * table$SE) table$UL <- table$BETA + (1.96 * table$SE) + n_groups <- length(unique(table[[group_column]])) + plot_thing <- ggplot2::ggplot(table, ggplot2::aes(y = if(!is.na(q_stat_column)) paste0(.data[[first_column_name]], "\n Q-stat=", .data[[q_stat_column]]) else .data[[first_column_name]], x = BETA, @@ -52,21 +84,37 @@ grouped_forest_plot <- function(table, title, group_column, output_file, p_value ggplot2::scale_colour_brewer(type="qual") + ggplot2::geom_vline(xintercept = 0) + ggplot2::geom_pointrange(cex = 1, fatten = 2, position=ggplot2::position_dodge(width = 0.5)) + - ggplot2::theme(legend.position = "bottom") + ggplot2::ggtitle(title) + - ggplot2::theme(plot.title = ggplot2::element_text(hjust = 0.5)) + ggplot2::theme(plot.title = ggplot2::element_text(hjust = 0.5)) + + ggplot2::guides(colour = ggplot2::guide_legend(ncol = 1), + fill = ggplot2::guide_legend(ncol = 1)) + + ggplot2::theme(legend.position = "right") - if (!is.na(p_value_column)) { + p <- if (!is.na(p_value_column)) { plot_thing + ggplot2::geom_text(ggplot2::aes(label = paste("P=", signif(.data[[p_value_column]]), digits=3), group = .data[[group_column]]), position = ggplot2::position_dodge(width = 0.5)) } else if (!is.na(q_stat_column)) { plot_thing + ggplot2::geom_text(ggplot2::aes(x = 1.5, label = .data[[q_stat_column]])) + } else { + plot_thing } - forest_plot_height <- max(nrow(table)*50, 2000) - ggplot2::ggsave(output_file, width = 2000, units = "px", height = forest_plot_height) + n_unique_y <- length(unique(table[[first_column_name]])) + dpi <- 300 + dims <- dynamic_plot_dims(n_unique_y, dpi = dpi, px_per_item = 40L, min_height_px = 800L, + base_width_px = 2400L) + + ggplot2::ggsave( + output_file, + plot = p, + width = dims$width, + height = dims$height, + units = "px", + dpi = dpi, + limitsize = FALSE + ) } -#' @import ggplot2 + grouped_bar_chart <- function(data, title, x_column, y_column, group_column, output_file) { ggplot2::ggplot(data, ggplot2::aes(x = .data[[x_column]], y = .data[[y_column]], fill = .data[[group_column]])) + ggplot2::ggtitle(title) + @@ -81,13 +129,14 @@ grouped_bar_chart <- function(data, title, x_column, y_column, group_column, out #' manhattan_and_qq: produce manhattan and qq plot from a GWAS file #' -#' @param gwas_filename: a file of a gwas that includes CHR, CP, P, and SNP -#' @param name: name of plots to be saved (and named as a header in graph) -#' @param save_dir: defaults to 'scratch/results' +#' @param gwas_filename file of a gwas that includes CHR, CP, P, and SNP +#' @param manhattan_filename file to save the manhattan plot +#' @param qq_filename file to save the qq plot +#' @param include_qq logical flag on if to include the qq plot #' @return 2 plots: one manhattan plot and one QQ plot (with lambda included) -#' @import grDevices -#' @import qqman -#' @import graphics + + + #' @export manhattan_and_qq <- function(gwas_filename, manhattan_filename, qq_filename, include_qq = T) { manhattan_columns <- c("SNP", "CHR", "BP", "P") @@ -112,12 +161,16 @@ manhattan_and_qq <- function(gwas_filename, manhattan_filename, qq_filename, inc #' miami_plot: produce miami plot of GWAS data from two gwases #' -#' @param gwas_dataframe: a dataframe that includes CHR, CP, P, and SNP -#' @param name: name of plots to be saved (and named as a header in graph) -#' @param save_dir: defaults to 'scratch/results' -#' @import grDevices -#' @import qqman -#' @import graphics +#' @param first_gwas_filename filename of first GWAS +#' @param second_gwas_filename filename of second GWAS +#' @param miami_plot_file file to save the miami plot +#' @param title title of the plot +#' @param chr chromosome to perform miami plot on +#' @param bp base pair position to perform miami plot on +#' @param range range to filter in base pairs + + + #' @export miami_plot <- function(first_gwas_filename, second_gwas_filename, @@ -180,8 +233,16 @@ miami_plot <- function(first_gwas_filename, grDevices::dev.off() } -#' @import ggplot2 -#' @import ggrepel +#' volcano_plot: produce a volcano plot from a GWAS file +#' @param results_file file to save the plot +#' @param title title of the plot +#' @param label column to use for the label (default "EXPOSURE") +#' @param num_labels number of labels to include in the plot (default 30) +#' @param output_file file to save the plot +#' @param p_val column to use for the p-value (default "p.adjusted") +#' @return ggplot2 object + + #' @export volcano_plot <- function(results_file, title="Volcano Plot of Results", label="EXPOSURE", num_labels=30, output_file, p_val="p.adjusted") { table <- get_file_or_dataframe(results_file) @@ -196,13 +257,27 @@ volcano_plot <- function(results_file, title="Volcano Plot of Results", label="E get({{p_val}}) < 0.05 & BETA > 0 ~ "Protective" )) - #filter label to only showing the more 'important' labels - important_labels <- dplyr::filter(table, get({{p_val}}) < 0.05) |> - dplyr::arrange(dplyr::desc(-log10(get({{p_val}}) * abs(BETA)))) - important_labels <- head(important_labels, num_labels)[[label]] - table[[label]] <- ifelse(table[[label]] %in% important_labels, table[[label]], NA) - - ggplot2::ggplot(data = table, ggplot2::aes(x = BETA , y = -log10(.data[[p_val]]), col = category, label = .data[[label]])) + + # One label per distinct exposure/method key: MR outputs often repeat the same EXPOSURE + # (e.g. IVW rows); label only the best-scoring row per label value to avoid duplicate tags. + table <- dplyr::mutate(table, .volcano_row_id = dplyr::row_number()) + sig <- dplyr::filter(table, .data[[p_val]] < 0.05) + representatives <- sig |> + dplyr::group_by(.data[[label]]) |> + dplyr::slice_max( + order_by = (-log10(.data[[p_val]]) * abs(BETA)), + n = 1L, + with_ties = FALSE + ) |> + dplyr::ungroup() |> + dplyr::arrange(dplyr::desc(-log10(.data[[p_val]]) * abs(BETA))) |> + dplyr::slice_head(n = num_labels) + + rep_ids <- representatives$.volcano_row_id + lab_chr <- as.character(table[[label]]) + table$.volcano_plot_label <- ifelse(table$.volcano_row_id %in% rep_ids, lab_chr, NA_character_) + table$.volcano_row_id <- NULL + + p <- ggplot2::ggplot(data = table, ggplot2::aes(x = BETA , y = -log10(.data[[p_val]]), col = category, label = .volcano_plot_label)) + ggplot2::geom_vline(xintercept = c(-0.1, 0.1), col = "gray", linetype = 'dashed') + ggplot2::geom_hline(yintercept = -log10(0.05), col = "tomato2", linetype = 'dashed') + ggplot2::geom_point(size = 1) + @@ -213,20 +288,47 @@ volcano_plot <- function(results_file, title="Volcano Plot of Results", label="E ggplot2::theme(plot.title = ggplot2::element_text(hjust = 0.5)) + ggrepel::geom_text_repel(max.overlaps = Inf, show.legend = F) - ggplot2::ggsave(output_file) + ggplot2::ggsave(output_file, plot = p) } -#' @import ggplot2 -plot_heritability_contribution_per_ancestry <- function(heterogeneity_results_qj, output_file) { - graph_width <- max(1000, nrow(heterogeneity_results_qj) * 100) - plot <- tidyr::gather(as.data.frame(heterogeneity_results_qj), "key", "value", -SNP) - - ggplot2::ggplot(plot, ggplot2::aes(x=SNP, y=-log10(value))) + - ggplot2::geom_point(ggplot2::aes(colour=key)) + - ggplot2::scale_colour_brewer(type="qual") + - ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, vjust = 0.5, hjust=1)) + - ggplot2::ggtitle("Contribution to heterogeneity score broken down by population") + - ggplot2::theme(plot.title = ggplot2::element_text(hjust = 0.5)) - ggplot2::ggsave(output_file, width = graph_width, units = "px") +plot_heritability_contribution_per_ancestry <- function(heterogeneity_results_qj, output_file) { + dpi <- 300 + n_snps <- nrow(heterogeneity_results_qj) + dims <- dynamic_plot_dims(n_snps, dpi = dpi, px_per_item = 35L, min_height_px = 800L, + base_width_px = 2400L) + + qj_df <- as.data.frame(heterogeneity_results_qj, check.names = TRUE) + snp_levels <- qj_df$SNP + plot_data <- tidyr::pivot_longer(qj_df, cols = -SNP, names_to = "key", values_to = "value") + plot_data$SNP <- factor(plot_data$SNP, levels = unique(snp_levels)) + + y_text_size <- if (n_snps > 80) ggplot2::rel(0.5) + else if (n_snps > 40) ggplot2::rel(0.65) + else ggplot2::rel(0.85) + + p <- ggplot2::ggplot(plot_data, ggplot2::aes(x = -log10(value), y = SNP)) + + ggplot2::geom_point(ggplot2::aes(colour = key), size = 0.35) + + ggplot2::scale_colour_brewer(type = "qual", name = "GWAS") + + ggplot2::guides(colour = ggplot2::guide_legend(ncol = 1)) + + ggplot2::labs( + x = expression(-log[10] * " (per-GWAS contribution p-value)"), + y = "SNP" + ) + + ggplot2::ggtitle("Contribution to heterogeneity score by GWAS") + + ggplot2::theme( + plot.title = ggplot2::element_text(hjust = 0.5), + axis.text.y = ggplot2::element_text(size = y_text_size), + legend.position = "right" + ) + + ggplot2::ggsave( + output_file, + plot = p, + width = dims$width, + height = dims$height, + units = "px", + dpi = dpi, + limitsize = FALSE + ) } diff --git a/R/gwas_comparisons.R b/R/gwas_comparisons.R index 8796489..b9038d6 100644 --- a/R/gwas_comparisons.R +++ b/R/gwas_comparisons.R @@ -1,11 +1,11 @@ #' compare_replication_across_all_gwas_permutations #' @description takes a vector of gwasse and associated clumps, then runs the expected vs. observed algorithm #' -#' @param gwas_filenames: list of filenames of gwases -#' @param clumped_filenames: list of clumped list filesnames genetred from gwas, in same order as gwases -#' @param result_output_file: data frame of all results of gwas comparisons concatenated -#' @param variants_output_file: data frame of every SNP comparison from clumped list concatenated -#' @import vroom +#' @param gwas_filenames list of filenames of gwases +#' @param clumped_filenames list of clumped list filesnames genetred from gwas, in same order as gwases +#' @param result_output_file data frame of all results of gwas comparisons concatenated +#' @param variants_output_file data frame of every SNP comparison from clumped list concatenated + compare_replication_across_all_gwas_permutations <- function(gwas_filenames = c(), clumped_filenames = c(), result_output_file, @@ -37,8 +37,8 @@ compare_replication_across_all_gwas_permutations <- function(gwas_filenames = c( vroom::vroom_write(merged_variants, variants_output_file) } -#' @import dplyr -#' @import data.table + + compare_two_gwases_from_clumped_hits <- function(first_gwas, second_gwas, clumped_snps) { comparison_name <- paste0(file_prefix(first_gwas), "_vs_", file_prefix(second_gwas)) comparison_columns <- c("SNP", "BETA", "SE", "P", "RSID") @@ -77,7 +77,7 @@ compare_two_gwases_from_clumped_hits <- function(first_gwas, second_gwas, clumpe #' @param b_rep corresponding vector of associations in progression #' @param se_rep standard errors of effects in progression #' @param alpha p-value threshold to check for replication of incidence hits in progression (e.g. try 0.05 or 1e-5) -#' @import tibble + expected_vs_observed_replication <- function(b_disc, se_disc, b_rep, se_rep, alpha = 0.05) { p_sign <- pnorm(-abs(b_disc) / se_disc) * pnorm(-abs(b_disc) / se_rep) + ( @@ -120,10 +120,17 @@ expected_vs_observed_replication <- function(b_disc, se_disc, b_rep, se_rep, alp return(list(res = res, variants = res_per_variant)) } -#' @import data.table -#' @import dplyr -#' @import rlang -#' @import stats + + + + +dataset_label_from_gwas_path <- function(path) { + x <- basename(path) + x <- sub("\\.gz$", "", x, ignore.case = TRUE) + tools::file_path_sans_ext(x) +} + + compare_heterogeneity_across_ancestries <- function(gwas_filenames = c(), clumped_filenames = c(), ancestry_list = c(), @@ -143,57 +150,64 @@ compare_heterogeneity_across_ancestries <- function(gwas_filenames = c(), clumped_snps <- data.table::rbindlist(lapply(clumped_filenames, data.table::fread))$SNP clumped_snps <- unique(clumped_snps) + dataset_labels <- make.unique( + vapply(gwas_filenames, dataset_label_from_gwas_path, character(1)), + sep = "_" + ) + gwases <- lapply(gwas_filenames, function(gwas_filename) { get_file_or_dataframe(gwas_filename, columns = comparison_columns) |> dplyr::filter(RSID %in% clumped_snps) }) - gwas_ancestry_names <- c() for (i in seq_along(gwases)) { - gwases[[i]]$ancestry <- ancestry_list[[i]] - gwas_ancestry_names[[i]] <- paste0(i, "_", ancestry_list[[i]]) + gwases[[i]]$dataset <- dataset_labels[[i]] } harmonised_gwases <- rlang::inject(harmonise_gwases(!!!gwases)) - gwases_by_ancestry <- stats::setNames(harmonised_gwases, gwas_ancestry_names) + gwases_by_dataset <- stats::setNames(harmonised_gwases, dataset_labels) - heterogeneity_results <- calculate_heterogeneity_scores(gwases_by_ancestry, heterogeneity_score_file) + heterogeneity_results <- calculate_heterogeneity_scores(gwases_by_dataset, heterogeneity_score_file) plot_heritability_contribution_per_ancestry(heterogeneity_results$Qj, heterogeneity_plot_file) - plot_snps_with_heterogeneity(gwases_by_ancestry, heterogeneity_results$Q, heterogeneity_plots_per_snp_file) + plot_snps_with_heterogeneity(gwases_by_dataset, heterogeneity_results$Q, heterogeneity_plots_per_snp_file) } -#' @import data.table -plot_snps_with_heterogeneity <- function(gwases_by_ancestry, heterogeneity_resuts_q, heterogeneity_forest_plot_file) { + +plot_snps_with_heterogeneity <- function(gwases_by_dataset, heterogeneity_resuts_q, heterogeneity_forest_plot_file) { are_heterogeneous <- which(heterogeneity_resuts_q$Qpval < 0.05 / nrow(heterogeneity_resuts_q)) - forest_plot_rows <- data.table::rbindlist(lapply(gwases_by_ancestry, function(gwas) { + forest_plot_rows <- data.table::rbindlist(lapply(gwases_by_dataset, function(gwas) { gwas$Qpval <- heterogeneity_resuts_q$Qpval gwas[are_heterogeneous, ] })) + if ("dataset" %in% colnames(forest_plot_rows)) { + data.table::setnames(forest_plot_rows, "dataset", "GWAS") + } + grouped_forest_plot(forest_plot_rows, title = "Plot of SNPs That Are Heterogeneous", - group_column = "ancestry", + group_column = "GWAS", output_file = heterogeneity_forest_plot_file, - #p_value_column = "Qpval" ?? ) } -#' Test for heterogeneity of effect estimates between populations +#' Test for heterogeneity of effect estimates between GWAS datasets #' #' @description For each SNP this function will provide a Cochran's Q test statistic - -#' a measure of heterogeneity of effect sizes between populations. A low p-value means high heterogeneity. -#' In addition, for every SNP it gives a per population p-value - -#' this can be interpreted as asking for each SNP is a particular giving an outlier estimate. +#' a measure of heterogeneity of effect sizes between datasets. A low p-value means high heterogeneity. +#' In addition, for every SNP it gives a per-dataset p-value — +#' interpreted as whether a particular study is an outlier at that SNP. #' -#' @param gwases_by_ancestry Named list of data frames, one for each population, with at least bhat, se and snp columns +#' @param gwases_by_ancestry Named list of data frames, one per GWAS dataset, with at least BETA, SE and SNP columns +#' @param heterogeneity_score_file file to save the heterogeneity scores #' #' @return List #' - Q = vector of p-values for Cochrane's Q statistic for each SNP #' - Qj = Data frame of per-population outlier q values for each SNP -#' @import tibble -#' @import dplyr -#' @import vroom -#' @import stats + + + + calculate_heterogeneity_scores <- function(gwases_by_ancestry, heterogeneity_score_file) { beta <- lapply(gwases_by_ancestry, \(x) x$BETA) |> dplyr::bind_cols() se <- lapply(gwases_by_ancestry, \(x) x$SE) |> dplyr::bind_cols() diff --git a/R/gwas_formatting.R b/R/gwas_formatting.R index f450215..0286e52 100644 --- a/R/gwas_formatting.R +++ b/R/gwas_formatting.R @@ -1,18 +1,26 @@ #' standardise_gwas: takes an input gwas, changes headers, standardises allelic input, adds RSID, makes life easier -#' @param gwas: filename of gwas to standardise or dataframe of gwas -#' @param output_file: file to save standardised gwas -#' @param N: sample size of GWAS (if GWAS has defined N column, defaults to that value) -#' @param populate_rsid_option: if you want RSID populated or not -#' @param input_reference_build: reference build of CHR and BP of data. Defaults to GRCh37 -#' @param output_reference_build: reference build of CHR and BP of data. Defaults to GRCh37 -#' @param input_columns: column header map used for renaming GWAS: +#' @param gwas filename of gwas to standardise or dataframe of gwas +#' @param output_file file to save standardised gwas +#' @param N sample size of GWAS (if GWAS has defined N column, defaults to that value) +#' @param populate_rsid_option if you want RSID populated or not +#' @param input_reference_build reference build of CHR and BP of data. Defaults to GRCh37 +#' @param output_reference_build reference build of CHR and BP of data. Defaults to GRCh37 +#' @param input_columns column header map used for renaming GWAS: #' can be list() of key/value pairs, string of row in predefined_column_maps, or comma separated list of keys=values -#' @param output_columns: column header map used for renaming GWAS: +#' @param output_columns column header map used for renaming GWAS: #' can be list() of key/value pairs, string of row in predefined_column_maps, or comma separated list of keys=values -#' @param r: reference build of CHR and BP of data. Defaults to GRCh37 -#' @return modified gwas: saves new gwas in {output_file} if present -#' @import vroom -#' @import shiny +#' @param remove_extra_columns logical flag on if to remove extra columns +#' @param populate_eaf If TRUE, fill missing \code{EAF} from the 1000 Genomes \code{.frq} +#' file via RSID matching. Requires \code{ancestry}. A partial RSID population is +#' performed automatically when RSIDs are not already present. +#' @param ancestry Ancestry code (EUR, EAS, AFR, AMR, SAS) matching the reference panel prefix; required when \code{populate_eaf} is TRUE. +#' @param flip_alleles If TRUE (default), keep EA/OA in alphabetical order with flipped +#' BETA/EAF/Z in the output. When FALSE the original allele order and effect direction +#' are restored before saving; all internal steps still operate on the canonical +#' (alphabetically flipped) representation. +#' @return modified gwas: saves new gwas in output_file if present + + #' @export standardise_gwas <- function(gwas, output_file, @@ -22,22 +30,46 @@ standardise_gwas <- function(gwas, output_reference_build=reference_builds$GRCh37, input_columns="default", output_columns="default", - remove_extra_columns=F) { + remove_extra_columns=F, + populate_eaf=FALSE, + ancestry=NULL, + flip_alleles=TRUE) { input_gwas_columns <- resolve_column_map(input_columns) output_gwas_columns <- resolve_column_map(output_columns) - #TODO: if we need to add bespoke input format wrangling here, we can + if (populate_eaf && (is.null(ancestry) || length(ancestry) == 0 || !nzchar(as.character(ancestry)[1]))) { + stop("populate_eaf is TRUE but ancestry is missing; set ancestry to EUR, EAS, AFR, AMR, or SAS to match your LD reference panel.") + } gwas <- get_file_or_dataframe(gwas) |> change_column_names(input_gwas_columns, remove_extra_columns) |> standardise_columns(N) |> filter_incomplete_rows() |> - convert_reference_build_via_liftover(input_reference_build, output_reference_build) |> - standardise_alleles() |> + convert_reference_build_via_liftover(input_reference_build, output_reference_build) + + gwas <- standardise_alleles(gwas) + + rsid_option <- populate_rsid_option + if (isTRUE(populate_eaf) && identical(rsid_option, populate_rsid_options$none) && !"RSID" %in% colnames(gwas)) { + rsid_option <- populate_rsid_options$partial + } + + gwas <- gwas |> health_check() |> - populate_rsid(populate_rsid_option) |> - populate_gene_names() |> - change_column_names(output_gwas_columns) + populate_rsid(rsid_option) |> + populate_gene_names() + + if (isTRUE(populate_eaf)) { + gwas <- populate_eaf_from_reference_panel(gwas, ancestry) + } + + if (!flip_alleles) { + gwas <- unflip_alleles(gwas) + } else { + gwas$.ALLELES_FLIPPED <- NULL + } + + gwas <- change_column_names(gwas, output_gwas_columns) if (!missing(output_file) && shiny::isTruthy(output_file)) { vroom::vroom_write(gwas, output_file) @@ -45,12 +77,66 @@ standardise_gwas <- function(gwas, return(gwas) } +#' change_snp_identifiers: changes the SNP identifiers of a GWAS to the output reference build +#' @param gwas filename of gwas to standardise or dataframe of gwas +#' @param output_file file to save standardised gwas +#' @param populate_rsid_option if you want RSID populated or not +#' @param input_reference_build reference build of CHR and BP of data. Defaults to GRCh37 +#' @param output_reference_build reference build of CHR and BP of data. Defaults to GRCh37 +#' @param input_columns column header map used for renaming GWAS: +#' can be list() of key/value pairs, string of row in predefined_column_maps, or comma separated list of keys=values +#' @return modified gwas: saves new gwas in output_file if present + +#' @export +change_snp_identifiers <- function(gwas, + output_file, + populate_rsid_option=populate_rsid_options$none, + input_reference_build=reference_builds$GRCh37, + output_reference_build=reference_builds$GRCh37, + input_columns="default") { + input_gwas_columns <- resolve_column_map(input_columns) + + # Load the original GWAS (can be a filename or a dataframe) + original_gwas <- get_file_or_dataframe(gwas) + + # Track original row order so we can map new annotations back + original_gwas$row_number <- seq_len(nrow(original_gwas)) + + # Create an updated version with new coordinates / RSIDs, keeping row_number + updated_gwas <- original_gwas |> + change_column_names(input_gwas_columns) |> + convert_reference_build_via_liftover(input_reference_build, output_reference_build) |> + standardise_alleles() |> + populate_rsid(populate_rsid_option) |> + populate_gene_names() + + # Map new RSID and BP (if present) back onto the original data using row_number + annotation_cols <- c("RSID", "BP") + annotation_cols <- annotation_cols[annotation_cols %in% colnames(updated_gwas)] + + if (length(annotation_cols) > 0) { + updated_subset <- updated_gwas[, c("row_number", annotation_cols), drop = FALSE] + for (col in annotation_cols) { + original_gwas[[col]] <- updated_subset[[col]][match(original_gwas$row_number, updated_subset$row_number)] + } + } + + # Remove helper row_number column + original_gwas$row_number <- NULL + + if (!missing(output_file) && shiny::isTruthy(output_file)) { + vroom::vroom_write(original_gwas, output_file) + } + + return(original_gwas) +} + #' harmonise_gwases: takes a list of gwases, get the SNPs in common #' across all datasets arranged to be in the same order #' -#' @param: elipses of gwases -#' @return: list of harmonised gwases -#' @import dplyr +#' @param ... elipses of gwases +#' @return list of harmonised gwases + #' @export harmonise_gwases <- function(...) { gwases <- list(...) @@ -82,7 +168,16 @@ filter_incomplete_rows <- function(gwas) { return(filtered_gwas) } -#' @import tidyr +#' Normalise CHR for GWAS rows (chr-prefix, whitespace, leading zeros) so downstream joins match PLINK panels. +#' @keywords internal +normalise_gwas_chr <- function(x) { + x <- as.character(x) + x <- stringr::str_trim(x) + x <- sub("^chr", "", x, ignore.case = TRUE) + x <- sub("^0+", "", x) + x +} + standardise_columns <- function(gwas, N) { gwas_columns <- colnames(gwas) @@ -122,6 +217,10 @@ standardise_columns <- function(gwas, N) { gwas$BETA <- as.numeric(gwas$BETA) } + if ("CHR" %in% colnames(gwas)) { + gwas$CHR <- normalise_gwas_chr(gwas$CHR) + } + return(gwas) } @@ -139,9 +238,10 @@ health_check <- function(gwas) { #' change_column_names: private function that takes a named list of column names #' and changes the supplied data frame's column names accordingly #' -#' @param: gwas dataframe of gwas to standardise column names -#' @param: columns named list for -#' @param: opposite_mapping logical flag on if we are mapping from key to value or vice verca +#' @param gwas dataframe of gwas to standardise column names +#' @param columns named list for +#' @param remove_extra_columns logical flag on if we are removing extra columns +#' @return gwas with new column names change_column_names <- function(gwas, columns = list(), remove_extra_columns = F) { for (name in names(columns)) { #this deletes an existing column that we're about to rename, so we don't have 2 columns @@ -161,18 +261,21 @@ change_column_names <- function(gwas, columns = list(), remove_extra_columns = F return(gwas) } -#' @import dplyr +#' Reorder EA/OA alphabetically and flip BETA, EAF, Z to match. +#' Rows that were flipped are marked in a logical \code{.ALLELES_FLIPPED} column +#' so they can be restored later by \code{unflip_alleles()}. +#' @keywords internal standardise_alleles <- function(gwas) { gwas$EA <- toupper(gwas$EA) gwas$OA <- toupper(gwas$OA) to_flip <- (gwas$EA > gwas$OA) & (!gwas$EA %in% c("D", "I")) + gwas$.ALLELES_FLIPPED <- to_flip + if (any(to_flip)) { - gwas$EAF[to_flip] <- 1 - gwas$EAF[to_flip] + if ("EAF" %in% names(gwas)) gwas$EAF[to_flip] <- 1 - gwas$EAF[to_flip] gwas$BETA[to_flip] <- -1 * gwas$BETA[to_flip] - if ('Z' %in% names(gwas)) { - gwas$Z[to_flip] <- -1 * gwas$Z[to_flip] - } + if ("Z" %in% names(gwas)) gwas$Z[to_flip] <- -1 * gwas$Z[to_flip] temp <- gwas$OA[to_flip] gwas$OA[to_flip] <- gwas$EA[to_flip] @@ -185,13 +288,35 @@ standardise_alleles <- function(gwas) { return(gwas) } +#' Restore original allele order for rows that were flipped by \code{standardise_alleles()}. +#' Reverses BETA, EAF, Z and swaps EA/OA back, then rebuilds the SNP column. +#' @keywords internal +unflip_alleles <- function(gwas) { + if (!".ALLELES_FLIPPED" %in% names(gwas)) return(gwas) + + to_unflip <- gwas$.ALLELES_FLIPPED + if (any(to_unflip)) { + if ("EAF" %in% names(gwas)) gwas$EAF[to_unflip] <- 1 - gwas$EAF[to_unflip] + gwas$BETA[to_unflip] <- -1 * gwas$BETA[to_unflip] + if ("Z" %in% names(gwas)) gwas$Z[to_unflip] <- -1 * gwas$Z[to_unflip] + + temp <- gwas$OA[to_unflip] + gwas$OA[to_unflip] <- gwas$EA[to_unflip] + gwas$EA[to_unflip] <- temp + } + + gwas$.ALLELES_FLIPPED <- NULL + gwas$SNP <- toupper(paste0(gwas$CHR, ":", format(gwas$BP, scientific = F, trim = T), "_", gwas$EA, "_", gwas$OA)) + return(gwas) +} + #' convert_or_to_beta: Given an OR and lower and upper bounds, #' calculates the BETA, and SE. #' based on this answer: https://stats.stackexchange.com/a/327684 #' -#' @param gwas: dataframe with the following columns: OR, LB (lower bound), UB (upper bound) +#' @param gwas dataframe with the following columns: OR, LB (lower bound), UB (upper bound) #' @return gwas with new columns BETA and SE -#' @import stats + #' @export convert_or_to_beta <- function(gwas) { gwas <- get_file_or_dataframe(gwas) @@ -206,7 +331,10 @@ convert_or_to_beta <- function(gwas) { return(gwas) } -#' @import stats +#' convert_beta_to_or: Given a BETA and SE, calculates the OR and lower and upper bounds +#' @param gwas dataframe with the following columns: BETA, SE +#' @return gwas with new columns OR, OR_LB, OR_UB + #' @export convert_beta_to_or <- function(gwas) { gwas <- get_file_or_dataframe(gwas) @@ -241,7 +369,7 @@ convert_z_score_to_beta <- function(gwas, sample_size) { -#' @import stats + convert_z_to_p <- function(gwas) { gwas$P <- 2 * pnorm(-abs(gwas$Z)) return(gwas) @@ -263,14 +391,17 @@ convert_p_to_negative_log_p <- function(gwas) { return(gwas) } -#' @import stats +#' calculate_f_statistic: calculates the F-statistic from the P-value +#' @param gwas dataframe with P column +#' @return gwas with F_STAT column + #' @export calculate_f_statistic <- function(gwas) { gwas$F_STAT <- stats::qchisq(gwas$P, 1, low=F) return(gwas) } -#' @import stats + calculate_lambda_statistic <- function(gwas) { lambda <- median(stats::qchisq(1 - gwas$P, 1)) / stats::qchisq(0.5, 1) return(lambda) diff --git a/R/imports.R b/R/imports.R new file mode 100644 index 0000000..f479f2e --- /dev/null +++ b/R/imports.R @@ -0,0 +1,41 @@ +# NAMESPACE: consolidated imports with `except` to avoid R CMD check WARNINGs from +# symbol clashes between e.g. dplyr/stats, data.table/rlang, TwoSampleMR/MR, R.utils, … +# (see 00install.out: "Warning: replacing previous import … when loading 'GeneHackman'"). +# All @import and @rawNamespace must be in the same roxygen block before the following NULL +# (otherwise roxygen2 drops them from NAMESPACE). +# +#' @keywords internal +#' @rawNamespace import(MendelianRandomization, except = c(mr_ivw, mr_median)) +#' @rawNamespace import(SlopeHunter, except = c(format_data)) +#' @rawNamespace import(data.table, except = c(between, first, last)) +#' @rawNamespace import(dplyr, except = c(between, first, last, filter, lag)) +#' @rawNamespace import(stats, except = c(filter, lag)) +#' @rawNamespace import(rlang, except = c(`:=`)) +#' @rawNamespace import(R.utils, except = c(env, extract, reset, setProgress, timestamp, validate)) +#' @rawNamespace import(future, except = c(run)) +#' @rawNamespace import(shiny, except = c(setProgress, validate)) +#' @rawNamespace import(tidyr, except = c(extract)) +#' @rawNamespace import(utils, except = c(timestamp)) +#' @import coloc +#' @import ggplot2 +#' @import patchwork +#' @import ggrepel +#' @import grDevices +#' @import graphics +#' @import httr +#' @import prettydoc +#' @import qqman +#' @import rmarkdown +#' @import stringr +#' @import tibble +#' @import vroom +#' @import susieR +#' @import TwoSampleMR +#' @importFrom data.table setkey +#' @importFrom data.table setkeyv +#' @importFrom furrr future_map +#' @importFrom fst fst +#' @importFrom fst read_fst +#' @importFrom progressr progressor +#' @importFrom tibble as_tibble +NULL diff --git a/R/liftover.R b/R/liftover.R index 5354819..4d8a0b0 100644 --- a/R/liftover.R +++ b/R/liftover.R @@ -7,10 +7,10 @@ available_liftover_conversions[[paste0(reference_builds$GRCh37, reference_builds #' convert_reference_build_via_liftover: Change reference build of BP marker from allow list of liftOver conversions #' -#' @param gwas: GWAS (file or dataframe) of standardised GWAS -#' @param input_reference_build: string reference build, found in reference_builds list -#' @param output_reference_build: string reference build that GWAS is to change to, found in reference_builds list -#' @param output_file: optional output file name to save to +#' @param gwas GWAS (file or dataframe) of standardised GWAS +#' @param input_reference_build string reference build, found in reference_builds list +#' @param output_reference_build string reference build that GWAS is to change to, found in reference_builds list +#' @param output_file optional output file name to save to #' @return gwas input is altered and returned #' @export convert_reference_build_via_liftover <- function(gwas, @@ -26,7 +26,7 @@ convert_reference_build_via_liftover <- function(gwas, liftover_conversion <- available_liftover_conversions[[paste0(input_reference_build, output_reference_build)]] if (is.null(liftover_conversion)) { - stop(paste(c("Error: liftOver combination of", input_build, output_build, "not recocognised.", + stop(paste(c("Error: liftOver combination of", input_reference_build, output_reference_build, "not recocognised.", "Reference builds must be one of:", reference_builds), collapse = " ")) } @@ -66,8 +66,8 @@ convert_reference_build_via_liftover <- function(gwas, return(gwas) } -#' @import tibble -#' @import vroom + + create_bed_file_from_gwas <- function(gwas, output_file) { gwas <- get_file_or_dataframe(gwas) @@ -83,13 +83,22 @@ create_bed_file_from_gwas <- function(gwas, output_file) { } +#' Thin wrapper so tests can mock via testthat::local_mocked_bindings(). +#' @keywords internal +run_system <- function(command, wait = TRUE, intern = FALSE, + ignore.stdout = FALSE, ignore.stderr = FALSE) { + base::system(command, wait = wait, intern = intern, + ignore.stdout = ignore.stdout, ignore.stderr = ignore.stderr) +} + run_liftover <- function(bed_file_input, bed_file_output, input_build, output_build, unmapped) { - lifover_binary <- paste0(liftover_dir, "liftOver") liftover_conversion <- available_liftover_conversions[[paste0(input_build, output_build)]] - - chain_file <- paste0(liftover_dir, liftover_conversion) - liftover_command <- paste(lifover_binary, bed_file_input, chain_file, bed_file_output, unmapped) - system(liftover_command, wait=T) + chain_file <- file.path(liftover_chain_dir, liftover_conversion) + liftover_command <- paste(liftover_binary, bed_file_input, chain_file, bed_file_output, unmapped) + status <- run_system(liftover_command, wait = TRUE) + if (!is.null(status) && status != 0) { + stop("liftOver failed (exit code ", status, "): ", liftover_command) + } } diff --git a/R/locus_zoom.R b/R/locus_zoom.R new file mode 100644 index 0000000..d509ccf --- /dev/null +++ b/R/locus_zoom.R @@ -0,0 +1,259 @@ +#' Generate locus zoom plots for significant colocalization results +#' +#' For each coloc result with PP.H4.abf above the threshold, produces a stacked +#' locus zoom plot showing both traits at the shared locus, with LD computed +#' from the local 1000 Genomes reference panel. +#' +#' @param coloc_results_file path to coloc_results.tsv +#' @param gwas_files named character vector of standardised GWAS file paths, +#' names must match trait labels used in coloc_results (trait1/trait2 columns) +#' @param ancestry ancestry code for the LD reference panel (EUR, EAS, etc.) +#' @param ens_db Ensembl database object or package name for gene annotations +#' @param output_dir directory to write per-locus PNG files +#' @param pp_h4_threshold minimum PP.H4.abf to consider significant (default 0.8) +#' @param window_kb half-width of the region to plot in kb (default 500) +#' @param completion_file path to a sentinel file written on successful completion. +#' If NULL, no sentinel is written. +#' @return Invisibly returns \code{NULL}. On failure, stops with an error summarising +#' loci that could not be plotted. +#' @export +locus_zoom_coloc <- function(coloc_results_file, + gwas_files, + ancestry, + ens_db, + output_dir, + pp_h4_threshold = 0.8, + window_kb = 500, + completion_file = NULL) { + coloc_res <- vroom::vroom(coloc_results_file, show_col_types = FALSE) + + significant <- coloc_res[coloc_res$PP.H4.abf >= pp_h4_threshold, ] + message("Significant results to plot: (PP.H4.abf >= ", pp_h4_threshold, "): ", nrow(significant)) + if (!dir.exists(output_dir)) dir.create(output_dir, recursive = TRUE) + + if (nrow(significant) == 0) { + write_completion_file(completion_file, "0 loci plotted") + return(invisible(NULL)) + } + + traits_in_results <- unique(c(significant$trait1, significant$trait2)) + missing_traits <- setdiff(traits_in_results, names(gwas_files)) + + if (length(missing_traits) > 0) { + stop( + "Traits in coloc results but missing from gwas_files: ", + paste(missing_traits, collapse = ", "), + ". Available: ", + paste(names(gwas_files), collapse = ", ") + ) + } + + gwas_cache <- list() + load_gwas <- function(trait) { + if (is.null(gwas_cache[[trait]])) { + gwas_cache[[trait]] <<- get_file_or_dataframe(gwas_files[[trait]]) + } + gwas_cache[[trait]] + } + + failures <- character() + + for (i in seq_len(nrow(significant))) { + row <- significant[i, ] + locus_label <- paste0(row$trait1, " vs ", row$trait2, " at ", row$locus1, "/", row$locus2) + + failure_msg <- tryCatch({ + chr <- as.integer(strsplit(row$locus1, "_")[[1]][1]) + bp1 <- as.numeric(strsplit(row$locus1, "_")[[1]][2]) + bp2 <- as.numeric(strsplit(row$locus2, "_")[[1]][2]) + center_bp <- round(mean(c(bp1, bp2))) + + window_bp <- window_kb * 1000 + xrange <- c(center_bp - window_bp, center_bp + window_bp) + + index_snp <- find_index_snp_from_ld_panel(chr, center_bp, window_bp, ancestry) + plot_list <- list() + for (trait in c(row$trait1, row$trait2)) { + gwas <- load_gwas(trait) + p <- build_single_locus_plot(gwas, chr, xrange, ens_db, ancestry, + index_snp, trait) + if (!is.null(p)) { + plot_list[[trait]] <- p + } else { + message(" WARNING: build_single_locus_plot returned NULL for '", trait, "'") + } + } + + if (length(plot_list) < 2) { + return(paste0( + locus_label, + ": only ", + length(plot_list), + " of 2 trait plots could be built" + )) + } + + hit_info <- "" + if (!is.na(row$hit1) && !is.na(row$hit2)) { + hit_info <- paste0("_", row$hit1, "_vs_", row$hit2) + } + + safe_name <- gsub("[^A-Za-z0-9._-]+", "_", paste0( + row$trait1, "_vs_", row$trait2, "_chr", chr, "_", center_bp, hit_info + )) + out_file <- file.path(output_dir, paste0(safe_name, ".png")) + + stacked <- patchwork::wrap_plots(plot_list, ncol = 1, guides = "collect") + + patchwork::plot_annotation( + title = locus_label, + theme = ggplot2::theme( + plot.title = ggplot2::element_text(face = "bold", size = 12, hjust = 0.5) + ) + ) + + ggplot2::ggsave(out_file, plot = stacked, + width = 10, height = 5 * length(plot_list), + dpi = 300, limitsize = FALSE) + NULL + }, error = function(e) { + paste0(locus_label, ": ", conditionMessage(e)) + }) + + if (!is.null(failure_msg)) { + failures <- c(failures, failure_msg) + } + } + + if (length(failures) > 0) { + message("Locus zoom failed for ", length(failures), " of ", nrow(significant), " locus/loci:") + for (msg in failures) { + message(" - ", msg) + } + stop( + "Locus zoom failed for ", + length(failures), + " of ", + nrow(significant), + " locus/loci. See log messages above." + ) + } + + write_completion_file(completion_file, paste(nrow(significant), "loci plotted")) + return(invisible(NULL)) +} + + +#' Write a completion sentinel file for Snakemake +#' @keywords internal +write_completion_file <- function(completion_file, content = "done") { + if (is.null(completion_file)) return(invisible(NULL)) + dir_path <- dirname(completion_file) + if (!dir.exists(dir_path)) dir.create(dir_path, recursive = TRUE) + writeLines(content, completion_file) +} + + +#' Find the top SNP near a locus center from the LD reference panel +#' @keywords internal +find_index_snp_from_ld_panel <- function(chr, center_bp, window_bp, ancestry) { + bfile <- file.path(thousand_genomes_dir, ancestry) + bim_file <- paste0(bfile, ".bim") + if (!file.exists(bim_file)) return(NULL) + + bim <- data.table::fread(bim_file, header = FALSE, + select = c(1, 2, 4), + col.names = c("CHR", "SNP", "BP")) + bim <- bim[CHR == chr & BP >= (center_bp - window_bp) & BP <= (center_bp + window_bp)] + if (nrow(bim) == 0) return(NULL) + + bim[which.min(abs(bim$BP - center_bp)), ]$SNP +} + + +#' Build a single ggplot2 locus zoom panel for one trait +#' @keywords internal +build_single_locus_plot <- function(gwas, chr, xrange, ens_db, ancestry, + index_snp, trait_label) { + gwas_chr <- as.numeric(gwas$CHR) + gwas_bp <- as.numeric(gwas$BP) + keep <- !is.na(gwas_chr) & gwas_chr == chr & + !is.na(gwas_bp) & gwas_bp >= xrange[1] & gwas_bp <= xrange[2] + gwas_sub <- gwas[keep, ] + + if (nrow(gwas_sub) < 2) return(NULL) + + plot_df <- data.frame( + chrom = as.numeric(gwas_sub$CHR), + pos = as.numeric(gwas_sub$BP), + p = as.numeric(gwas_sub$P), + stringsAsFactors = FALSE + ) + if ("RSID" %in% colnames(gwas_sub)) { + plot_df$rsid <- gwas_sub$RSID + } else if ("SNP" %in% colnames(gwas_sub)) { + plot_df$rsid <- gwas_sub$SNP + } + if ("BETA" %in% colnames(gwas_sub)) plot_df$beta <- as.numeric(gwas_sub$BETA) + + plot_df <- plot_df[!is.na(plot_df$p) & plot_df$p > 0, ] + if (nrow(plot_df) < 2) return(NULL) + + ld_r2 <- compute_ld_r2_for_locus(plot_df$rsid, chr, ancestry, index_snp) + if (!is.null(ld_r2)) { + plot_df$r2 <- ld_r2[match(plot_df$rsid, names(ld_r2))] + } else { + message("LD r2 computation returned NULL; plotting without LD colouring") + } + + ld_arg <- if ("r2" %in% colnames(plot_df)) "r2" else NULL + + loc <- locuszoomr::locus( + data = plot_df, + seqname = as.character(chr), + xrange = xrange, + ens_db = ens_db, + chrom = "chrom", + pos = "pos", + p = "p", + labs = "rsid", + index_snp = index_snp, + LD = ld_arg + ) + + p <- locuszoomr::locus_ggplot(loc, labels = "index", + filter_gene_biotype = "protein_coding") + # locus_ggplot returns an assembled patchwork; wrap it and add a visible title + p <- patchwork::wrap_elements(p) + + patchwork::plot_annotation( + title = trait_label, + theme = ggplot2::theme( + plot.title = ggplot2::element_text( + size = 14, face = "bold", hjust = 0, margin = ggplot2::margin(b = 10) + ) + ) + ) + p +} + + +#' Compute r2 relative to an index SNP using the local LD reference panel +#' @keywords internal +compute_ld_r2_for_locus <- function(rsids, chr, ancestry, index_snp) { + if (is.null(index_snp) || !index_snp %in% rsids) return(NULL) + + rsids_unique <- unique(rsids[!is.na(rsids) & nchar(rsids) > 0]) + if (length(rsids_unique) < 2) return(NULL) + + ld_result <- compute_ld_matrix(rsids_unique, chr, ancestry) + if (is.null(ld_result)) return(NULL) + + R <- ld_result$matrix + snps <- ld_result$snps + + if (!index_snp %in% snps) return(NULL) + + idx <- which(snps == index_snp) + r2_vec <- R[idx, ]^2 + names(r2_vec) <- snps + r2_vec +} diff --git a/R/markdown/coloc.Rmd b/R/markdown/coloc.Rmd new file mode 100644 index 0000000..818a54d --- /dev/null +++ b/R/markdown/coloc.Rmd @@ -0,0 +1,112 @@ +--- +title: "GWAS pairwise colocalization" +date: "`r format(Sys.time(), '%d %B, %Y')`" +output: + prettydoc::html_pretty: + theme: cayman +--- +```{r include = FALSE} +library(vroom) + +traits_v <- if (!is.null(params$traits_ordered) && nzchar(as.character(params$traits_ordered))) { + unlist(strsplit(as.character(params$traits_ordered), "|", fixed = TRUE)) +} else { + character() +} +ancestries_v <- if (!is.null(params$ancestries_ordered) && nzchar(as.character(params$ancestries_ordered))) { + unlist(strsplit(as.character(params$ancestries_ordered), "|", fixed = TRUE)) +} else { + character() +} + +if (length(traits_v) != length(ancestries_v)) { + traits_v <- character() + ancestries_v <- character() +} + +anc_trim <- ancestries_v[nchar(trimws(ancestries_v)) > 0L] +uniq_anc <- sort(unique(trimws(anc_trim))) +cross_ancestry <- length(uniq_anc) > 1L + +coloc_exist <- nzchar(as.character(params$coloc_results)) && file.exists(as.character(params$coloc_results)) +coloc_tbl <- if (coloc_exist) { + suppressWarnings(vroom::vroom(as.character(params$coloc_results), show_col_types = FALSE)) +} else { + NULL +} +``` + +### Input GWAS + +Lists the GWAS datasets that were standardised, clumped, and finemapped as input to the colocalization analysis, along with the ancestry used for LD reference. + +```{r echo = FALSE} +if (length(traits_v) > 0L && identical(length(traits_v), length(ancestries_v))) { + kv <- data.frame(trait_prefix = traits_v, ancestry = ancestries_v, stringsAsFactors = FALSE) + knitr::kable(kv, caption = "Input GWAS (standardised → clumped → finemapped)") +} +``` + +### Colocalization priors & overlap + +Shows the prior probabilities and overlap window used to define signal pairs for BF-BF colocalization testing. + +Overlapping finemapped signals were defined within ± **`r params$overlap_kb`** kb (pairs of credible-set leads on the same chromosome with lead positions within this window). + +| Setting | Value | +|---------|-------| +| `overlap_kb` | `r params$overlap_kb` | +| `p1` | `r params$p1` | +| `p2` | `r params$p2` | +| `p12` | `r params$p12` | + +```{r echo = FALSE, results = 'asis'} +if (cross_ancestry) { + ld_phrase <- if (length(uniq_anc) == 2L) { + paste0("between ", uniq_anc[1], " and ", uniq_anc[2]) + } else { + paste0("among the populations (", paste(uniq_anc, collapse = ", "), ")") + } + cat(paste0( + "> **Cross-ancestry colocalization detected.**\n>\n>", + "> Differences in LD structure ", ld_phrase, + " can **result in false-negative colocalization signals** (**H3**; incompatible ", + "causal variants in the region — see PP.H3.abf vs PP.H4.abf in the `coloc` output).\n>\n>", + "> Fine-mapping / LD reference ancestries used in this run: **", + paste(ancestries_v, collapse = "**, **"), + "**.\n" + )) +} +``` + +```{r echo = FALSE, results = 'asis'} +if (!cross_ancestry && length(uniq_anc) == 1L) { + cat(paste0( + '*All GWAS in this pipeline were analysed with ancestry **', + uniq_anc[1], + '** for LD (clumping / fine-mapping).*\n\n' + )) +} +``` + + +### Colocalization results + +Pairwise BF-BF colocalization results for overlapping finemapped signals — PP.H4.abf > 0.8 indicates strong evidence that two traits share the same causal variant at that locus. + +```{r echo = FALSE} +if (!is.null(coloc_tbl) && nrow(coloc_tbl) > 0L) { + knitr::kable(dplyr::arrange(coloc_tbl, dplyr::desc(PP.H4.abf)) |> dplyr::filter(PP.H4.abf > 0.8), caption = "All significant (PP.H4.abf > 0.8) pairwise colocalisation results") +} else { + cat("(No significant pairwise colocalisation rows returned — insufficient overlapping finemap signals across traits.)") +} +``` + +### Additional info + +**Please do us a favour and cite this pipeline: https://doi.org/10.5281/zenodo.10624713** + +* This pipeline was run using the github repository: https://github.com/MRCIEU/GeneHackman +* and the docker image: docker://mrcieu/genehackman:`r get_docker_image_tag()` +* `coloc` R package: https://doi.org/10.1371/journal.pgen.1004383 +* `coloc` guide: https://chr1swallace.github.io/coloc/ diff --git a/R/markdown/compare_gwases.Rmd b/R/markdown/compare_gwases.Rmd index cedc4f2..52da9c0 100644 --- a/R/markdown/compare_gwases.Rmd +++ b/R/markdown/compare_gwases.Rmd @@ -8,6 +8,7 @@ output: ### LDSC Result Files +LD Score Regression (LDSC) estimates SNP heritability (h²) and genetic correlation (rg) between each pair of GWAS; each file below corresponds to one ancestry group. ```{r echo = F} ldsc_result_files <- Sys.glob(paste0(params$ldsc_directory, "*")) @@ -20,18 +21,22 @@ for (file in ldsc_result_files) { ### Plots Comparing Heterogeneity Across Significant SNPs -`r sprintf("![Plot comparing the heritability across clumped SNPs](%s)", params$heterogeneity_plot)` -`r sprintf("![Forest Plot Comparing clumped SNPs that fail heterogeniety score](%s)", params$heterogeneity_snp_comparison)` +The first plot shows the per-GWAS contribution to the Cochran's Q heterogeneity statistic for **all** clumped SNPs (one row per SNP). The second plot is a grouped forest plot of effect sizes for the **subset** of SNPs whose heterogeneity Q p-value passes Bonferroni correction (Q p-value < 0.05 / number of SNPs tested), i.e. only the significantly heterogeneous SNPs. + +`r sprintf("![Per-GWAS contribution to heterogeneity Q-statistic (all clumped SNPs)](%s)", params$heterogeneity_plot)` +`r sprintf("![Forest plot of effect sizes for SNPs with significant heterogeneity (Bonferroni-corrected)](%s)", params$heterogeneity_snp_comparison)` ### Expected vs Observed Results -This shows all expected vs. observed results run across all GWAS +Compares the expected replication rate (based on discovery effect sizes and replication standard errors) against the observed replication rate across all GWAS pairs — a discrepancy suggests winner's curse or population differences. + ```{r echo=F} expected_vs_observed <- vroom::vroom(params$results, show_col_types = F) knitr::kable(expected_vs_observed) ``` ### Table of Heterogeneity Results +Per-SNP Cochran's Q p-values and per-GWAS contribution p-values (Qj) for all clumped SNPs tested for heterogeneity of effect sizes across the input GWAS. ```{r echo=F} heterogeneity_results <- vroom::vroom(params$heterogeneity_scores, show_col_types = F) diff --git a/R/markdown/disease_progression.Rmd b/R/markdown/disease_progression.Rmd index f39afd4..095b892 100644 --- a/R/markdown/disease_progression.Rmd +++ b/R/markdown/disease_progression.Rmd @@ -10,10 +10,16 @@ library(vroom) library(dplyr, quietly=T) ``` -### Unadjusted Manhattan Plots of GWAS -`r sprintf("![Miami Plot Comparing Unadjusted GWAS at p = 0.0001](%s)", params$unadjuested_miami_plot)` +### Incident GWAS vs Collider Bias Adjusted GWAS + +Miami plot comparing the incident (discovery) GWAS against the collider-bias-adjusted subsequent GWAS — regions that diverge suggest variants whose apparent progression effect was driven by index-event bias. + +`r sprintf("![Miami Plot Comparing Incident GWAS and Collider Bias Adjusted GWAS](%s)", params$unadjuested_miami_plot)` ### Comparison of Collider Bias Results + +Summary of the collider bias correction model (e.g. SlopeHunter or CWLS): estimated correction slope, standard error, and significance — quantifies how strongly incidence confounds the subsequent GWAS effect estimates. + ```{r echo=F} collider_bias_results <- vroom::vroom(params$collider_bias_results, show_col_types = F) knitr::kable(collider_bias_results) @@ -21,9 +27,9 @@ knitr::kable(collider_bias_results) ### Expected vs Observed Results -This shows all expected vs. observed results when comparing the incident vs. subsequent GWASes +Compares the expected replication rate against observed replication between incident and subsequent GWASes, before and after collider-bias adjustment — improvement after correction indicates the adjustment is recovering true progression signals. -First table: Expected vs. Observed before adjusted for Collider Bias +First table: Expected vs. Observed before adjusting for Collider Bias Second table: Expected vs. Observed after adjusting for Collider Bias ```{r echo=F} @@ -34,12 +40,16 @@ adjusted_expected_vs_observed <- vroom::vroom(params$adjusted_expected_vs_observ knitr::kable(adjusted_expected_vs_observed) ``` -### Adjusted Manhattan Plots of GWAS -`r sprintf("![Miami Plot of Adjusted GWAS at p = 0.0001](%s)", params$adjusted_miami_plot)` +### Subsequent GWAS vs Collider Bias Adjusted GWAS + +Miami plot comparing the unadjusted subsequent GWAS against the collider-bias-adjusted subsequent GWAS — highlights which genomic regions are most affected by the correction. + +`r sprintf("![Miami Plot Comparing Subsequent GWAS and Collider Bias Adjusted GWAS](%s)", params$adjusted_miami_plot)` ### Comparing GWAS Results Across Top Hits in Subsequent GWAS -This is a comparison of all top hits returned from clumped subsequent unadjusted +Table of effect estimates (BETA, SE, P) for each clumped lead SNP from the subsequent GWAS, shown side-by-side across the incidence, subsequent, and adjusted GWASes — allows direct comparison of effect direction and magnitude before and after correction. + ```{r echo=F} clumped_snps <- data.table::fread(params$clumped_subsequent)$SNP columns <- c("SNP", "RSID", "BETA", "SE", "P") diff --git a/R/markdown/mr_for_qtls.Rmd b/R/markdown/mr_for_qtls.Rmd index 22b5b6b..8f6ebf9 100644 --- a/R/markdown/mr_for_qtls.Rmd +++ b/R/markdown/mr_for_qtls.Rmd @@ -13,9 +13,15 @@ options(scipen=1) ``` ### Volcano Plot of MR Results Against `r params$dataset` QTL Dataset + +Displays all MR exposure-outcome tests: x-axis is causal effect (BETA), y-axis is −log10 FDR-adjusted p-value — points above the significance line (red dashed) and labelled are exposures with evidence of a causal relationship with the outcome GWAS. + `r sprintf("![](%s)", params$volcano_plot)` ### Statistically Significant MR results (after FDR correction) + +Table of MR results that pass FDR < 0.05 and have a matching coloc result — these are the exposures with the strongest evidence of a causal effect on the outcome trait (up to top 20 shown). + Only showing top 20 results if there are more. ```{r echo=F} mr_results <- vroom::vroom(params$mr_results, show_col_types = F) |> @@ -26,8 +32,13 @@ knitr::kable(head(mr_results, 20)) ``` ### Coloc Results + +Colocalization results for each significant MR exposure — PP.H4.abf > 0.8 indicates that the MR signal and the QTL share the same causal variant, strengthening the causal inference beyond MR alone. + ```{r echo=F} -coloc_results <- coloc_results |> dplyr::filter(exposure %in% mr_results$EXPOSURE) +coloc_results <- coloc_results |> + dplyr::filter(exposure %in% mr_results$EXPOSURE) |> + dplyr::arrange(desc(PP.H4.abf)) knitr::kable(coloc_results) ``` diff --git a/R/mr.R b/R/mr.R index e1c2d58..689f7ae 100644 --- a/R/mr.R +++ b/R/mr.R @@ -1,21 +1,26 @@ +#' @keywords internal +mr_list_files <- function(dir, pattern, full.names = FALSE) { + list.files(dir, pattern = pattern, full.names = full.names) +} + #' perform_mr_on_metabrain_datasets <- function(gwas_filename, ancestry="EUR", subcategory=NULL, exposures=c(), results_output) { file_pattern <- paste0(tolower(subcategory), "_", tolower(ancestry)) - metabrain_top_hits <- list.files(metabrain_top_hits_dir, pattern = file_pattern, full.names = T) + metabrain_top_hits <- mr_list_files(metabrain_top_hits_dir, pattern = file_pattern, full.names = T) run_mr_on_qtl_data(gwas_filename, qtl_files = metabrain_top_hits, results_output = results_output, exposures = exposures) } perform_mr_on_eqtlgen_datasets <- function(gwas_filename, subcategory=NULL, exposures=c(), results_output) { file_pattern <- tolower(subcategory) - eqtlgen_top_hits <- list.files(eqtlgen_top_hits_dir, pattern = file_pattern, full.names = T) + eqtlgen_top_hits <- mr_list_files(eqtlgen_top_hits_dir, pattern = file_pattern, full.names = T) run_mr_on_qtl_data(gwas_filename, results_output = results_output, qtl_files = eqtlgen_top_hits, exposures = exposures) } -#' @import dplyr -#' @import vroom -#' @import TwoSampleMR + + + run_mr_on_qtl_data <- function(gwas_filename, qtl_files, results_output, exposures=c()) { all_qtl_mr_results <- lapply(qtl_files, function(qtl_file) { qtl_exposure <- TwoSampleMR::read_exposure_data(qtl_file, @@ -73,8 +78,8 @@ run_mr_on_qtl_data <- function(gwas_filename, qtl_files, results_output, exposur #' #' -#' @import dplyr -#' @import vroom + + compare_interesting_mr_results <- function(pqtl_mr_results, forest_plot_output_file) { interesting_pqtl_mr_results <- vroom::vroom(pqtl_mr_results, show_col_types=F) |> dplyr::filter(p.adjusted < 0.05) |> diff --git a/R/util.R b/R/util.R index 1782124..6fb7161 100644 --- a/R/util.R +++ b/R/util.R @@ -1,12 +1,12 @@ #' get_file_or_dataframe: function that takes either dataframe OR file name, and returns a subset of #' columns, rows, or both. #' accepted input file types: txt, csv, tsv, zip, gz -#' @param input: either data frame or string input of file -#' @param columns: vector of strings matching column names to filter columns -#' @param snps: vector of SNP names matching SNPs to filter rows -#' @return output: data frame of GWAS -#' @import dplyr -#' @import vroom +#' @param input either data frame or string input of file +#' @param columns vector of strings matching column names to filter columns +#' @param snps vector of SNP names matching SNPs to filter rows +#' @return output data frame of GWAS + + #' @export get_file_or_dataframe <- function(input, columns=NULL, snps=NULL) { if (is.data.frame(input)) { @@ -31,8 +31,8 @@ get_file_or_dataframe <- function(input, columns=NULL, snps=NULL) { } return(output) } -#' @import data.table -#' @import dplyr + + filter_gwas_by_clumped_results <- function(gwas, clumped_results) { #vroom has trouble reading plink --clump output, so using fread rsids <- data.table::fread(clumped_results, select = "SNP")$SNP @@ -43,7 +43,10 @@ filter_gwas_by_clumped_results <- function(gwas, clumped_results) { #' vroom_snps: If you only need to get a handful of SNPs out of a whole GWAS, this saves time and memory #' NOTE: only works with data that has been standardised, through `standardise_gwas`, or at least a tsv -#' @import vroom +#' @param gwas_file file of gwas to get SNPs from +#' @param snps vector of SNP names to get +#' @return dataframe of SNPs + #' @export vroom_snps <- function(gwas_file, snps=c()){ snps <- paste(snps, collapse="\t|") @@ -62,7 +65,13 @@ vroom_snps <- function(gwas_file, snps=c()){ return(snps_in_gwas) } -#' @import dplyr +#' gwas_region: filter a GWAS file to a region +#' @param gwas dataframe with the following columns: CHR, BP +#' @param chr chromosome +#' @param bp base pair position +#' @param range range to filter in base pairs +#' @return gwas with filtered rows + #' @export gwas_region <- function(gwas, chr, bp, range = 500000) { return(dplyr::filter(gwas, CHR == chr &BP > (bp - floor(range/2)) & BP < (bp + floor(range/2)))) @@ -96,7 +105,7 @@ file_prefix <- function(file_path) { return(file_prefix) } -#' @import stringr + create_dir_for_files <- function(...) { filenames <- list(...) @@ -111,9 +120,9 @@ map_rsid_list_to_snps <- function(gwas, rsids=c()) { return(gwas$SNP) } -#' @import rmarkdown -#' @import httr -#' @import prettydoc + + + create_html_from_rmd <- function(rmd_file, params = list(), output_file) { temp_file <- tempfile(fileext = ".Rmd", tmpdir = "/tmp") file.copy(rmd_file, temp_file, overwrite = TRUE) @@ -126,17 +135,68 @@ create_html_from_rmd <- function(rmd_file, params = list(), output_file) { } get_docker_image_tag <- function() { - return("1.0.0") + docker_version <- get_env_var("DOCKER_VERSION") + if (is.null(docker_version)) { + return("latest") + } else { + return(docker_version) + } #return(packageVersion("GeneHackman")) } -#' @import vroom -run_sqlite_command <- function(db_name, query, col_names = c()) { - query <- paste0('"', query, '"') - sqlite_command <- paste("sqlite3", db_name, query) - output <- system(sqlite_command, intern = T, wait = T) - result <- vroom::vroom(output, col_names = F, col_type = vroom::cols(vroom::col_character()), show_col_types=F) - - if (is.vector(col_names)) colnames(result) <- col_names - return(result) +#' Generate log Bayes Factor from Z-score +#' +#' @param z Z-score +#' @param se Standard error +#' @param eaf Allele frequency +#' @param sample_size Sample size +#' @param study_type Study type +#' @param effect_priors Effect priors +#' +#' @return Log Bayes Factor +#' @keywords internal +convert_z_to_lbf <- function( + z, + se, + eaf, + sample_size, + study_type, + effect_priors = c(continuous = 0.15, categorical = 0.2) +) { + estimated_sd <- estimate_variance(se, eaf, sample_size) + if (study_type == study_types$continuous) { + sd_prior <- effect_priors[study_types$continuous] * estimated_sd + } else { + sd_prior <- effect_priors[study_types$categorical] + } + r <- sd_prior^2 / (sd_prior^2 + se^2) + lbf <- (log(1 - r) + (r * z^2)) / 2 + return(lbf) } + +#' Estimate trait standard deviation given vectors of variance of coefficients, MAF and sample size +#' +#' Estimate is based on var(beta-hat) = var(Y) / (n * var(X)) +#' var(X) = 2*maf*(1-maf) +#' so we can estimate var(Y) by regressing n*var(X) against 1/var(beta) +#' +#' @title Estimate trait variance, internal function +#' @param se vector of standard errors +#' @param eaf vector of MAF (same length as SE) +#' @param n sample size +#' +#' @return estimated standard deviation of Y +#' @keywords internal +estimate_variance <- function(se, eaf, n) { + oneover <- 1 / se^2 + nvx <- 2 * n * eaf * (1 - eaf) + m <- lm(nvx ~ oneover - 1) + cf <- coef(m)[["oneover"]] + if (cf < 0) { + stop( + "Estimated sdY is negative - this can happen with small datasets, or those with errors. ", + "A reasonable estimate of sdY is required to continue." + ) + } + return(sqrt(cf)) +} \ No newline at end of file diff --git a/README.md b/README.md index 5646238..3fb2379 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # GeneHackman +![CI Tests](https://github.com/MRCIEU/GeneHackman/actions/workflows/main.yml/badge.svg) + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10624713.svg)](https://doi.org/10.5281/zenodo.10624713) A pipeline for performing common genetic epidemiology tasks at the University of Bristol. @@ -11,47 +13,121 @@ Goals: ## Available Pipelines -Here is a list of available pipelines, and the steps they run +There are **six** Snakemake pipelines (grouped as two tables of three). Each pipeline is a `.smk` file under `snakemake/`; see [PIPELINES.md](snakemake/PIPELINES.md) for YAML inputs and parameters. + +### Pipelines — table 1 + +| standardise_gwas.smk | compare_gwases.smk | disease_progression.smk | +|-------------------------------------------------------|---------------------------------------------------------------------------|----------------------------------------------------------------------| +| Takes in any of: VCF, CSV, TSV, TXT (also zip/gz) | Runs **standardise_gwas** for each GWAS, then pairwise comparison tooling | Runs **standardise_gwas** for incident and subsequent GWASes | +| Optional liftover (e.g. GRCh38 → GRCh37) | PLINK clumping | Runs collider-bias-aware analyses (e.g. SlopeHunter, MR-IVW) and compares results | +| Optional RSID and EAF fill from reference panels | Heterogeneity across GWASes; LDSC *h*² and *r*g | Miami plots of unadjusted vs adjusted GWAS | +| Converts z-scores and odds ratios to BETA/SE | Expected vs observed replication metrics | Expected vs observed (before and after collider adjustment) | +| Harmonised SNP ID = `CHR:BP_EA_OA` (EA/OA sorted) | HTML report of LDSC, plots, and tables | HTML report; optional instructions to refit subsequent GWAS from collider output | +| Optional gene ↔ Ensembl mapping | | | + +### Pipelines — table 2 -| standardise_gwas.smk | compare_gwases.smk | disease_progression.smk | qtl_mr.smk | -|---------------------------------------------------------|----------------------------------------|-------------------------------------------------------|-------------------------------------------------| -| Takes in any of: vcf, csv, tsv, txt (and zip, gz) | All steps in 'standardise_gwas.smk' | All steps in 'standardise_gwas.smk' | All steps in 'standardise_gwas.smk' | -| Convert Reference Build
(GRCh38 -> GRCh37) | PLINK clumping | Runs some collider bias corrections, compares results | Run MR against top hits of specific QTL dataset | -| Populate RSID from CHR, BP, EA, and OA | Calculate heterogeneity between GWASes | Miami Plot of Collider Bias Results | Volcano Plot of Results | -| Converts z-scores and OR to BETA | LDSC h2 and rg | Expected vs. Observed Comparison | Run coloc of significant top hit MR results | -| Auto-populate GENE ID <-> ENSEMBL ID | Expected vs. Observed Comparison | | | -| Unique SNP = CHR:BP_EA_OA
(EA < OA alphabetically) | | | | +| qtl_mr.smk | finemap.smk | coloc.smk | +|----------------------------------------------------------------------------|----------------------------------------------------------------------------|---------------------------------------------------------------------------| +| Runs **standardise_gwas**, clumping, and **SuSiE** fine-mapping on one outcome GWAS | Same **standardise** + **clump** + **SuSiE** path for **each** input GWAS (one or more) | Same **standardise** + **clump** + **SuSiE** for **≥2** GWASes (required for colocalization) | +| Runs Mendelian randomization vs a chosen QTL panel (e.g. eQTLGen, MetaBrain) | Per-locus fine-mapping using summary stats and **ancestry-matched LD** (PLINK reference); outputs credible sets and LBF columns per locus | **Pairwise** `coloc::coloc.bf_bf` on overlapping finemapped signals (same chr, leads within ±`overlap_kb` kb) across all trait pairs | +| Volcano plot of MR results; **BF-BF coloc** for exposures that pass MR FDR | Finemap-only: no MR or coloc between traits (use when you only need SuSiE outputs) | Full coloc table + **HTML report** (`result_coloc.html`), including a disclaimer when ancestries differ between GWASes | +| Requires **QTL_DATA_DIR** (see `.env_example`) for QTL files | Each GWAS must declare **ancestry** (for LD) | Configurable `finemap` and `coloc` priors/overlap; see [PIPELINES.md](snakemake/PIPELINES.md) | ## Onboarding ### 1. Clone the repository into your personal space on BlueCrystal 4 `git clone git@github.com:MRCIEU/GeneHackman.git && cd GeneHackman` -### 2. Ensure you [have conda installed and initialised before activating](https://www.acrc.bris.ac.uk/protected/hpc-docs/software/python_conda.html) +### 2. Ensure you [have conda installed and initialised before activating](https://www.anaconda.com/docs/getting-started/miniconda/install/overview) + +```conda env create --file environment.yml``` + +or if you have already created the environment + +```conda activate genehackman``` + +### 3. Get reference data (Google Cloud Storage) + +The data to run the pipelines have been split into two buckets, the mandatory bucket, and QTL bucket (only needed if you want to run MR-QTL pipeline). To download, [install gsutil](https://docs.cloud.google.com/storage/docs/gsutil_install) + +* Mandatory: `gs://genehackman` + * `gsutil -m rsync -r gs://genehackman/ /path/to/my_pipeline_data/` + * Update `PIPELINE_DATA_DIR` to `/path/to/my_pipeline_data/` in the .env file +* Optional: `gs://genehackman-qtl` + * `gsutil -m rsync -r gs://genehackman-qtl/ /path/to/my_qtl_data/` + * Update `QTL_DATA_DIR` to `/path/to/my_qtl_data/` in the .env file + +To copy only selected prefixes instead of the QTL bucket, use `gsutil -m cp -r gs://genehackman-qtl/SOME_PREFIX/ ...` as needed. For example, you may only be interested in cis, not trans data. + +Then point your **`.env`** at those directories (trailing slashes are fine): + +```bash +PIPELINE_DATA_DIR=/path/to/my_pipeline_data/ +QTL_DATA_DIR=/path/to/my_qtl_data/ +``` -`conda activate /mnt/storage/private/mrcieu/data/genomic_data/pipeline/genehackman` +Alternatives if you don’t set **`QTL_DATA_DIR`** directly: keep the same directory layout under **`PIPELINE_DATA_DIR/qtl_datasets/`**, or follow [PLATFORM_SETUP.md](PLATFORM_SETUP.md) (bind mounts and defaulting **`QTL_DATA_DIR`** to **`PIPELINE_DATA_DIR/qtl_datasets`**). -(you can alternatively create your own conda environment if you like: `conda env create --file environment.yml`) +### 4. Populate `.env` and `input.yaml` files -### 3. Populate .env and input.json files +Copy the template and edit paths for your machine: -### `cp .env_example .env` -* populate the DATA_DIR, RESULTS_DIR and RDFS_DIR environment variables in .env file -These should probably be in your *work* or *scratch* space (`/user/work/userid/...`) -* RDFS_DIR is optional. All generated files can be copied automatically. Please ensure the path -ends in `working/` +```bash +cp .env_example .env +``` -### Fill out input.json file -* Ex: `cp snakemake/input_templates/compare_gwases.json input.json` -* Each pipeline (as defined in `snakemake` directory) has its own input format. - * [Here are example pipelines here, copy to input.json](snakemake/input_templates/) - * [Documentation per pipeline](snakemake/PIPELINES.md) -* You can either copy into input.json, or supply the file into the script from another location +Variables match [.env_example](.env_example): -### 4. Run the pipeline +**Mandatory** -`./run_pipeline.sh snakemake/.smk ` +| Variable | Purpose | +|----------|---------| +| **`PROJECT_DIR`** | Root folder for this analysis. The pipeline uses **`PROJECT_DIR/data/`** for inputs (GWAS, clumps, …) and **`PROJECT_DIR/results/`** for outputs (finemap, coloc, plots, …). | +| **`PIPELINE_DATA_DIR`** | Path where you unpacked the shared **`genehackman`** reference data (see §3). Also used for the Apptainer image: **`PIPELINE_DATA_DIR/genomic_data/pipeline/genehackman_.sif`**. If that SIF is missing and the directory is writable, `run_pipeline.sh` builds it from `docker://mrcieu/genehackman:`. | +| **`DOCKER_VERSION`** | Docker/Apptainer image tag (e.g. `1.1.0` or `develop`). Must match the SIF filename and the image you intend to run. | +**Optional** + +| Variable | Purpose | +|----------|---------| +| **`QTL_DATA_DIR`** | Path to **`genehackman-qtl`** data if you run **`qtl_mr`**. Leave empty for other pipelines. You can instead place QTL data under **`PIPELINE_DATA_DIR/qtl_datasets/`** (see [PLATFORM_SETUP.md](PLATFORM_SETUP.md)). | +| **`SNAKEMAKE_PROFILE`** | Snakemake profile directory. Default in `.env_example`: **`snakemake/profiles/local/`** (local Apptainer). On HPC use e.g. **`snakemake/profiles/slurm/`**. | +| **`APPTAINER_MODULE`** | Environment module name to load Apptainer/Singularity before running on HPC (only used when the profile is not `local`). | +| **`SLURM_PARTITION`** | Slurm partition for cluster jobs. If unset, `run_pipeline.sh` tries to detect one from `sinfo`, else uses **`compute`**. | +| **`SLURM_ACCOUNT`** | Slurm account for cluster jobs. If unset, `run_pipeline.sh` tries to infer it from `sacctmgr`. | + +Example `.env`: + +```bash +PROJECT_DIR=/path/to/my_project +PIPELINE_DATA_DIR=/path/to/my_pipeline_data/ +DOCKER_VERSION=1.1.0 +QTL_DATA_DIR=/path/to/my_qtl_data/ +SNAKEMAKE_PROFILE=snakemake/profiles/local/ +``` + +**`input.yaml`** + +* Example: `cp snakemake/input_templates/compare_gwases.yaml input.yaml` +* Each pipeline has its own shape; examples live under [`snakemake/input_templates/`](snakemake/input_templates/). +* See [PIPELINES.md](snakemake/PIPELINES.md) for all fields. +* Pass the input YAML as the **second** argument to `run_pipeline.sh`, or rely on **`input.yaml`** in the working directory. To call **`snakemake`** yourself without the wrapper, set **`PROJECT_DIR`** in **`.env`** and export **`DATA_DIR="${PROJECT_DIR%/}/data"`** and **`RESULTS_DIR="${PROJECT_DIR%/}/results"`** (or load the same paths Snakemake uses) so shell rules and profile bind mounts resolve; also pass **`--config genehackman_input=/path/to/file.yaml`**. + +### 5. Run the pipeline + +To ensure you have configured everything correctly, you can run a test pipeline from `run_test_pipelines.sh` + +`./run_pipeline.sh snakemake/standardise_gwas.smk tests/testthat/data/snakemake_inputs/standardise_gwas.yaml` + +To run your pipeline: + +`./run_pipeline.sh snakemake/.smk ` + +Snakemake execution profiles (**`--profile`**) live under **`snakemake/profiles/`** (see also [PLATFORM_SETUP.md](PLATFORM_SETUP.md)). + +* By default `run_pipeline.sh` uses **`SNAKEMAKE_PROFILE=snakemake/profiles/local/`** (local Apptainer). On HPC, set e.g. **`SNAKEMAKE_PROFILE=snakemake/profiles/slurm/`** (generic Slurm) or add a site-specific directory beside **`snakemake/profiles/local/`** and **`snakemake/profiles/slurm/`**. * `run_pipeline.sh` is just a convience wrapper around the `snakemake` command, if you want to do anything out of the ordinary, [please read up on snakemake](https://snakemake.readthedocs.io/en/v7.26.0/) * If there are errors while running the pipeline, you can find error messages either directly on the screen, or in slurm log file that is outputted on error * It is recommended that you run the your pipeline [inside a tmux session](https://github.com/MRCIEU/GeneHackman/wiki/Common-Errors#ssh-disconnection-while-pipeline-is-running). @@ -65,19 +141,16 @@ The standard column naming for GWASes are: A full list of names and default values [can be found here](inst/extdata/predefined_column_maps.csv) -There are 3 main components to the pipeline +There are 2 main components to the pipeline 1. Snakemake to define the steps to complete for each pipeline. 2. Docker / Singularity container with installed languages (R and python), packages, os libraries, and code -3. Slurm: each snakemake step spins up a singularity container inside a slurm job. Each step can specify different slurm requirements. -### Repository Organisation +The pipeline can be run either on its own, or via your institutions HPC. Each snakemake step spins up a singularity container inside an HPC job (ex. slurm). Each step can specify different cpu/memory requirements. + +### Platform Setup -* `R` directory holds R package code that can also be called and reused by any step in the pipeline (accessed by a cli script) -* `scripts` directory holds the scripts that can be easily called by snakemake (`Rscript example.R --input_ex example_input`) -* `snakemake` directory, which defines the pipeline steps and configuration, and shared code between pipelines -* `docker` directory holds the information for creating the docker image that the pipeline runs -* `tests` directory holds all R tests, and a end to end pipeline test script +**Running on macOS, Linux, Slurm, or PBS?** See **[PLATFORM_SETUP.md](PLATFORM_SETUP.md)** for platform-specific setup (Apptainer/Lima, SIF cache, Snakemake profiles under **`snakemake/profiles/`**, `qsub` template). ### Making changes -If you want to make any additions / changes please contact andrew.elmore@bristol.ac.uk, or open an issue in this repo. +See **[CONTRIBUTING.md](CONTRIBUTING.md)** for development setup, Docker rebuilds, unit tests, and end-to-end tests. You can also contact open an issue in this repo. diff --git a/docker/Dockerfile b/docker/Dockerfile index f0ae8a0..01146fc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,21 +1,40 @@ -FROM python:3.11-slim +# This recipe uses amd64-only assets (Posit R .deb, PLINK, Miniconda, libpng12 .deb). On ARM hosts, +# build with: docker build --platform linux/amd64 ... +FROM python:3.12-bookworm RUN apt update && \ - apt install -y --no-install-recommends r-base=4.2.2.20221110-2 r-base-dev=4.2.2.20221110-2 \ - build-essential software-properties-common dirmngr gnupg pandoc lmodern texlive-latex-base \ - libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev libfreetype6-dev libharfbuzz-dev libfribidi-dev \ - libxml2-dev libcurl4-gnutls-dev libssl-dev libgmp-dev libnlopt-dev cmake libcairo2-dev libxt-dev \ + apt install -y --no-install-recommends \ + build-essential software-properties-common dirmngr gnupg pandoc lmodern texlive-latex-base unzip \ + libpng-dev libtiff5-dev libjpeg-dev libfreetype6-dev libharfbuzz-dev libfribidi-dev \ + libxml2-dev libcurl4-openssl-dev libssl-dev libgmp-dev libnlopt-dev cmake libcairo2-dev libxt-dev \ texlive-latex-recommended r-cran-sass r-cran-mime libudunits2-dev libgdal-dev \ - plink1.9 plink2 sqlite3 wget vim ripgrep git && \ + libfontconfig1-dev libuv1-dev libicu-dev libbz2-dev liblzma-dev libglib2.0-dev \ + sqlite3 wget vim ripgrep git && \ + wget https://cdn.posit.co/r/debian-12/pkgs/r-4.4.2_1_amd64.deb && \ + apt-get update && \ + apt-get install -y ./r-4.4.2_1_amd64.deb && \ + rm r-4.4.2_1_amd64.deb && \ + ln -s /opt/R/4.4.2/bin/R /usr/local/bin/R && \ + ln -s /opt/R/4.4.2/bin/Rscript /usr/local/bin/Rscript && \ + mkdir -p /usr/lib/R/site-library && chmod 777 /usr/lib/R/site-library && \ mkdir -p /home/scripts && \ rm -rf /var/lib/apt/lists/* && \ + wget https://s3.amazonaws.com/plink1-assets/plink_linux_x86_64_20231211.zip -O /tmp/plink.zip && \ + unzip /tmp/plink.zip -d /usr/local/bin/ && \ + rm /tmp/plink.zip && \ + ln -s /usr/local/bin/plink /usr/local/bin/plink1.9 && \ + wget https://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/liftOver -O /usr/local/bin/liftOver && \ + chmod +x /usr/local/bin/liftOver && \ wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ chmod +x miniconda.sh && \ ./miniconda.sh -b -f -p /bin && \ wget http://snapshot.debian.org/archive/debian/20160413T160058Z/pool/main/libp/libpng/libpng12-0_1.2.54-6_amd64.deb && \ dpkg -i libpng12-0_1.2.54-6_amd64.deb -RUN cd /home && \ +# Miniconda 25+ requires explicit acceptance of Anaconda repo channel ToS before installs. +RUN /bin/condabin/conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main && \ + /bin/condabin/conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r && \ + cd /home && \ git clone https://github.com/MRCIEU/PHESANT.git && \ git clone --recurse-submodules https://github.com/samtools/htslib.git && \ git clone https://github.com/samtools/bcftools.git && \ @@ -28,13 +47,9 @@ RUN cd /home && \ cd /home/ldsc && \ /bin/condabin/conda env create --file environment.yml -#TODO: other things to think about installing: bcftools, finemap, some sort of meta-analysis tool (METAL, GWAMA, etc) -#RUN wget http://christianbenner.com/finemap_v1.4.2_x86_64.tgz && \ -# tar -xf finemap_v1.4.2_x86_64.tgz && \ -# mv finemap_v1.4.2_x86_64 /usr/local/bin/finemap - -COPY docker/requirements.* docker/ +COPY DESCRIPTION docker/requirements.* docker/ RUN Rscript docker/requirements.R -RUN pip install -r docker/requirements.txt +ENV R_LIBS_SITE=/usr/lib/R/site-library +RUN pip install --no-cache-dir -r docker/requirements.txt CMD ["/bin/bash"] diff --git a/docker/requirements.R b/docker/requirements.R index 6f3961f..53b9665 100644 --- a/docker/requirements.R +++ b/docker/requirements.R @@ -1,8 +1,22 @@ -install.packages(c("devtools", "argparser", "gmp", "corrplot", "broom", "conflicted", "nloptr", "Cairo"), - repos = "http://cran.us.r-project.org") +options(repos = c(CRAN = "https://cloud.r-project.org")) +.libPaths(c("/usr/lib/R/site-library", .libPaths())) -#also needed: forestplot, data.table, devtools -#install.packages('https://homepages.uni-regensburg.de/~wit59712/easyqc/EasyQC_23.8.tar.gz', repos = NULL, type = 'source') -.libPaths( c( .libPaths(), "/usr/lib/R/site-library") ) +install.packages("remotes") +pkgs <- c("argparser", "gmp", "corrplot", "broom", "conflicted", "nloptr", "Cairo", "Rfast", "susieR") +install.packages(pkgs) -devtools::install_github(c("MRCIEU/GeneHackman", "phenoscanner/phenoscanner", "suchestoncampbelllab/gwasurvivr")) +install.packages("devtools") + + +if (!requireNamespace("BiocManager", quietly = TRUE)) + install.packages("BiocManager") +BiocManager::install(c("ensembldb", "EnsDb.Hsapiens.v75"), update = FALSE, ask = FALSE) + +install.packages(c("locuszoomr", "patchwork")) + +remotes::install_deps( + "docker", + dependencies = c("Depends", "Imports", "LinkingTo"), + upgrade = "never", + repos = BiocManager::repositories() +) diff --git a/docker/requirements.txt b/docker/requirements.txt index a5468e4..d86d577 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,3 +1,10 @@ click==8.1.3 snakemake==7.25.4 synapseclient==2.7.2 +PyYAML>=6.0.1 +numpy>=1.26.4 +scipy>=1.4.1 +tqdm>=4.67.1 +numba>=0.51.0 +pandas>=2.0.0 +MultiSuSiE @ git+https://github.com/jordanero/MultiSuSiE.git \ No newline at end of file diff --git a/environment.yml b/environment.yml index 861c5b9..f118f4b 100644 --- a/environment.yml +++ b/environment.yml @@ -6,6 +6,7 @@ dependencies: - python=3.11 - snakemake=7.26 - python-dotenv=1.0.0 + - pyyaml=6.0.1 - pip=24.0 - pip: - pygsheets==2.0.6 diff --git a/man/adjust_gwas_data_from_weights_and_save.Rd b/man/adjust_gwas_data_from_weights_and_save.Rd index d074d64..d7be85a 100644 --- a/man/adjust_gwas_data_from_weights_and_save.Rd +++ b/man/adjust_gwas_data_from_weights_and_save.Rd @@ -16,17 +16,17 @@ adjust_gwas_data_from_weights_and_save( ) } \arguments{ -\item{gwas:}{gwas that the new collider bias correction will be saved to. Swaps out BETA, SE, and P} +\item{gwas}{gwas that the new collider bias correction will be saved to. Swaps out BETA, SE, and P} -\item{harmonised_effects:}{a dataframe that includes BETA.incidence, BETA.prognosis, SE.incidence, SE.prognosis} +\item{harmonised_effects}{a dataframe that includes BETA.incidence, BETA.prognosis, SE.incidence, SE.prognosis} -\item{collider_bias_type:}{string name of adjustment (eg. slopehunter)} +\item{collider_bias_type}{string name of adjustment (eg. slopehunter)} -\item{beta:}{number, slope of the correction} +\item{beta}{number, slope of the correction} -\item{se:}{number, SE of the corrected slope} +\item{se}{number, SE of the corrected slope} -\item{output_file:}{name of file that the corrected GWAS will be saved to} +\item{output_file}{name of file that the corrected GWAS will be saved to} } \description{ adjust_gwas_data_from_weights: Apply a slope correction weight (and SE) to an existing GWAS and save result to disk. diff --git a/man/build_bf_matrix.Rd b/man/build_bf_matrix.Rd new file mode 100644 index 0000000..6b48a20 --- /dev/null +++ b/man/build_bf_matrix.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{build_bf_matrix} +\alias{build_bf_matrix} +\title{Build a named BF matrix from a finemapped locus data frame} +\usage{ +build_bf_matrix(locus_df, lbf_cols) +} +\description{ +Build a named BF matrix from a finemapped locus data frame +} +\keyword{internal} diff --git a/man/build_lbf_matrix.Rd b/man/build_lbf_matrix.Rd new file mode 100644 index 0000000..123979a --- /dev/null +++ b/man/build_lbf_matrix.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc_bf.R +\name{build_lbf_matrix} +\alias{build_lbf_matrix} +\title{Build a named BF matrix from locus data} +\usage{ +build_lbf_matrix(locus_df, lbf_cols) +} +\description{ +Build a named BF matrix from locus data +} +\keyword{internal} diff --git a/man/build_single_locus_plot.Rd b/man/build_single_locus_plot.Rd new file mode 100644 index 0000000..71dbc45 --- /dev/null +++ b/man/build_single_locus_plot.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/locus_zoom.R +\name{build_single_locus_plot} +\alias{build_single_locus_plot} +\title{Build a single ggplot2 locus zoom panel for one trait} +\usage{ +build_single_locus_plot( + gwas, + chr, + xrange, + ens_db, + ancestry, + index_snp, + trait_label +) +} +\description{ +Build a single ggplot2 locus zoom panel for one trait +} +\keyword{internal} diff --git a/man/calculate_f_statistic.Rd b/man/calculate_f_statistic.Rd new file mode 100644 index 0000000..80d1123 --- /dev/null +++ b/man/calculate_f_statistic.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gwas_formatting.R +\name{calculate_f_statistic} +\alias{calculate_f_statistic} +\title{calculate_f_statistic: calculates the F-statistic from the P-value} +\usage{ +calculate_f_statistic(gwas) +} +\arguments{ +\item{gwas}{dataframe with P column} +} +\value{ +gwas with F_STAT column +} +\description{ +calculate_f_statistic: calculates the F-statistic from the P-value +} diff --git a/man/calculate_heterogeneity_scores.Rd b/man/calculate_heterogeneity_scores.Rd index 4e91139..c332df4 100644 --- a/man/calculate_heterogeneity_scores.Rd +++ b/man/calculate_heterogeneity_scores.Rd @@ -2,12 +2,14 @@ % Please edit documentation in R/gwas_comparisons.R \name{calculate_heterogeneity_scores} \alias{calculate_heterogeneity_scores} -\title{Test for heterogeneity of effect estimates between populations} +\title{Test for heterogeneity of effect estimates between GWAS datasets} \usage{ calculate_heterogeneity_scores(gwases_by_ancestry, heterogeneity_score_file) } \arguments{ -\item{gwases_by_ancestry}{Named list of data frames, one for each population, with at least bhat, se and snp columns} +\item{gwases_by_ancestry}{Named list of data frames, one per GWAS dataset, with at least BETA, SE and SNP columns} + +\item{heterogeneity_score_file}{file to save the heterogeneity scores} } \value{ List @@ -18,7 +20,7 @@ List } \description{ For each SNP this function will provide a Cochran's Q test statistic - -a measure of heterogeneity of effect sizes between populations. A low p-value means high heterogeneity. -In addition, for every SNP it gives a per population p-value - -this can be interpreted as asking for each SNP is a particular giving an outlier estimate. +a measure of heterogeneity of effect sizes between datasets. A low p-value means high heterogeneity. +In addition, for every SNP it gives a per-dataset p-value — +interpreted as whether a particular study is an outlier at that SNP. } diff --git a/man/change_column_names.Rd b/man/change_column_names.Rd index f236057..b002329 100644 --- a/man/change_column_names.Rd +++ b/man/change_column_names.Rd @@ -8,7 +8,14 @@ and changes the supplied data frame's column names accordingly} change_column_names(gwas, columns = list(), remove_extra_columns = F) } \arguments{ -\item{:}{opposite_mapping logical flag on if we are mapping from key to value or vice verca} +\item{gwas}{dataframe of gwas to standardise column names} + +\item{columns}{named list for} + +\item{remove_extra_columns}{logical flag on if we are removing extra columns} +} +\value{ +gwas with new column names } \description{ change_column_names: private function that takes a named list of column names diff --git a/man/change_snp_identifiers.Rd b/man/change_snp_identifiers.Rd new file mode 100644 index 0000000..8909f2d --- /dev/null +++ b/man/change_snp_identifiers.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gwas_formatting.R +\name{change_snp_identifiers} +\alias{change_snp_identifiers} +\title{change_snp_identifiers: changes the SNP identifiers of a GWAS to the output reference build} +\usage{ +change_snp_identifiers( + gwas, + output_file, + populate_rsid_option = populate_rsid_options$none, + input_reference_build = reference_builds$GRCh37, + output_reference_build = reference_builds$GRCh37, + input_columns = "default" +) +} +\arguments{ +\item{gwas}{filename of gwas to standardise or dataframe of gwas} + +\item{output_file}{file to save standardised gwas} + +\item{populate_rsid_option}{if you want RSID populated or not} + +\item{input_reference_build}{reference build of CHR and BP of data. Defaults to GRCh37} + +\item{output_reference_build}{reference build of CHR and BP of data. Defaults to GRCh37} + +\item{input_columns}{column header map used for renaming GWAS: +can be list() of key/value pairs, string of row in predefined_column_maps, or comma separated list of keys=values} +} +\value{ +modified gwas: saves new gwas in output_file if present +} +\description{ +change_snp_identifiers: changes the SNP identifiers of a GWAS to the output reference build +} diff --git a/man/chrpos_to_rsid.Rd b/man/chrpos_to_rsid.Rd new file mode 100644 index 0000000..1109bf5 --- /dev/null +++ b/man/chrpos_to_rsid.Rd @@ -0,0 +1,50 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/chrpos_to_rsid.R +\name{chrpos_to_rsid} +\alias{chrpos_to_rsid} +\title{Chromosome & position data to variant RSID} +\usage{ +chrpos_to_rsid( + dt, + chr_col, + pos_col, + ea_col = NULL, + nea_col = NULL, + flip = "allow", + alt_rsids = FALSE, + build = "b37_dbsnp156", + dbsnp_dir = NA_character_, + parallel_cores = parallel::detectCores(), + verbose = TRUE +) +} +\arguments{ +\item{dt}{a data.frame like object, or file path, with at least columns (chrom\if{html}{\out{}}, pos\if{html}{\out{}}, ea\if{html}{\out{}}, nea\if{html}{\out{}})} + +\item{chr_col}{a string column name; chromosome position} + +\item{pos_col}{a string column name; base position} + +\item{ea_col}{a string column name; effect allele} + +\item{nea_col}{a string column name; non effect allele} + +\item{flip}{a string, options: "report", "allow", "no_flip"} + +\item{alt_rsids}{a logical, whether to return additional alternate RSIDs} + +\item{build}{a string, options: "b37_dbsnp156", "b38_dbsnp156" (corresponds to the appropriate data directory)} + +\item{dbsnp_dir}{a string file path to the dbSNP .fst file directory - see setup documentation} + +\item{parallel_cores}{an integer, the number of cores/workers to set up the \code{future::multisession} with} + +\item{verbose}{a logical, runtime reporting} +} +\value{ +a tibble with an RSID column, or (if \code{alt_rsids} is \code{TRUE}) a list with +tibbles: \code{data} and \code{alt_rsids} +} +\description{ +Chromosome & position data to variant RSID +} diff --git a/man/coloc_analysis.Rd b/man/coloc_analysis.Rd index 5c4aa67..4f492d5 100644 --- a/man/coloc_analysis.Rd +++ b/man/coloc_analysis.Rd @@ -15,9 +15,19 @@ coloc_analysis( ) } \arguments{ -\item{first_gwas:}{first gwas to be run through coloc. This is the gwas that results will be based off} +\item{first_gwas}{first gwas to be run through coloc. This is the gwas that results will be based off} -\item{second_gwas:}{second gwas to be run through coloc} +\item{second_gwas}{second gwas to be run through coloc} + +\item{exposure_name}{name of exposure to perform coloc on} + +\item{chr}{chromosome to perform coloc on} + +\item{bp}{base pair position to perform coloc on} + +\item{range}{range to filter in base pairs} + +\item{default_n}{default sample size} } \value{ tibble of coloc results (h0 - h4) diff --git a/man/coloc_bf_bf_for_locus_pair.Rd b/man/coloc_bf_bf_for_locus_pair.Rd new file mode 100644 index 0000000..c1c94cb --- /dev/null +++ b/man/coloc_bf_bf_for_locus_pair.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc_bf.R +\name{coloc_bf_bf_for_locus_pair} +\alias{coloc_bf_bf_for_locus_pair} +\title{Run coloc::coloc.bf_bf for one pair of overlapping loci} +\usage{ +coloc_bf_bf_for_locus_pair(l1, l2, trait1_name, trait2_name, p1, p2, p12) +} +\description{ +Run coloc::coloc.bf_bf for one pair of overlapping loci +} +\keyword{internal} diff --git a/man/coloc_bf_bf_qtl_analysis.Rd b/man/coloc_bf_bf_qtl_analysis.Rd new file mode 100644 index 0000000..ffb9ee3 --- /dev/null +++ b/man/coloc_bf_bf_qtl_analysis.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{coloc_bf_bf_qtl_analysis} +\alias{coloc_bf_bf_qtl_analysis} +\title{Run BF-BF coloc between a finemapped GWAS locus and QTL data} +\usage{ +coloc_bf_bf_qtl_analysis( + finemap_dir, + qtl_gwas, + exposure_name, + chr, + bp, + study_type = "continuous", + default_n = NA, + range_bp = 1e+06 +) +} +\arguments{ +\item{finemap_dir}{finemap output directory containing locus TSV files} + +\item{qtl_gwas}{QTL GWAS data frame} + +\item{exposure_name}{exposure label} + +\item{chr}{chromosome of the MR hit} + +\item{bp}{base position of the MR hit} + +\item{study_type}{study type for LBF conversion} + +\item{default_n}{default sample size} + +\item{range_bp}{overlap window in bp (default 1e6 = ±1Mb)} +} +\value{ +tibble with coloc results or NULL +} +\description{ +Finds the finemapped locus file covering the region around chr:bp, +extracts the GWAS LBF matrix, computes QTL LBF via \code{convert_z_to_lbf}, +and runs \code{coloc::coloc.bf_bf}. +} diff --git a/man/compare_replication_across_all_gwas_permutations.Rd b/man/compare_replication_across_all_gwas_permutations.Rd index 269ac18..74b38e6 100644 --- a/man/compare_replication_across_all_gwas_permutations.Rd +++ b/man/compare_replication_across_all_gwas_permutations.Rd @@ -12,13 +12,13 @@ compare_replication_across_all_gwas_permutations( ) } \arguments{ -\item{gwas_filenames:}{list of filenames of gwases} +\item{gwas_filenames}{list of filenames of gwases} -\item{clumped_filenames:}{list of clumped list filesnames genetred from gwas, in same order as gwases} +\item{clumped_filenames}{list of clumped list filesnames genetred from gwas, in same order as gwases} -\item{result_output_file:}{data frame of all results of gwas comparisons concatenated} +\item{result_output_file}{data frame of all results of gwas comparisons concatenated} -\item{variants_output_file:}{data frame of every SNP comparison from clumped list concatenated} +\item{variants_output_file}{data frame of every SNP comparison from clumped list concatenated} } \description{ takes a vector of gwasse and associated clumps, then runs the expected vs. observed algorithm diff --git a/man/compute_ld_matrix.Rd b/man/compute_ld_matrix.Rd new file mode 100644 index 0000000..98a78f9 --- /dev/null +++ b/man/compute_ld_matrix.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/finemap.R +\name{compute_ld_matrix} +\alias{compute_ld_matrix} +\title{Compute LD correlation matrix from 1000 Genomes via plink} +\usage{ +compute_ld_matrix(rsids, chr, ancestry) +} +\arguments{ +\item{rsids}{character vector of RSIDs} + +\item{chr}{chromosome number} + +\item{ancestry}{ancestry code (EUR, EAS, etc.)} +} +\value{ +list with components \code{matrix} (numeric LD matrix) and +\code{snps} (character vector of SNP IDs in matrix order), or NULL on failure +} +\description{ +Compute LD correlation matrix from 1000 Genomes via plink +} diff --git a/man/compute_ld_r2_for_locus.Rd b/man/compute_ld_r2_for_locus.Rd new file mode 100644 index 0000000..02018bc --- /dev/null +++ b/man/compute_ld_r2_for_locus.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/locus_zoom.R +\name{compute_ld_r2_for_locus} +\alias{compute_ld_r2_for_locus} +\title{Compute r2 relative to an index SNP using the local LD reference panel} +\usage{ +compute_ld_r2_for_locus(rsids, chr, ancestry, index_snp) +} +\description{ +Compute r2 relative to an index SNP using the local LD reference panel +} +\keyword{internal} diff --git a/man/conduct_collider_bias_analysis.Rd b/man/conduct_collider_bias_analysis.Rd index 6fdd486..06eb2af 100644 --- a/man/conduct_collider_bias_analysis.Rd +++ b/man/conduct_collider_bias_analysis.Rd @@ -13,12 +13,33 @@ conduct_collider_bias_analysis( incidence_gwas, subsequent_gwas, clumped_snps_file, + adjustment_type, + adjustment_pval, collider_bias_results_file, harmonised_effects_result_file, - slopehunter_adjusted_file, - p_value_thresholds = c(0.1, 0.01, 0.001, 1e-05) + adjusted_output_file, + p_value_thresholds = thresholds ) } +\arguments{ +\item{incidence_gwas}{incidence GWAS} + +\item{subsequent_gwas}{subsequent GWAS} + +\item{clumped_snps_file}{clumped SNP list to run collider bias corrections against} + +\item{adjustment_type}{adjustment type to save the adjusted GWAS} + +\item{adjustment_pval}{adjustment pval to save the adjusted GWAS} + +\item{collider_bias_results_file}{file to save the collider bias results} + +\item{harmonised_effects_result_file}{file to save the harmonised effects} + +\item{adjusted_output_file}{file to save the adjusted GWAS} + +\item{p_value_thresholds}{p value thresholds to run corrections} +} \value{ 2 plots: one manhattan plot and one QQ plot (with lambda included) } diff --git a/man/convert_beta_to_or.Rd b/man/convert_beta_to_or.Rd new file mode 100644 index 0000000..4977177 --- /dev/null +++ b/man/convert_beta_to_or.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gwas_formatting.R +\name{convert_beta_to_or} +\alias{convert_beta_to_or} +\title{convert_beta_to_or: Given a BETA and SE, calculates the OR and lower and upper bounds} +\usage{ +convert_beta_to_or(gwas) +} +\arguments{ +\item{gwas}{dataframe with the following columns: BETA, SE} +} +\value{ +gwas with new columns OR, OR_LB, OR_UB +} +\description{ +convert_beta_to_or: Given a BETA and SE, calculates the OR and lower and upper bounds +} diff --git a/man/convert_or_to_beta.Rd b/man/convert_or_to_beta.Rd index 368dd6b..d2d2a62 100644 --- a/man/convert_or_to_beta.Rd +++ b/man/convert_or_to_beta.Rd @@ -9,7 +9,7 @@ based on this answer: https://stats.stackexchange.com/a/327684} convert_or_to_beta(gwas) } \arguments{ -\item{gwas:}{dataframe with the following columns: OR, LB (lower bound), UB (upper bound)} +\item{gwas}{dataframe with the following columns: OR, LB (lower bound), UB (upper bound)} } \value{ gwas with new columns BETA and SE diff --git a/man/convert_reference_build_via_liftover.Rd b/man/convert_reference_build_via_liftover.Rd index 60f2d78..3ed414e 100644 --- a/man/convert_reference_build_via_liftover.Rd +++ b/man/convert_reference_build_via_liftover.Rd @@ -12,13 +12,13 @@ convert_reference_build_via_liftover( ) } \arguments{ -\item{gwas:}{GWAS (file or dataframe) of standardised GWAS} +\item{gwas}{GWAS (file or dataframe) of standardised GWAS} -\item{input_reference_build:}{string reference build, found in reference_builds list} +\item{input_reference_build}{string reference build, found in reference_builds list} -\item{output_reference_build:}{string reference build that GWAS is to change to, found in reference_builds list} +\item{output_reference_build}{string reference build that GWAS is to change to, found in reference_builds list} -\item{output_file:}{optional output file name to save to} +\item{output_file}{optional output file name to save to} } \value{ gwas input is altered and returned diff --git a/man/convert_z_score_to_beta.Rd b/man/convert_z_score_to_beta.Rd deleted file mode 100644 index 3b3ea4e..0000000 --- a/man/convert_z_score_to_beta.Rd +++ /dev/null @@ -1,13 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/gwas_formatting.R -\name{convert_z_score_to_beta} -\alias{convert_z_score_to_beta} -\title{convert_z_score_to_beta -taken from here: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8432599/ under "Correlation of trans-eQTL effects"} -\usage{ -convert_z_score_to_beta(gwas) -} -\description{ -convert_z_score_to_beta -taken from here: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8432599/ under "Correlation of trans-eQTL effects" -} diff --git a/man/convert_z_to_lbf.Rd b/man/convert_z_to_lbf.Rd new file mode 100644 index 0000000..a9654ea --- /dev/null +++ b/man/convert_z_to_lbf.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/util.R +\name{convert_z_to_lbf} +\alias{convert_z_to_lbf} +\title{Generate log Bayes Factor from Z-score} +\usage{ +convert_z_to_lbf( + z, + se, + eaf, + sample_size, + study_type, + effect_priors = c(continuous = 0.15, categorical = 0.2) +) +} +\arguments{ +\item{z}{Z-score} + +\item{se}{Standard error} + +\item{eaf}{Allele frequency} + +\item{sample_size}{Sample size} + +\item{study_type}{Study type} + +\item{effect_priors}{Effect priors} +} +\value{ +Log Bayes Factor +} +\description{ +Generate log Bayes Factor from Z-score +} +\keyword{internal} diff --git a/man/estimate_variance.Rd b/man/estimate_variance.Rd new file mode 100644 index 0000000..a24864e --- /dev/null +++ b/man/estimate_variance.Rd @@ -0,0 +1,27 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/util.R +\name{estimate_variance} +\alias{estimate_variance} +\title{Estimate trait variance, internal function} +\usage{ +estimate_variance(se, eaf, n) +} +\arguments{ +\item{se}{vector of standard errors} + +\item{eaf}{vector of MAF (same length as SE)} + +\item{n}{sample size} +} +\value{ +estimated standard deviation of Y +} +\description{ +Estimate trait standard deviation given vectors of variance of coefficients, MAF and sample size +} +\details{ +Estimate is based on var(beta-hat) = var(Y) / (n * var(X)) +var(X) = 2\emph{maf}(1-maf) +so we can estimate var(Y) by regressing n*var(X) against 1/var(beta) +} +\keyword{internal} diff --git a/man/find_finemap_locus_for_region.Rd b/man/find_finemap_locus_for_region.Rd new file mode 100644 index 0000000..e7323a6 --- /dev/null +++ b/man/find_finemap_locus_for_region.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{find_finemap_locus_for_region} +\alias{find_finemap_locus_for_region} +\title{Find the finemapped locus file that covers a genomic region} +\usage{ +find_finemap_locus_for_region(finemap_dir, chr, bp, range_bp = 1e+06) +} +\description{ +Find the finemapped locus file that covers a genomic region +} +\keyword{internal} diff --git a/man/find_index_snp_from_ld_panel.Rd b/man/find_index_snp_from_ld_panel.Rd new file mode 100644 index 0000000..e9ed1a1 --- /dev/null +++ b/man/find_index_snp_from_ld_panel.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/locus_zoom.R +\name{find_index_snp_from_ld_panel} +\alias{find_index_snp_from_ld_panel} +\title{Find the top SNP near a locus center from the LD reference panel} +\usage{ +find_index_snp_from_ld_panel(chr, center_bp, window_bp, ancestry) +} +\description{ +Find the top SNP near a locus center from the LD reference panel +} +\keyword{internal} diff --git a/man/finemap_gwas.Rd b/man/finemap_gwas.Rd new file mode 100644 index 0000000..fef0205 --- /dev/null +++ b/man/finemap_gwas.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/finemap.R +\name{finemap_gwas} +\alias{finemap_gwas} +\title{Run SuSiE fine-mapping across all clumped loci in a GWAS} +\usage{ +finemap_gwas( + gwas, + clumped_file, + ancestry, + default_n, + output_finemap_dir, + completion_file = NULL, + window_kb = 1000, + max_causal = 10, + coverage = 0.95, + min_abs_corr = 0.5 +) +} +\arguments{ +\item{gwas}{filename or dataframe of a standardised GWAS} + +\item{clumped_file}{plink --clump output file} + +\item{ancestry}{ancestry code matching 1000 Genomes panel prefix (EUR, EAS, AFR, AMR, SAS)} + +\item{default_n}{GWAS sample size when not inferrable from \code{gwas}; see} + +\item{output_finemap_dir}{directory to write one +\verb{__finemap.tsv.gz} per clumped locus.} + +\item{completion_file}{path to a sentinel file written on successful completion +(one line: expected lead count). If NULL, defaults to +\verb{/finemap_complete.txt}.} + +\item{window_kb}{half-width of the fine-mapping window in kb (default 1000 = ±1 Mb)} + +\item{max_causal}{maximum number of causal signals per locus (SuSiE L, default 10)} + +\item{coverage}{credible set coverage (default 0.95)} + +\item{min_abs_corr}{minimum absolute correlation for credible sets (default 0.5)} +} +\value{ +invisibly, the combined per-SNP finemap data.table (LD-matched SNPs only) +} +\description{ +For each lead SNP from plink --clump output, extracts a window, computes an +LD matrix from the 1000 Genomes reference panel via plink, and runs +susieR::susie_rss. Writes one TSV per locus: all GWAS variants within the +genomic window around the lead SNP, with SuSiE Z-scores, credible-set +membership, and per-credible-set (independent signal) columns \code{LBF_1}, \code{LBF_2}, ... +} diff --git a/man/forest_plot.Rd b/man/forest_plot.Rd new file mode 100644 index 0000000..236fd51 --- /dev/null +++ b/man/forest_plot.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/graphs.R +\name{forest_plot} +\alias{forest_plot} +\title{forest_plot: produce a forest plot from a GWAS file} +\usage{ +forest_plot(table, title, output_file, y_column = NA) +} +\arguments{ +\item{table}{dataframe with the following columns: BETA, SE} + +\item{title}{title of the plot} + +\item{output_file}{file to save the plot} + +\item{y_column}{column to use for the y-axis} +} +\value{ +ggplot2 object +} +\description{ +forest_plot: produce a forest plot from a GWAS file +} diff --git a/man/get_file_or_dataframe.Rd b/man/get_file_or_dataframe.Rd index 8ed9354..2660d6a 100644 --- a/man/get_file_or_dataframe.Rd +++ b/man/get_file_or_dataframe.Rd @@ -9,14 +9,14 @@ accepted input file types: txt, csv, tsv, zip, gz} get_file_or_dataframe(input, columns = NULL, snps = NULL) } \arguments{ -\item{input:}{either data frame or string input of file} +\item{input}{either data frame or string input of file} -\item{columns:}{vector of strings matching column names to filter columns} +\item{columns}{vector of strings matching column names to filter columns} -\item{snps:}{vector of SNP names matching SNPs to filter rows} +\item{snps}{vector of SNP names matching SNPs to filter rows} } \value{ -output: data frame of GWAS +output data frame of GWAS } \description{ get_file_or_dataframe: function that takes either dataframe OR file name, and returns a subset of diff --git a/man/grouped_forest_plot.Rd b/man/grouped_forest_plot.Rd new file mode 100644 index 0000000..334bf12 --- /dev/null +++ b/man/grouped_forest_plot.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/graphs.R +\name{grouped_forest_plot} +\alias{grouped_forest_plot} +\title{grouped_forest_plot: produce a grouped forest plot from a GWAS file} +\usage{ +grouped_forest_plot( + table, + title, + group_column, + output_file, + p_value_column = NA, + q_stat_column = NA +) +} +\arguments{ +\item{table}{dataframe with the following columns: BETA, SE} + +\item{title}{title of the plot} + +\item{group_column}{column to use for the group} + +\item{output_file}{file to save the plot} + +\item{p_value_column}{column to use for the p-value} + +\item{q_stat_column}{column to use for the Q-statistic} +} +\value{ +ggplot2 object +} +\description{ +grouped_forest_plot: produce a grouped forest plot from a GWAS file +} diff --git a/man/gwas_region.Rd b/man/gwas_region.Rd new file mode 100644 index 0000000..8b8ccbb --- /dev/null +++ b/man/gwas_region.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/util.R +\name{gwas_region} +\alias{gwas_region} +\title{gwas_region: filter a GWAS file to a region} +\usage{ +gwas_region(gwas, chr, bp, range = 5e+05) +} +\arguments{ +\item{gwas}{dataframe with the following columns: CHR, BP} + +\item{chr}{chromosome} + +\item{bp}{base pair position} + +\item{range}{range to filter in base pairs} +} +\value{ +gwas with filtered rows +} +\description{ +gwas_region: filter a GWAS file to a region +} diff --git a/man/harmonise_gwases.Rd b/man/harmonise_gwases.Rd index 6736dc1..996cc99 100644 --- a/man/harmonise_gwases.Rd +++ b/man/harmonise_gwases.Rd @@ -8,10 +8,10 @@ across all datasets arranged to be in the same order} harmonise_gwases(...) } \arguments{ -\item{:}{elipses of gwases} +\item{...}{elipses of gwases} } \value{ -: list of harmonised gwases +list of harmonised gwases } \description{ harmonise_gwases: takes a list of gwases, get the SNPs in common diff --git a/man/load_all_finemap_loci.Rd b/man/load_all_finemap_loci.Rd new file mode 100644 index 0000000..2ee2f1d --- /dev/null +++ b/man/load_all_finemap_loci.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc_bf.R +\name{load_all_finemap_loci} +\alias{load_all_finemap_loci} +\title{Load all finemapped locus files from a set of directories} +\usage{ +load_all_finemap_loci(finemap_dirs) +} +\description{ +Load all finemapped locus files from a set of directories +} +\keyword{internal} diff --git a/man/load_qtl_gwas_for_mr_result.Rd b/man/load_qtl_gwas_for_mr_result.Rd new file mode 100644 index 0000000..7f69d3f --- /dev/null +++ b/man/load_qtl_gwas_for_mr_result.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{load_qtl_gwas_for_mr_result} +\alias{load_qtl_gwas_for_mr_result} +\title{Load the QTL GWAS file for an MR result row} +\usage{ +load_qtl_gwas_for_mr_result(mr_result, qtl_dataset) +} +\description{ +Load the QTL GWAS file for an MR result row +} +\keyword{internal} diff --git a/man/locus_zoom_coloc.Rd b/man/locus_zoom_coloc.Rd new file mode 100644 index 0000000..2526475 --- /dev/null +++ b/man/locus_zoom_coloc.Rd @@ -0,0 +1,45 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/locus_zoom.R +\name{locus_zoom_coloc} +\alias{locus_zoom_coloc} +\title{Generate locus zoom plots for significant colocalization results} +\usage{ +locus_zoom_coloc( + coloc_results_file, + gwas_files, + ancestry, + ens_db, + output_dir, + pp_h4_threshold = 0.8, + window_kb = 500, + completion_file = NULL +) +} +\arguments{ +\item{coloc_results_file}{path to coloc_results.tsv} + +\item{gwas_files}{named character vector of standardised GWAS file paths, +names must match trait labels used in coloc_results (trait1/trait2 columns)} + +\item{ancestry}{ancestry code for the LD reference panel (EUR, EAS, etc.)} + +\item{ens_db}{Ensembl database object or package name for gene annotations} + +\item{output_dir}{directory to write per-locus PNG files} + +\item{pp_h4_threshold}{minimum PP.H4.abf to consider significant (default 0.8)} + +\item{window_kb}{half-width of the region to plot in kb (default 500)} + +\item{completion_file}{path to a sentinel file written on successful completion. +If NULL, no sentinel is written.} +} +\value{ +Invisibly returns \code{NULL}. On failure, stops with an error summarising +loci that could not be plotted. +} +\description{ +For each coloc result with PP.H4.abf above the threshold, produces a stacked +locus zoom plot showing both traits at the shared locus, with LD computed +from the local 1000 Genomes reference panel. +} diff --git a/man/manhattan_and_qq.Rd b/man/manhattan_and_qq.Rd index 8aa63f1..8d05879 100644 --- a/man/manhattan_and_qq.Rd +++ b/man/manhattan_and_qq.Rd @@ -12,11 +12,13 @@ manhattan_and_qq( ) } \arguments{ -\item{gwas_filename:}{a file of a gwas that includes CHR, CP, P, and SNP} +\item{gwas_filename}{file of a gwas that includes CHR, CP, P, and SNP} -\item{name:}{name of plots to be saved (and named as a header in graph)} +\item{manhattan_filename}{file to save the manhattan plot} -\item{save_dir:}{defaults to 'scratch/results'} +\item{qq_filename}{file to save the qq plot} + +\item{include_qq}{logical flag on if to include the qq plot} } \value{ 2 plots: one manhattan plot and one QQ plot (with lambda included) diff --git a/man/miami_plot.Rd b/man/miami_plot.Rd index 30e2fd2..d51a87c 100644 --- a/man/miami_plot.Rd +++ b/man/miami_plot.Rd @@ -15,11 +15,19 @@ miami_plot( ) } \arguments{ -\item{gwas_dataframe:}{a dataframe that includes CHR, CP, P, and SNP} +\item{first_gwas_filename}{filename of first GWAS} -\item{name:}{name of plots to be saved (and named as a header in graph)} +\item{second_gwas_filename}{filename of second GWAS} -\item{save_dir:}{defaults to 'scratch/results'} +\item{miami_plot_file}{file to save the miami plot} + +\item{title}{title of the plot} + +\item{chr}{chromosome to perform miami plot on} + +\item{bp}{base pair position to perform miami plot on} + +\item{range}{range to filter in base pairs} } \description{ miami_plot: produce miami plot of GWAS data from two gwases diff --git a/man/normalise_gwas_chr.Rd b/man/normalise_gwas_chr.Rd new file mode 100644 index 0000000..f73afeb --- /dev/null +++ b/man/normalise_gwas_chr.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gwas_formatting.R +\name{normalise_gwas_chr} +\alias{normalise_gwas_chr} +\title{Normalise CHR for GWAS rows (chr-prefix, whitespace, leading zeros) so downstream joins match PLINK panels.} +\usage{ +normalise_gwas_chr(x) +} +\description{ +Normalise CHR for GWAS rows (chr-prefix, whitespace, leading zeros) so downstream joins match PLINK panels. +} +\keyword{internal} diff --git a/man/parse_bf_bf_result.Rd b/man/parse_bf_bf_result.Rd new file mode 100644 index 0000000..65fe55f --- /dev/null +++ b/man/parse_bf_bf_result.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{parse_bf_bf_result} +\alias{parse_bf_bf_result} +\title{Parse coloc.bf_bf result for QTL analysis} +\usage{ +parse_bf_bf_result(result, exposure_name, locus_name, n_snps) +} +\description{ +Parse coloc.bf_bf result for QTL analysis +} +\keyword{internal} diff --git a/man/parse_pairwise_bf_bf_result.Rd b/man/parse_pairwise_bf_bf_result.Rd new file mode 100644 index 0000000..e9ca628 --- /dev/null +++ b/man/parse_pairwise_bf_bf_result.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc_bf.R +\name{parse_pairwise_bf_bf_result} +\alias{parse_pairwise_bf_bf_result} +\title{Parse the result of coloc.bf_bf for a pairwise GWAS comparison} +\usage{ +parse_pairwise_bf_bf_result(result, trait1, trait2, locus1, locus2, n_snps) +} +\description{ +Parse the result of coloc.bf_bf for a pairwise GWAS comparison +} +\keyword{internal} diff --git a/man/populate_eaf_from_reference_panel.Rd b/man/populate_eaf_from_reference_panel.Rd new file mode 100644 index 0000000..6799682 --- /dev/null +++ b/man/populate_eaf_from_reference_panel.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_conversions.R +\name{populate_eaf_from_reference_panel} +\alias{populate_eaf_from_reference_panel} +\title{Populate missing EAF from the LD reference panel \code{.frq} file via RSID matching.} +\usage{ +populate_eaf_from_reference_panel(gwas, ancestry) +} +\arguments{ +\item{gwas}{Standardised GWAS tibble with RSID, EA, OA columns.} + +\item{ancestry}{One of EUR, EAS, AFR, AMR, SAS matching the reference panel prefix.} +} +\description{ +Expects the GWAS to have an \code{RSID} column (populated by \code{populate_rsid}) +and alleles in canonical (alphabetically sorted) order from \code{standardise_alleles}. +Only reads the lightweight \code{.frq} file; the \code{.bim} is not needed. +} diff --git a/man/populate_gene_names.Rd b/man/populate_gene_names.Rd new file mode 100644 index 0000000..4e5d2b7 --- /dev/null +++ b/man/populate_gene_names.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_conversions.R +\name{populate_gene_names} +\alias{populate_gene_names} +\title{populate_gene_names: populate the gene names from the ENSEMBL_ID column} +\usage{ +populate_gene_names(gwas) +} +\arguments{ +\item{gwas}{dataframe with the following columns: ENSEMBL_ID} +} +\value{ +gwas with new column GENE_NAME +} +\description{ +populate_gene_names: populate the gene names from the ENSEMBL_ID column +} diff --git a/man/populate_rsid.Rd b/man/populate_rsid.Rd new file mode 100644 index 0000000..4d2f85a --- /dev/null +++ b/man/populate_rsid.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_conversions.R +\name{populate_rsid} +\alias{populate_rsid} +\title{populate_rsid: populate the RSID column from the SNP column} +\usage{ +populate_rsid(gwas, option = populate_rsid_options$none) +} +\arguments{ +\item{gwas}{dataframe with the following columns: SNP} + +\item{option}{option to populate the RSID column} +} +\value{ +gwas with new column RSID +} +\description{ +populate_rsid: populate the RSID column from the SNP column +} diff --git a/man/run_bf_bf_coloc.Rd b/man/run_bf_bf_coloc.Rd new file mode 100644 index 0000000..ddc2cfc --- /dev/null +++ b/man/run_bf_bf_coloc.Rd @@ -0,0 +1,38 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc_bf.R +\name{run_bf_bf_coloc} +\alias{run_bf_bf_coloc} +\title{Run pairwise BF-BF colocalization on finemapped GWAS results} +\usage{ +run_bf_bf_coloc( + finemap_dirs, + overlap_kb = 1000, + p1 = 1e-04, + p2 = 1e-04, + p12 = 5e-06, + output_file = NULL +) +} +\arguments{ +\item{finemap_dirs}{named character vector of finemap output directories, +one per GWAS. Names are used as trait labels in the output.} + +\item{overlap_kb}{distance in kb to define overlapping signals (default 1000 = ±1 Mb)} + +\item{p1}{prior probability a SNP is associated with trait 1} + +\item{p2}{prior probability a SNP is associated with trait 2} + +\item{p12}{prior probability a SNP is associated with both traits} + +\item{output_file}{path to write the combined coloc results TSV} +} +\value{ +tibble of coloc results for every overlapping signal pair +} +\description{ +For each pair of GWAS datasets, identifies overlapping finemapped signals +(lead SNPs within ±\code{overlap_kb} kb), then runs \code{coloc::coloc.bf_bf} +on shared SNPs using the per-signal LBF columns produced by the fine-mapping +pipeline. +} diff --git a/man/run_coloc_on_list_of_datasets.Rd b/man/run_coloc_on_list_of_datasets.Rd new file mode 100644 index 0000000..c57e270 --- /dev/null +++ b/man/run_coloc_on_list_of_datasets.Rd @@ -0,0 +1,40 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{run_coloc_on_list_of_datasets} +\alias{run_coloc_on_list_of_datasets} +\title{run_coloc_on_list_of_datasets: run coloc on a list of datasets} +\usage{ +run_coloc_on_list_of_datasets( + first_gwas_list = list(), + second_gwas_list = list(), + exposure_name_list = list(), + chr_list = list(), + bp_list = list(), + range = 5e+05, + default_n = NA, + output_file +) +} +\arguments{ +\item{first_gwas_list}{list of first GWAS files} + +\item{second_gwas_list}{list of second GWAS files} + +\item{exposure_name_list}{list of exposure names} + +\item{chr_list}{list of chromosomes} + +\item{bp_list}{list of base pair positions} + +\item{range}{range to filter in base pairs} + +\item{default_n}{default sample size} + +\item{output_file}{file to save the results} +} +\value{ +tibble of coloc results +} +\description{ +run_coloc_on_list_of_datasets: run coloc on a list of datasets +} diff --git a/man/run_coloc_on_qtl_mr_results.Rd b/man/run_coloc_on_qtl_mr_results.Rd new file mode 100644 index 0000000..272e952 --- /dev/null +++ b/man/run_coloc_on_qtl_mr_results.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/coloc.R +\name{run_coloc_on_qtl_mr_results} +\alias{run_coloc_on_qtl_mr_results} +\title{Run BF-BF colocalization on significant QTL MR results using finemapped data} +\usage{ +run_coloc_on_qtl_mr_results( + mr_results_file, + finemap_dir, + qtl_dataset, + study_type = "continuous", + exposures = c(), + default_n = NA, + output_file +) +} +\arguments{ +\item{mr_results_file}{MR results file} + +\item{finemap_dir}{directory containing finemapped GWAS locus files} + +\item{qtl_dataset}{QTL dataset name (metabrain or eqtlgen)} + +\item{study_type}{study type for LBF conversion ("continuous" or "categorical")} + +\item{exposures}{optional character vector of exposures to filter} + +\item{default_n}{default sample size for QTL data} + +\item{output_file}{output file path} +} +\description{ +For each significant MR result, loads the finemapped GWAS locus that covers +the MR hit, computes LBF scores for the QTL data via \code{convert_z_to_lbf}, +and runs \code{coloc::coloc.bf_bf}. +} diff --git a/man/run_susie_for_locus.Rd b/man/run_susie_for_locus.Rd new file mode 100644 index 0000000..5f04c7c --- /dev/null +++ b/man/run_susie_for_locus.Rd @@ -0,0 +1,40 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/finemap.R +\name{run_susie_for_locus} +\alias{run_susie_for_locus} +\title{Run SuSiE on a single locus} +\usage{ +run_susie_for_locus( + z_scores, + ld_matrix, + snp_info, + n, + lead_snp, + max_causal = 10, + coverage = 0.95, + min_abs_corr = 0.5 +) +} +\arguments{ +\item{z_scores}{numeric vector of z-scores} + +\item{ld_matrix}{square LD correlation matrix} + +\item{snp_info}{data frame with SNP, CHR, BP, RSID columns (same order as z_scores)} + +\item{n}{sample size} + +\item{lead_snp}{RSID of the lead (clumped) SNP} + +\item{max_causal}{max causal signals (L)} + +\item{coverage}{credible set coverage} + +\item{min_abs_corr}{minimum absolute correlation for CS purity} +} +\value{ +data.table with SNP, CHR, BP, RSID, Z, CS, and \code{LBF_1}, \code{LBF_2}, ... per signal +} +\description{ +Run SuSiE on a single locus +} diff --git a/man/run_susie_rss_impl.Rd b/man/run_susie_rss_impl.Rd new file mode 100644 index 0000000..fef9b89 --- /dev/null +++ b/man/run_susie_rss_impl.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/finemap.R +\name{run_susie_rss_impl} +\alias{run_susie_rss_impl} +\title{Wrapper so tests can mock SuSiE with \code{testthat::local_mocked_bindings()}.} +\usage{ +run_susie_rss_impl(z, R, n, L, coverage, min_abs_corr, verbose = FALSE) +} +\description{ +Wrapper so tests can mock SuSiE with \code{testthat::local_mocked_bindings()}. +} +\keyword{internal} diff --git a/man/run_system.Rd b/man/run_system.Rd new file mode 100644 index 0000000..b14a4c0 --- /dev/null +++ b/man/run_system.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/liftover.R +\name{run_system} +\alias{run_system} +\title{Thin wrapper so tests can mock via testthat::local_mocked_bindings().} +\usage{ +run_system( + command, + wait = TRUE, + intern = FALSE, + ignore.stdout = FALSE, + ignore.stderr = FALSE +) +} +\description{ +Thin wrapper so tests can mock via testthat::local_mocked_bindings(). +} +\keyword{internal} diff --git a/man/standardise_alleles.Rd b/man/standardise_alleles.Rd new file mode 100644 index 0000000..be615b8 --- /dev/null +++ b/man/standardise_alleles.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gwas_formatting.R +\name{standardise_alleles} +\alias{standardise_alleles} +\title{Reorder EA/OA alphabetically and flip BETA, EAF, Z to match. +Rows that were flipped are marked in a logical \code{.ALLELES_FLIPPED} column +so they can be restored later by \code{unflip_alleles()}.} +\usage{ +standardise_alleles(gwas) +} +\description{ +Reorder EA/OA alphabetically and flip BETA, EAF, Z to match. +Rows that were flipped are marked in a logical \code{.ALLELES_FLIPPED} column +so they can be restored later by \code{unflip_alleles()}. +} +\keyword{internal} diff --git a/man/standardise_gwas.Rd b/man/standardise_gwas.Rd index dc935a8..dcb3aa5 100644 --- a/man/standardise_gwas.Rd +++ b/man/standardise_gwas.Rd @@ -13,32 +13,46 @@ standardise_gwas( output_reference_build = reference_builds$GRCh37, input_columns = "default", output_columns = "default", - remove_extra_columns = F + remove_extra_columns = F, + populate_eaf = FALSE, + ancestry = NULL, + flip_alleles = TRUE ) } \arguments{ -\item{gwas:}{filename of gwas to standardise or dataframe of gwas} +\item{gwas}{filename of gwas to standardise or dataframe of gwas} -\item{output_file:}{file to save standardised gwas} +\item{output_file}{file to save standardised gwas} -\item{N:}{sample size of GWAS (if GWAS has defined N column, defaults to that value)} +\item{N}{sample size of GWAS (if GWAS has defined N column, defaults to that value)} -\item{populate_rsid_option:}{if you want RSID populated or not} +\item{populate_rsid_option}{if you want RSID populated or not} -\item{input_reference_build:}{reference build of CHR and BP of data. Defaults to GRCh37} +\item{input_reference_build}{reference build of CHR and BP of data. Defaults to GRCh37} -\item{output_reference_build:}{reference build of CHR and BP of data. Defaults to GRCh37} +\item{output_reference_build}{reference build of CHR and BP of data. Defaults to GRCh37} -\item{input_columns:}{column header map used for renaming GWAS: +\item{input_columns}{column header map used for renaming GWAS: can be list() of key/value pairs, string of row in predefined_column_maps, or comma separated list of keys=values} -\item{output_columns:}{column header map used for renaming GWAS: +\item{output_columns}{column header map used for renaming GWAS: can be list() of key/value pairs, string of row in predefined_column_maps, or comma separated list of keys=values} -\item{r:}{reference build of CHR and BP of data. Defaults to GRCh37} +\item{remove_extra_columns}{logical flag on if to remove extra columns} + +\item{populate_eaf}{If TRUE, fill missing \code{EAF} from the 1000 Genomes \code{.frq} +file via RSID matching. Requires \code{ancestry}. A partial RSID population is +performed automatically when RSIDs are not already present.} + +\item{ancestry}{Ancestry code (EUR, EAS, AFR, AMR, SAS) matching the reference panel prefix; required when \code{populate_eaf} is TRUE.} + +\item{flip_alleles}{If TRUE (default), keep EA/OA in alphabetical order with flipped +BETA/EAF/Z in the output. When FALSE the original allele order and effect direction +are restored before saving; all internal steps still operate on the canonical +(alphabetically flipped) representation.} } \value{ -modified gwas: saves new gwas in {output_file} if present +modified gwas: saves new gwas in output_file if present } \description{ standardise_gwas: takes an input gwas, changes headers, standardises allelic input, adds RSID, makes life easier diff --git a/man/susie_lbf_columns.Rd b/man/susie_lbf_columns.Rd new file mode 100644 index 0000000..73bfac2 --- /dev/null +++ b/man/susie_lbf_columns.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/finemap.R +\name{susie_lbf_columns} +\alias{susie_lbf_columns} +\title{Build per-credible-set LBF columns (\code{LBF_1}, \code{LBF_2}, ...) from a SuSiE fit.} +\usage{ +susie_lbf_columns(fitted, p) +} +\description{ +Build per-credible-set LBF columns (\code{LBF_1}, \code{LBF_2}, ...) from a SuSiE fit. +} +\keyword{internal} diff --git a/man/unflip_alleles.Rd b/man/unflip_alleles.Rd new file mode 100644 index 0000000..dd6e256 --- /dev/null +++ b/man/unflip_alleles.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gwas_formatting.R +\name{unflip_alleles} +\alias{unflip_alleles} +\title{Restore original allele order for rows that were flipped by \code{standardise_alleles()}. +Reverses BETA, EAF, Z and swaps EA/OA back, then rebuilds the SNP column.} +\usage{ +unflip_alleles(gwas) +} +\description{ +Restore original allele order for rows that were flipped by \code{standardise_alleles()}. +Reverses BETA, EAF, Z and swaps EA/OA back, then rebuilds the SNP column. +} +\keyword{internal} diff --git a/man/volcano_plot.Rd b/man/volcano_plot.Rd new file mode 100644 index 0000000..1b3a388 --- /dev/null +++ b/man/volcano_plot.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/graphs.R +\name{volcano_plot} +\alias{volcano_plot} +\title{volcano_plot: produce a volcano plot from a GWAS file} +\usage{ +volcano_plot( + results_file, + title = "Volcano Plot of Results", + label = "EXPOSURE", + num_labels = 30, + output_file, + p_val = "p.adjusted" +) +} +\arguments{ +\item{results_file}{file to save the plot} + +\item{title}{title of the plot} + +\item{label}{column to use for the label (default "EXPOSURE")} + +\item{num_labels}{number of labels to include in the plot (default 30)} + +\item{output_file}{file to save the plot} + +\item{p_val}{column to use for the p-value (default "p.adjusted")} +} +\value{ +ggplot2 object +} +\description{ +volcano_plot: produce a volcano plot from a GWAS file +} diff --git a/man/vroom_snps.Rd b/man/vroom_snps.Rd index 7d059c5..6841cb5 100644 --- a/man/vroom_snps.Rd +++ b/man/vroom_snps.Rd @@ -7,6 +7,14 @@ NOTE: only works with data that has been standardised, through \code{standardise \usage{ vroom_snps(gwas_file, snps = c()) } +\arguments{ +\item{gwas_file}{file of gwas to get SNPs from} + +\item{snps}{vector of SNP names to get} +} +\value{ +dataframe of SNPs +} \description{ vroom_snps: If you only need to get a handful of SNPs out of a whole GWAS, this saves time and memory NOTE: only works with data that has been standardised, through \code{standardise_gwas}, or at least a tsv diff --git a/man/write_completion_file.Rd b/man/write_completion_file.Rd new file mode 100644 index 0000000..a4d69ff --- /dev/null +++ b/man/write_completion_file.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/locus_zoom.R +\name{write_completion_file} +\alias{write_completion_file} +\title{Write a completion sentinel file for Snakemake} +\usage{ +write_completion_file(completion_file, content = "done") +} +\description{ +Write a completion sentinel file for Snakemake +} +\keyword{internal} diff --git a/man/write_finemap_complete_marker.Rd b/man/write_finemap_complete_marker.Rd new file mode 100644 index 0000000..d7fd2c8 --- /dev/null +++ b/man/write_finemap_complete_marker.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/finemap.R +\name{write_finemap_complete_marker} +\alias{write_finemap_complete_marker} +\title{Write a completion sentinel file (one line: expected lead count) for Snakemake.} +\usage{ +write_finemap_complete_marker( + completion_file, + n_loci, + output_finemap_dir = NULL +) +} +\description{ +Write a completion sentinel file (one line: expected lead count) for Snakemake. +} +\keyword{internal} diff --git a/run_pipeline.sh b/run_pipeline.sh index 5f0cf2a..c86b188 100755 --- a/run_pipeline.sh +++ b/run_pipeline.sh @@ -1,29 +1,150 @@ #!/bin/bash set -e -if [[ $# -lt 1 ]] ; then +export GRPC_VERBOSITY=NONE +export GRPC_TRACE= +export APPTAINER_SILENT=true + +if [[ $# -lt 1 || "$1" =~ "help" ]] ; then echo """ Error: You have to provide at least 1 argument: PIPELINE_FILE (ex. snakemake/standardise_gwas.smk) - INPUT_FILE (defaults to input.json) + INPUT_FILE YAML (optional, defaults to input.yaml; omit when passing only Snakemake flags) + + Examples: + ./run_pipeline.sh snakemake/coloc.smk + ./run_pipeline.sh snakemake/coloc.smk my_input.yaml + ./run_pipeline.sh snakemake/coloc.smk --unlock + ./run_pipeline.sh snakemake/coloc.smk my_input.yaml --dry-run + + Default profile is local Apptainer (snakemake/profiles/local/). For Slurm, set + SNAKEMAKE_PROFILE=snakemake/profiles/slurm/ or create a new profile under snakemake/profiles/ """ exit 1 fi if [ -f .env ] then - export $(cat .env | xargs) + export $(grep -vE '^[[:space:]]*#' .env | xargs) else echo "Error: .env file missing" exit 1 fi -#if [$(hostname)...] set PROFILE accordingly, for when multiple HPCs are -PROFILE=snakemake/bp1/ +# GWAS working dirs: always PROJECT_DIR/data and PROJECT_DIR/results (exported as DATA_DIR / RESULTS_DIR for shell rules and Singularity binds). +if [[ -z "${PROJECT_DIR:-}" || -z "${PIPELINE_DATA_DIR:-}" ]]; then + echo "Error: PROJECT_DIR and PIPELINE_DATA_DIR must be set in .env." + exit 1 +fi +_pd="$(echo "${PROJECT_DIR}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s|/*$||')" +export DATA_DIR="${_pd}/data" +export RESULTS_DIR="${_pd}/results" +unset _pd +export PIPELINE_LOG_DIR="${DATA_DIR%/}/snakemake_logs" +mkdir -p $PIPELINE_LOG_DIR $DATA_DIR $RESULTS_DIR + +if [[ -z "${SLURM_PARTITION:-}" ]]; then + if command -v sinfo >/dev/null 2>&1; then + SLURM_PARTITION="$(sinfo -h -o '%P' 2>/dev/null | grep '\*' | head -n1 | tr -d '*[:space:]')" + fi + SLURM_PARTITION="${SLURM_PARTITION:-compute}" +fi +export SLURM_PARTITION + +if [[ -z "${SLURM_ACCOUNT:-}" ]] && command -v sacctmgr >/dev/null 2>&1; then + export USER=$(whoami) + export ACCOUNT_ID=$(sacctmgr show user withassoc format=account where user="$USER" | grep '[0-9]' | head -n1) + export SLURM_ACCOUNT="${ACCOUNT_ID:-}" +fi + +export GENEHACKMAN_EXTRA_SINGULARITY_BINDS="" +_trim_qtl="$(echo "${QTL_DATA_DIR:-}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" +if [[ -n "${_trim_qtl}" ]]; then + export GENEHACKMAN_EXTRA_SINGULARITY_BINDS="-B ${_trim_qtl%/}:${_trim_qtl%/}" +fi +unset _trim_qtl + +# Default: local Apptainer. HPC e.g.: SNAKEMAKE_PROFILE=snakemake/profiles/slurm/ or snakemake/slurm_singularity/ +PROFILE="${SNAKEMAKE_PROFILE:-snakemake/profiles/local/}" +if [[ -z "${SNAKEMAKE_PROFILE:-}" && ( "${GENEHACKMAN_LOCAL:-}" == "1" || "${GENEHACKMAN_LOCAL:-}" == "true" ) ]]; then + PROFILE="snakemake/profiles/local/" +fi +if [[ "${PROFILE}" == snakemake/profiles/local/* ]] || [[ "${PROFILE}" == snakemake/local/* ]]; then + export GENEHACKMAN_LOCAL=1 +else + unset GENEHACKMAN_LOCAL +fi SMK_FILE=$1 -export INPUT_FILE=$2 -ADDITIONAL_ARGS="${@:3}" +shift +# Second arg is the input YAML only if it is present and not a Snakemake/cli flag (e.g. --unlock). +if [[ $# -ge 1 && "${1}" != -* ]]; then + PIPELINE_INPUT="$1" + shift +else + PIPELINE_INPUT="input.yaml" +fi + +if [[ "${PROFILE}" != snakemake/profiles/local/* ]] && [[ "${PROFILE}" != snakemake/local/* ]]; then + module load ${APPTAINER_MODULE} +fi + +# Image tag: default GENEHACKMAN_VERSION from DOCKER_VERSION so pull and SIF basename stay aligned. +GENEHACKMAN_VERSION="${GENEHACKMAN_VERSION:-${DOCKER_VERSION:-}}" +SIF_VERSION="${DOCKER_VERSION:-${GENEHACKMAN_VERSION:-}}" +if [[ -z "${SIF_VERSION}" ]]; then + echo "Error: Set DOCKER_VERSION in .env (e.g. 1.1.0) so the SIF name matches the Docker image tag." + exit 1 +fi + +SIF_NAME="genehackman_${SIF_VERSION}.sif" +PIPELINE_GENOMIC_DIR="${PIPELINE_DATA_DIR%/}/genomic_data/pipeline" +PIPELINE_GENOMIC_DIR="${PIPELINE_GENOMIC_DIR%/}" +SIF_PATH="${PIPELINE_GENOMIC_DIR}/${SIF_NAME}" + +echo "SIF_PATH: ${SIF_PATH}" +if [[ ! -f "${SIF_PATH}" ]]; then + if ! mkdir -p "${PIPELINE_GENOMIC_DIR}" 2>/dev/null || [[ ! -w "${PIPELINE_GENOMIC_DIR}" ]]; then + SIF_PATH=".snakemake/singularity/${SIF_NAME}" + echo "Note: caching SIF under ${SIF_PATH} (${PIPELINE_GENOMIC_DIR} missing or not writable)" + fi +fi +mkdir -p "$(dirname "${SIF_PATH}")" + +SINGULARITY_DOCKER_REFERENCE="docker://mrcieu/genehackman:${GENEHACKMAN_VERSION}" + +# mrcieu/genehackman is linux/amd64 only. On ARM (Apple Silicon), Apptainer defaults to arm64 and +# fails with: no child with platform linux/arm64 in index. Force amd64 for the OCI/docker pull. +SINGULARITY_BUILD_ARCH_ARGS=() +if [[ "${GENEHACKMAN_SINGULARITY_NO_ARCH:-}" != "1" && "${GENEHACKMAN_SINGULARITY_NO_ARCH:-}" != "true" ]]; then + case "$(uname -m)" in + aarch64|arm64) + SINGULARITY_BUILD_ARCH_ARGS=(--arch "${GENEHACKMAN_SINGULARITY_BUILD_ARCH:-amd64}") + ;; + esac +fi + +if [[ ! -f "${SIF_PATH}" ]]; then + echo "SIF file not found: ${SIF_PATH}" + echo "Building container with singularity from: ${SINGULARITY_DOCKER_REFERENCE}" + if [[ "${#SINGULARITY_BUILD_ARCH_ARGS[@]}" -gt 0 ]]; then + echo "(host is ARM: using singularity build ${SINGULARITY_BUILD_ARCH_ARGS[*]} for amd64 image)" + fi + # Build next to the final path (same bind-mount as SIF_DIR). Do not use host /tmp: if singularity + # runs inside Lima/VM, VM /tmp is not the Mac's /tmp, so a follow-up cp from /tmp would fail. + SIF_BUILD_TMP="${SIF_PATH}.tmp.$$" + if singularity build "${SINGULARITY_BUILD_ARCH_ARGS[@]}" "${SIF_BUILD_TMP}" "${SINGULARITY_DOCKER_REFERENCE}"; then + mv -f "${SIF_BUILD_TMP}" "${SIF_PATH}" + else + rm -f "${SIF_BUILD_TMP}" + exit 1 + fi +else + echo "Using pre-built SIF file: ${SIF_PATH}" +fi + + +echo "Running pipeline with profile: ${PROFILE}" -module load apptainer/1.3.1-ksax -snakemake --snakefile ${SMK_FILE} --profile ${PROFILE} ${ADDITIONAL_ARGS} +export GENEHACKMAN_INPUT="${PIPELINE_INPUT}" +snakemake --snakefile "${SMK_FILE}" --profile "${PROFILE}" "$@" diff --git a/scripts/align_alleles.sh b/scripts/align_alleles.sh new file mode 100755 index 0000000..76bc671 --- /dev/null +++ b/scripts/align_alleles.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Usage: ./align_alleles.sh + +INPUT=$1 +OUTPUT=$2 + +if [ -z "$INPUT" ] || [ -z "$OUTPUT" ]; then + echo "Usage: ./align_alleles.sh " + exit 1 +fi + +echo "Step 1: Extracting alleles and determining alphabetical A1..." + +# This awk command looks at Column 5 and 6 of the .bim file. +# It picks the alphabetically 'smaller' one to be the new A1. +awk '{ + if ($5 < $6) + print $2, $5; + else + print $2, $6; +}' "${INPUT}.bim" > a1_list.txt + +echo "Step 2: Running PLINK to update allele order..." + +# --make-bed creates the new triplet +# --a1-allele forces the alleles in our list to be in the A1 position +plink --bfile "$INPUT" \ + --a1-allele a1_list.txt \ + --make-bed \ + --out "$OUTPUT" + +# Cleanup temporary file +rm a1_list.txt + +echo "Done! New files saved as ${OUTPUT}.bed, .bim, .fam" \ No newline at end of file diff --git a/scripts/change_reference_build_via_liftover.R b/scripts/change_reference_build_via_liftover.R index aea6c24..cdd8c5a 100644 --- a/scripts/change_reference_build_via_liftover.R +++ b/scripts/change_reference_build_via_liftover.R @@ -23,8 +23,9 @@ parser <- add_argument(parser, "--output_gwas", args <- parse_args(parser) create_dir_for_files(args$output_gwas) -convert_reference_build_via_liftover(args$input_gwas, - input_reference_build = args$input_reference_build, - output_reference_build = args$output_reference_build, - output_file = args$output_gwas -) +invisible(convert_reference_build_via_liftover( + args$input_gwas, + input_reference_build = args$input_reference_build, + output_reference_build = args$output_reference_build, + output_file = args$output_gwas +)) diff --git a/scripts/change_snp_identifiers.R b/scripts/change_snp_identifiers.R new file mode 100644 index 0000000..09b3915 --- /dev/null +++ b/scripts/change_snp_identifiers.R @@ -0,0 +1,51 @@ +source("load.R") +library(argparser, quietly = TRUE) + +parser <- arg_parser("Standardise a GWAS to be ready for the rest of the pipeline") + +parser <- add_argument(parser, "--input_gwas", + help = "Comma separated list of filenames of GWASes to standardise", + type = "character" +) +parser <- add_argument(parser, "--output_gwas", + help = "Comma separated list of filenames of the standardised GWASes", + type = "character" +) +parser <- add_argument(parser, "--input_columns", + help = "Map of column names for pipeline to change it to", + type = "character", + default = "default" +) +parser <- add_argument(parser, "--output_columns", + help = "Map of column names for pipeline to change it to", + type = "character", + default = "default" +) +parser <- add_argument(parser, "--input_build", + help = paste(c("Input reference builds, options:", reference_builds), collapse = " "), + type = "character", + default = NULL +) +parser <- add_argument(parser, "--output_build", + help = paste(c("Output reference builds, options:", reference_builds), collapse = " "), + type = "character", + default = reference_builds$GRCh37 +) +parser <- add_argument(parser, "--populate_rsid", + help = paste(c("Should GWAS Populate RSID from CHR and BP, options:", populate_rsid_options), collapse = " "), + type = "character", + default = "none" +) + +args <- parse_args(parser) +create_dir_for_files(args$output_gwas) + +invisible(change_snp_identifiers( + gwas = args$input_gwas, + output_file = args$output_gwas, + populate_rsid_option = args$populate_rsid, + input_reference_build = args$input_build, + output_reference_build = args$output_build, + input_columns = args$input_columns, + output_columns = args$output_columns +)) diff --git a/scripts/coloc_of_mr_results.R b/scripts/coloc_of_mr_results.R index 056dae6..caceb14 100644 --- a/scripts/coloc_of_mr_results.R +++ b/scripts/coloc_of_mr_results.R @@ -1,20 +1,25 @@ source("load.R") library(argparser, quietly = TRUE) -parser <- arg_parser("Perform a coloc analysis (and create a miami plot) for the coloc analysis") +parser <- arg_parser("Run BF-BF colocalization on significant MR results using finemapped GWAS data") -parser <- add_argument(parser, "--mr_results_filename", help = "filename of first GWAS", type = "character" ) -parser <- add_argument(parser, "--gwas_filename", - help = "filename of first GWAS", +parser <- add_argument(parser, "--mr_results_filename", help = "MR results filename", type = "character") +parser <- add_argument(parser, "--finemap_dir", + help = "Directory containing finemapped GWAS locus files", type = "character" ) parser <- add_argument(parser, "--N", - help = "Sample size of GWAS", + help = "Default sample size for QTL data", type = "numeric", default = 0 ) +parser <- add_argument(parser, "--study_type", + help = "Study type for LBF conversion (continuous or categorical)", + type = "character", + default = "continuous" +) parser <- add_argument(parser, "--exposures", - help = "List of exposures to perform coloc on, if none provided, runs on exposures with lowest p-vals", + help = "List of exposures to perform coloc on", type = "character", default = "", nargs = Inf @@ -25,7 +30,7 @@ parser <- add_argument(parser, "--dataset", type = "character" ) parser <- add_argument(parser, "--output_file", - help = "filename of coloc analysis to save", + help = "Output filename for coloc results", type = "character" ) @@ -33,4 +38,12 @@ args <- parse_args(parser) create_dir_for_files(args$output_file, paste0(user_results_dir, "/plots")) exposures <- split_string_into_vector(args$exposures) -run_coloc_on_qtl_mr_results(args$mr_results_filename, args$gwas_filename, args$dataset, exposures, output_file=args$output_file, default_n=args$N) +run_coloc_on_qtl_mr_results( + mr_results_file = args$mr_results_filename, + finemap_dir = args$finemap_dir, + qtl_dataset = args$dataset, + study_type = args$study_type, + exposures = exposures, + default_n = args$N, + output_file = args$output_file +) diff --git a/scripts/load.R b/scripts/load.R index 948abe3..c161923 100644 --- a/scripts/load.R +++ b/scripts/load.R @@ -2,4 +2,7 @@ source("../R/util.R") source("../R/constants.R") lapply(list.files("../R/", full.names = T, pattern="\\.R$"), source) +# Prevent stray Rplots.pdf from being written in batch mode (all pipelines). +if (!interactive()) pdf(file = nullfile()) + options(error = function() traceback(20)) diff --git a/scripts/run_coloc.R b/scripts/run_coloc.R new file mode 100644 index 0000000..9ce4710 --- /dev/null +++ b/scripts/run_coloc.R @@ -0,0 +1,60 @@ +source("load.R") +library(argparser, quietly = TRUE) + +parser <- arg_parser("Run BF-BF colocalization on finemapped GWAS results") + +parser <- add_argument(parser, "--finemap_dirs", + help = "Space-separated list of finemap output directories", + type = "character", + nargs = Inf +) +parser <- add_argument(parser, "--trait_names", + help = "Space-separated trait labels (same order as finemap_dirs)", + type = "character", + nargs = Inf +) +parser <- add_argument(parser, "--overlap_kb", + help = "Distance in kb to define overlapping signals (default 1000 = ±1 Mb)", + type = "numeric", + default = 1000 +) +parser <- add_argument(parser, "--p1", + help = "Prior: SNP associated with trait 1", + type = "numeric", + default = 1e-4 +) +parser <- add_argument(parser, "--p2", + help = "Prior: SNP associated with trait 2", + type = "numeric", + default = 1e-4 +) +parser <- add_argument(parser, "--p12", + help = "Prior: SNP associated with both traits", + type = "numeric", + default = 5e-6 +) +parser <- add_argument(parser, "--output_file", + help = "Output file for coloc results", + type = "character" +) + +args <- parse_args(parser) + +finemap_dirs <- split_string_into_vector(args$finemap_dirs) +trait_names <- split_string_into_vector(args$trait_names) + +if (length(trait_names) != length(finemap_dirs)) { + stop("Number of trait names must match number of finemap directories") +} +names(finemap_dirs) <- trait_names + +create_dir_for_files(args$output_file) + +run_bf_bf_coloc( + finemap_dirs = finemap_dirs, + overlap_kb = args$overlap_kb, + p1 = args$p1, + p2 = args$p2, + p12 = args$p12, + output_file = args$output_file +) diff --git a/scripts/run_finemap.R b/scripts/run_finemap.R new file mode 100644 index 0000000..45e70d4 --- /dev/null +++ b/scripts/run_finemap.R @@ -0,0 +1,66 @@ +source("load.R") +library(argparser, quietly = TRUE) + +parser <- arg_parser("Run SuSiE fine-mapping on clumped loci from a GWAS") + +parser <- add_argument(parser, "--gwas_filename", + help = "Standardised GWAS filename", + type = "character" +) +parser <- add_argument(parser, "--clumped_filename", + help = "plink --clump output file", + type = "character" +) +parser <- add_argument(parser, "--ancestry", + help = "Ancestry code matching 1000 Genomes panel (EUR, EAS, AFR, AMR, SAS)", + type = "character" +) +parser <- add_argument(parser, "--N", + help = "GWAS sample size", + type = "numeric" +) +parser <- add_argument(parser, "--output_finemap_dir", + help = "Directory for one __finemap.tsv.gz per clumped locus", + type = "character" +) +parser <- add_argument(parser, "--window_kb", + help = "Fine-mapping window half-width in kb (1000 = ±1 Mb)", + type = "numeric", + default = 1000 +) +parser <- add_argument(parser, "--max_causal", + help = "Maximum number of causal signals per locus (SuSiE L)", + type = "numeric", + default = 10 +) +parser <- add_argument(parser, "--coverage", + help = "Credible set coverage", + type = "numeric", + default = 0.95 +) +parser <- add_argument(parser, "--min_abs_corr", + help = "Minimum absolute correlation for credible set purity", + type = "numeric", + default = 0.5 +) +parser <- add_argument(parser, "--completion_file", + help = "Sentinel file written on successful completion", + type = "character" +) + +args <- parse_args(parser) +if (!dir.exists(args$output_finemap_dir)) { + dir.create(args$output_finemap_dir, recursive = TRUE) +} + +invisible(finemap_gwas(gwas = args$gwas_filename, + clumped_file = args$clumped_filename, + ancestry = args$ancestry, + default_n = args$N, + output_finemap_dir = args$output_finemap_dir, + completion_file = args$completion_file, + window_kb = args$window_kb, + max_causal = args$max_causal, + coverage = args$coverage, + min_abs_corr = args$min_abs_corr +)) diff --git a/scripts/run_ldsc.sh b/scripts/run_ldsc.sh index b2f087a..943ae45 100755 --- a/scripts/run_ldsc.sh +++ b/scripts/run_ldsc.sh @@ -19,6 +19,7 @@ NS=$2 ANCESTRY=$3 OUTPUT=$4 +LDSC_DIR=$PIPELINE_DATA_DIR/LDSCORE/b37_dbsnp156 LDSC_DATA=$DATA_DIR/ldsc N_LIST=(${NS//,/ }) diff --git a/scripts/run_locus_zoom.R b/scripts/run_locus_zoom.R new file mode 100644 index 0000000..df1a1ca --- /dev/null +++ b/scripts/run_locus_zoom.R @@ -0,0 +1,71 @@ +source("load.R") +library(argparser, quietly = TRUE) + +parser <- arg_parser("Generate locus zoom plots for significant coloc results") + +parser <- add_argument(parser, "--coloc_results", + help = "Path to coloc_results.tsv", + type = "character" +) +parser <- add_argument(parser, "--gwas_files", + help = "Space-separated list of standardised GWAS file paths", + type = "character", + nargs = Inf +) +parser <- add_argument(parser, "--trait_names", + help = "Space-separated trait labels (same order as gwas_files)", + type = "character", + nargs = Inf +) +parser <- add_argument(parser, "--ancestry", + help = "Ancestry for LD reference panel (EUR, EAS, etc.)", + type = "character", + default = "EUR" +) +parser <- add_argument(parser, "--pp_h4_threshold", + help = "Minimum PP.H4.abf to plot (default 0.8)", + type = "numeric", + default = 0.8 +) +parser <- add_argument(parser, "--window_kb", + help = "Half-width of plotting window in kb (default 500)", + type = "numeric", + default = 500 +) +parser <- add_argument(parser, "--output_dir", + help = "Directory to write locus zoom PNGs", + type = "character" +) +parser <- add_argument(parser, "--completion_file", + help = "Sentinel file written on successful completion", + type = "character" +) + +args <- parse_args(parser) + +gwas_files <- split_string_into_vector(args$gwas_files) +trait_names <- split_string_into_vector(args$trait_names) + +if (length(trait_names) != length(gwas_files)) { + stop("Number of trait names must match number of GWAS files") +} +names(gwas_files) <- trait_names + +if (!requireNamespace("EnsDb.Hsapiens.v75", quietly = TRUE)) { + stop("EnsDb.Hsapiens.v75 is required for gene annotations. ", + "Install with: BiocManager::install('EnsDb.Hsapiens.v75')") +} +# locuszoomr requires the EnsDb package to be loaded (not just installed); see ?locuszoomr::locus +suppressPackageStartupMessages(library(EnsDb.Hsapiens.v75)) +ens_db <- "EnsDb.Hsapiens.v75" + +locus_zoom_coloc( + coloc_results_file = args$coloc_results, + gwas_files = gwas_files, + ancestry = args$ancestry, + ens_db = ens_db, + output_dir = args$output_dir, + pp_h4_threshold = args$pp_h4_threshold, + window_kb = args$window_kb, + completion_file = args$completion_file +) diff --git a/scripts/run_multisusie.py b/scripts/run_multisusie.py new file mode 100644 index 0000000..3510099 --- /dev/null +++ b/scripts/run_multisusie.py @@ -0,0 +1,628 @@ +""" +Multi-ancestry fine-mapping using MultiSuSiE. + +For each clumped locus (union of lead SNPs across ancestries), extracts GWAS +summary statistics and LD matrices per ancestry, then runs MultiSuSiE's +multisusie_rss to produce cross-ancestry credible sets. +""" + +import argparse +import gzip +import os +import shutil +import subprocess +import sys +import tempfile +from concurrent.futures import ProcessPoolExecutor, as_completed + +import numpy as np +import pandas as pd + +import MultiSuSiE + + +def available_memory_mb(): + """Slurm allocated memory (MB) when under Slurm, else total system memory.""" + slurm = os.environ.get("SLURM_MEM_PER_NODE", "").strip() + if slurm: + try: + value = float(slurm) + if value > 0: + return value + except ValueError: + pass + if sys.platform == "linux": + try: + with open("/proc/meminfo", encoding="utf-8") as meminfo: + line = meminfo.readline() + kb = int("".join(c for c in line if c.isdigit())) + return kb / 1024 + except (OSError, ValueError): + pass + elif sys.platform == "darwin": + try: + raw = subprocess.check_output( + ["sysctl", "-n", "hw.memsize"], stderr=subprocess.DEVNULL + ) + return int(raw.strip()) / (1024 ** 2) + except (subprocess.SubprocessError, ValueError): + pass + return None + + +def available_cpus(): + """Logical CPUs on this host.""" + cpus = os.cpu_count() + return max(1, cpus if cpus is not None else 1) + + +def calculate_parallelism(max_workers=10, memory_per_worker_mb=8000): + slurm = os.environ.get("SLURM_CPUS_ON_NODE", "").strip() + if slurm: + try: + return max(1, int(slurm) - 1) + except ValueError: + pass + cpus = available_cpus() + mem = available_memory_mb() + if mem is not None and mem > 0: + by_mem = int(mem // memory_per_worker_mb) + return max(1, min(max_workers, by_mem, cpus)) + return max(1, min(max_workers, cpus)) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run MultiSuSiE multi-ancestry fine-mapping") + parser.add_argument("--gwas_files", nargs="+", required=True, + help="Standardised GWAS files (one per ancestry)") + parser.add_argument("--clumped_files", nargs="+", required=True, + help="Plink clumped files (one per ancestry)") + parser.add_argument("--ancestries", nargs="+", required=True, + help="Ancestry codes matching 1000 Genomes panels (e.g. EUR EAS)") + parser.add_argument("--sample_sizes", nargs="+", type=int, required=True, + help="GWAS sample sizes (one per ancestry)") + parser.add_argument("--output_dir", required=True, + help="Output directory for per-locus results") + parser.add_argument("--window_kb", type=int, default=1000, + help="Half-width of fine-mapping window in kb") + parser.add_argument("--max_causal", type=int, default=10, + help="Maximum number of causal signals (L)") + parser.add_argument("--coverage", type=float, default=0.95, + help="Credible set coverage") + parser.add_argument("--min_abs_corr", type=float, default=0.5, + help="Minimum absolute correlation for credible sets") + parser.add_argument("--thousand_genomes_dir", required=True, + help="Path to 1000 Genomes reference panel directory") + parser.add_argument("--completion_file", required=True, + help="Sentinel file written on success") + return parser.parse_args() + + +def read_gwas(gwas_file): + """Read a standardised GWAS TSV (possibly gzipped).""" + return pd.read_csv(gwas_file, sep="\t", dtype={"CHR": str}) + + +def read_clumped(clumped_file): + """Read plink --clump output (whitespace-delimited).""" + try: + df = pd.read_csv(clumped_file, sep=r"\s+", engine="python") + if "SNP" not in df.columns or len(df) == 0: + return pd.DataFrame(columns=["SNP", "CHR", "BP"]) + return df[["SNP", "CHR", "BP"]] + except Exception: + return pd.DataFrame(columns=["SNP", "CHR", "BP"]) + + +def get_union_loci(clumped_dfs, window_bp): + """Merge lead SNPs across ancestries into non-overlapping loci.""" + all_leads = pd.concat(clumped_dfs, ignore_index=True) + if len(all_leads) == 0: + return [] + + all_leads["CHR"] = all_leads["CHR"].astype(str) + all_leads["BP"] = all_leads["BP"].astype(int) + all_leads = all_leads.sort_values(["CHR", "BP"]).reset_index(drop=True) + + loci = [] + current_chr = None + current_start = None + current_end = None + current_leads = [] + + for _, row in all_leads.iterrows(): + bp = int(row["BP"]) + chrom = str(row["CHR"]) + if current_chr is None or chrom != current_chr or bp - current_end > window_bp: + if current_chr is not None: + loci.append({ + "chr": current_chr, + "start": current_start, + "end": current_end, + "center": int((current_start + current_end) / 2), + "leads": current_leads + }) + current_chr = chrom + current_start = bp - window_bp + current_end = bp + window_bp + current_leads = [row["SNP"]] + else: + current_end = max(current_end, bp + window_bp) + current_leads.append(row["SNP"]) + + if current_chr is not None: + loci.append({ + "chr": current_chr, + "start": current_start, + "end": current_end, + "center": int((current_start + current_end) / 2), + "leads": current_leads + }) + + return loci + + +def _read_plink_ld_matrix(ld_file, n_snps): + """Parse plink --r square output into an n_snps x n_snps matrix.""" + try: + R = np.loadtxt(ld_file) + except (OSError, ValueError): + return None + + if n_snps < 2: + return None + if R.ndim == 0: + return None + if R.ndim == 1: + if len(R) == n_snps * n_snps: + R = R.reshape(n_snps, n_snps) + else: + return None + if R.shape[0] != n_snps or R.shape[1] != n_snps: + return None + return R + + +def compute_ld_matrix(rsids, chrom, ancestry, thousand_genomes_dir): + """Compute LD correlation matrix from 1000 Genomes via plink.""" + if len(rsids) < 2: + return None, None + + bfile = os.path.join(thousand_genomes_dir, ancestry) + tmpdir = tempfile.mkdtemp(prefix="ld_") + snp_file = os.path.join(tmpdir, "snps.txt") + out_prefix = os.path.join(tmpdir, "ld") + + try: + with open(snp_file, "w", encoding="utf-8") as f: + f.write("\n".join(rsids) + "\n") + + cmd = [ + "plink1.9", + "--bfile", bfile, + "--chr", str(chrom), + "--extract", snp_file, + "--r", "square", + "--out", out_prefix, + ] + result = subprocess.run(cmd, capture_output=True, text=True) + ld_file = out_prefix + ".ld" + + if result.returncode != 0 or not os.path.exists(ld_file): + return None, None + + bim_file = out_prefix + ".bim" + if os.path.exists(bim_file): + bim = pd.read_csv(bim_file, sep="\t", header=None, usecols=[1], names=["SNP"]) + snp_order = bim["SNP"].tolist() + else: + bim_ref = bfile + ".bim" + bim = pd.read_csv( + bim_ref, sep="\t", header=None, usecols=[0, 1, 3], + names=["CHR", "SNP", "BP"], + ) + bim = bim[(bim["CHR"].astype(str) == str(chrom)) & (bim["SNP"].isin(rsids))] + bim = bim.sort_values("BP") + snp_order = bim["SNP"].tolist() + + R = _read_plink_ld_matrix(ld_file, len(snp_order)) + if R is None: + return None, None + + return R, snp_order + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _validate_multisusie_inputs(b_list, s_list, R_list, ancestries, locus_label): + """Ensure every population has the same variant count before MultiSuSiE.""" + p = int(R_list[0].shape[0]) + for i, ancestry in enumerate(ancestries): + b_arr = np.asarray(b_list[i]) + s_arr = np.asarray(s_list[i]) + r_arr = np.asarray(R_list[i]) + if ( + b_arr.ndim != 1 + or s_arr.ndim != 1 + or r_arr.ndim != 2 + or b_arr.shape[0] != p + or s_arr.shape[0] != p + or r_arr.shape != (p, p) + ): + print( + f" MultiSuSiE input mismatch for {locus_label} " + f"({ancestry}): b={b_arr.shape}, s={s_arr.shape}, " + f"R={r_arr.shape}, expected P={p}", + file=sys.stderr, + ) + return False + return True + + +def estimate_var_y(se, eaf, n): + """Estimate outcome variance from GWAS SE, EAF, and sample size.""" + se = np.asarray(se, dtype=float) + if eaf is None: + return 1.0 + eaf = np.asarray(eaf, dtype=float) + valid = np.isfinite(se) & (se > 0) & np.isfinite(eaf) & (eaf > 0) & (eaf < 1) + if valid.sum() < 2: + return 1.0 + oneover = 1.0 / (se[valid] ** 2) + nvx = 2 * n * eaf[valid] * (1 - eaf[valid]) + denom = np.dot(oneover, oneover) + if denom <= 0: + return 1.0 + var_y = np.dot(nvx, oneover) / denom + return float(var_y) if var_y > 0 else 1.0 + + +def assign_credible_set_columns(sets, n_variants): + """Map MultiSuSiE credible sets to IN_CS and CS_ID per variant.""" + in_cs = np.zeros(n_variants, dtype=int) + cs_id = np.full(n_variants, np.nan) + if sets is None or not isinstance(sets, (list, tuple)) or len(sets) == 0: + return in_cs, cs_id + + cs_list = sets[0] + include_mask = sets[3] if len(sets) > 3 else [True] * len(cs_list) + cs_counter = 0 + for l_idx, indices in enumerate(cs_list): + if l_idx >= len(include_mask) or not include_mask[l_idx] or len(indices) == 0: + continue + cs_counter += 1 + for idx in np.asarray(indices, dtype=int): + if 0 <= idx < n_variants: + in_cs[idx] = 1 + if np.isnan(cs_id[idx]): + cs_id[idx] = cs_counter + return in_cs, cs_id + + +def build_multisusie_result_df(fit, final_rsids, ref_gwas, ancestries): + """Build per-variant output table from a MultiSuSiE fit.""" + n = len(final_rsids) + result_df = pd.DataFrame({ + "SNP": ref_gwas["SNP"].values if "SNP" in ref_gwas.columns else final_rsids, + "CHR": ref_gwas["CHR"].values, + "BP": ref_gwas["BP"].values, + "RSID": final_rsids, + }) + + if hasattr(fit, "pip") and fit.pip is not None and len(fit.pip) == n: + result_df["JOINT_PIP"] = fit.pip + else: + result_df["JOINT_PIP"] = np.nan + + in_cs, cs_id = assign_credible_set_columns( + getattr(fit, "sets", None), n + ) + result_df["IN_CS"] = in_cs + result_df["CS_ID"] = pd.array(cs_id, dtype="Int64") + + if hasattr(fit, "coef") and fit.coef is not None: + for k, ancestry in enumerate(ancestries): + if fit.coef.shape[0] > k and len(fit.coef[k]) == n: + result_df[f"POSTERIOR_COEF_{ancestry}"] = fit.coef[k] + + return result_df + + +def build_credible_sets_df(fit, final_rsids): + """Summarise each MultiSuSiE credible set for locus_credible_sets.tsv.""" + sets = getattr(fit, "sets", None) + columns = ["CS_ID", "CS_SIZE", "PURITY", "MAX_PIP_RSID", "MAX_PIP", "LBF"] + if sets is None or not isinstance(sets, (list, tuple)) or len(sets) == 0: + return pd.DataFrame(columns=columns) + + cs_list = sets[0] + purity_list = sets[1] if len(sets) > 1 else [np.nan] * len(cs_list) + include_mask = sets[3] if len(sets) > 3 else [True] * len(cs_list) + pip = ( + np.asarray(fit.pip, dtype=float) + if hasattr(fit, "pip") and fit.pip is not None + else np.full(len(final_rsids), np.nan) + ) + lbf = ( + np.asarray(fit.lbf, dtype=float) + if hasattr(fit, "lbf") and fit.lbf is not None + else np.full(len(cs_list), np.nan) + ) + + rows = [] + cs_counter = 0 + for l_idx, indices in enumerate(cs_list): + if l_idx >= len(include_mask) or not include_mask[l_idx] or len(indices) == 0: + continue + cs_counter += 1 + indices = np.asarray(indices, dtype=int) + cs_pip = pip[indices] + best_local = int(np.nanargmax(cs_pip)) + best_idx = int(indices[best_local]) + rows.append({ + "CS_ID": cs_counter, + "CS_SIZE": len(indices), + "PURITY": purity_list[l_idx] if l_idx < len(purity_list) else np.nan, + "MAX_PIP_RSID": final_rsids[best_idx], + "MAX_PIP": float(cs_pip[best_local]), + "LBF": float(lbf[l_idx]) if l_idx < len(lbf) else np.nan, + }) + + return pd.DataFrame(rows, columns=columns) + + +def extract_locus_data(gwas_df, chrom, start, end): + """Extract GWAS data for a genomic window.""" + mask = ( + (gwas_df["CHR"].astype(str) == str(chrom)) & + (gwas_df["BP"].astype(float) >= start) & + (gwas_df["BP"].astype(float) <= end) & + gwas_df["BETA"].notna() & + gwas_df["SE"].notna() & + (gwas_df["SE"].astype(float) > 0) + ) + return gwas_df[mask].copy() + + +def run_multisusie_for_locus(locus, gwas_dfs, ancestries, sample_sizes, + thousand_genomes_dir, max_causal, coverage, + min_abs_corr): + """Run MultiSuSiE on a single locus across ancestries.""" + chrom = locus["chr"] + start = locus["start"] + end = locus["end"] + + ancestry_data = [] + for i, (gwas_df, ancestry, n) in enumerate(zip(gwas_dfs, ancestries, sample_sizes)): + locus_gwas = extract_locus_data(gwas_df, chrom, start, end) + if len(locus_gwas) < 2 or "RSID" not in locus_gwas.columns: + return None + locus_gwas = locus_gwas[locus_gwas["RSID"].notna() & (locus_gwas["RSID"].str.len() > 0)] + if len(locus_gwas) < 2: + return None + ancestry_data.append({ + "gwas": locus_gwas, + "ancestry": ancestry, + "n": n + }) + + shared_rsids = set(ancestry_data[0]["gwas"]["RSID"]) + for ad in ancestry_data[1:]: + shared_rsids &= set(ad["gwas"]["RSID"]) + + if len(shared_rsids) < 2: + return None + + shared_rsids_list = sorted(shared_rsids) + + ancestry_ld = [] + for ad in ancestry_data: + gwas_sub = ad["gwas"][ad["gwas"]["RSID"].isin(shared_rsids_list)].copy() + gwas_sub = gwas_sub.drop_duplicates(subset="RSID") + gwas_sub = gwas_sub.set_index("RSID").loc[shared_rsids_list].reset_index() + + R, snp_order = compute_ld_matrix( + shared_rsids_list, chrom, ad["ancestry"], thousand_genomes_dir + ) + if R is None: + return None + + ancestry_ld.append({ + "gwas_sub": gwas_sub, + "R": R, + "snp_order": snp_order, + "ancestry": ad["ancestry"], + "n": ad["n"], + }) + + final_rsids = set(shared_rsids_list) + for entry in ancestry_ld: + final_rsids &= set(entry["snp_order"]) + final_rsids = sorted(final_rsids) + if len(final_rsids) < 2: + return None + + b_list = [] + s_list = [] + R_list = [] + n_list = [] + varY_list = [] + + locus_label = f"chr{chrom}:{start}-{end}" + for entry in ancestry_ld: + snp_order = entry["snp_order"] + snp_to_idx = {rsid: idx for idx, rsid in enumerate(snp_order)} + ld_idx = [snp_to_idx[r] for r in final_rsids] + R_sub = np.ascontiguousarray(entry["R"][np.ix_(ld_idx, ld_idx)], dtype=np.float64) + + gwas_sub = entry["gwas_sub"].set_index("RSID").reindex(final_rsids) + if gwas_sub["BETA"].isna().any() or gwas_sub["SE"].isna().any(): + return None + beta = np.ascontiguousarray( + gwas_sub["BETA"].astype(float).to_numpy().ravel(), dtype=np.float64 + ) + se = np.ascontiguousarray( + gwas_sub["SE"].astype(float).to_numpy().ravel(), dtype=np.float64 + ) + eaf = gwas_sub["EAF"].astype(float).values if "EAF" in gwas_sub.columns else None + + b_list.append(beta) + s_list.append(se) + R_list.append(R_sub) + n_list.append(entry["n"]) + varY_list.append(estimate_var_y(se, eaf, entry["n"])) + + if not _validate_multisusie_inputs(b_list, s_list, R_list, ancestries, locus_label): + return None + + try: + fit = MultiSuSiE.multisusie_rss( + b_list=b_list, + s_list=s_list, + R_list=R_list, + varY_list=varY_list, + population_sizes=n_list, + L=max_causal, + coverage=coverage, + min_abs_corr=min_abs_corr, + variant_ids=final_rsids, + low_memory_mode=False, + float_type=np.float64, + ) + except Exception as e: + print(f" MultiSuSiE failed for locus chr{chrom}:{start}-{end}: {e}", file=sys.stderr) + return None + + ref_gwas = ancestry_data[0]["gwas"] + ref_gwas = ref_gwas[ref_gwas["RSID"].isin(final_rsids)] + ref_gwas = ref_gwas.drop_duplicates(subset="RSID") + ref_gwas = ref_gwas.set_index("RSID").loc[final_rsids].reset_index() + + variant_df = build_multisusie_result_df(fit, final_rsids, ref_gwas, ancestries) + cs_df = build_credible_sets_df(fit, final_rsids) + return variant_df, cs_df + + +_WORKER_CTX = {} + + +def _init_worker(ctx): + global _WORKER_CTX + _WORKER_CTX = ctx + + +def _run_locus_job(job): + """Process one locus in a worker process; writes output files.""" + i, n_loci, locus = job + ctx = _WORKER_CTX + locus_label = f"chr{locus['chr']}:{locus['start']}-{locus['end']}" + + result = run_multisusie_for_locus( + locus=locus, + gwas_dfs=ctx["gwas_dfs"], + ancestries=ctx["ancestries"], + sample_sizes=ctx["sample_sizes"], + thousand_genomes_dir=ctx["thousand_genomes_dir"], + max_causal=ctx["max_causal"], + coverage=ctx["coverage"], + min_abs_corr=ctx["min_abs_corr"], + ) + + if result is None or len(result[0]) == 0: + return i, n_loci, locus_label, False + + variant_df, cs_df = result + safe_name = f"{locus['chr']}_{locus['center']}" + output_dir = ctx["output_dir"] + out_file = os.path.join(output_dir, f"{safe_name}_finemap.tsv.gz") + variant_df.to_csv(out_file, sep="\t", index=False, compression="gzip") + cs_file = os.path.join(output_dir, f"{safe_name}_locus_credible_sets.tsv") + cs_df.to_csv(cs_file, sep="\t", index=False) + return i, n_loci, locus_label, True + + +def main(): + args = parse_args() + + n_ancestries = len(args.ancestries) + if n_ancestries < 2: + print("ERROR: MultiSuSiE requires at least 2 ancestries. " + "Use the standard finemap pipeline for single-ancestry.", + file=sys.stderr) + sys.exit(1) + + if len(args.gwas_files) != n_ancestries: + print("ERROR: Number of GWAS files must match number of ancestries.", file=sys.stderr) + sys.exit(1) + if len(args.clumped_files) != n_ancestries: + print("ERROR: Number of clumped files must match number of ancestries.", file=sys.stderr) + sys.exit(1) + if len(args.sample_sizes) != n_ancestries: + print("ERROR: Number of sample sizes must match number of ancestries.", file=sys.stderr) + sys.exit(1) + + os.makedirs(args.output_dir, exist_ok=True) + + window_bp = args.window_kb * 1000 + + print(f"Loading {n_ancestries} GWAS datasets...") + gwas_dfs = [read_gwas(f) for f in args.gwas_files] + clumped_dfs = [read_clumped(f) for f in args.clumped_files] + + print("Merging lead SNPs across ancestries into loci...") + loci = get_union_loci(clumped_dfs, window_bp) + print(f" {len(loci)} loci to fine-map") + + if len(loci) == 0: + print("No loci to fine-map.") + _write_completion(args.completion_file, 0) + return + + worker_ctx = { + "gwas_dfs": gwas_dfs, + "ancestries": args.ancestries, + "sample_sizes": args.sample_sizes, + "thousand_genomes_dir": args.thousand_genomes_dir, + "max_causal": args.max_causal, + "coverage": args.coverage, + "min_abs_corr": args.min_abs_corr, + "output_dir": args.output_dir, + } + jobs = [(i + 1, len(loci), locus) for i, locus in enumerate(loci)] + max_workers = min( + calculate_parallelism(memory_per_worker_mb=6000), + len(loci), + ) + print(f"Running up to {max_workers} loci in parallel...") + + n_success = 0 + n_fail = 0 + + with ProcessPoolExecutor( + max_workers=max_workers, + initializer=_init_worker, + initargs=(worker_ctx,), + ) as executor: + futures = [executor.submit(_run_locus_job, job) for job in jobs] + for future in as_completed(futures): + i, n_loci, locus_label, ok = future.result() + if ok: + n_success += 1 + print(f" [{i}/{n_loci}] Completed {locus_label}") + else: + n_fail += 1 + print(f" [{i}/{n_loci}] Skipped {locus_label} " + f"(insufficient shared data or MultiSuSiE failure)") + + print(f"\nMultiSuSiE complete: {n_success} loci succeeded, {n_fail} skipped/failed.") + _write_completion(args.completion_file, n_success) + + +def _write_completion(path, n_loci): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(str(n_loci) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/standardise_gwas.R b/scripts/standardise_gwas.R index 34b566e..0fcfa18 100644 --- a/scripts/standardise_gwas.R +++ b/scripts/standardise_gwas.R @@ -41,16 +41,32 @@ parser <- add_argument(parser, "--populate_rsid", type = "character", default = "none" ) +parser <- add_argument(parser, "--populate_eaf", + help = "If TRUE, fill missing EAF from LD reference panel (.bim + .frq under thousand genomes path)", + type = "logical", + default = FALSE +) +parser <- add_argument(parser, "--ancestry", + help = "Ancestry code for EAF reference (EUR, EAS, AFR, AMR, SAS); required when --populate_eaf is true", + type = "character", + default = "" +) parser <- add_argument(parser, "--remove_extra_columns", help = "Remove additional columns in GWAS that are not specified as an input column", type = "logical", default = FALSE ) +parser <- add_argument(parser, "--flip_alleles", + help = "If TRUE, reorder EA/OA alphabetically and flip BETA/EAF/Z. Set FALSE to keep alleles as-is.", + type = "logical", + default = TRUE +) args <- parse_args(parser) create_dir_for_files(args$output_gwas) -standardise_gwas(gwas = args$input_gwas, +invisible(standardise_gwas( + gwas = args$input_gwas, output_file = args$output_gwas, N = args$N, populate_rsid_option = args$populate_rsid, @@ -58,5 +74,8 @@ standardise_gwas(gwas = args$input_gwas, output_reference_build = args$output_build, input_columns = args$input_columns, output_columns = args$output_columns, - remove_extra_columns = args$remove_extra_columns -) + remove_extra_columns = args$remove_extra_columns, + populate_eaf = args$populate_eaf, + ancestry = args$ancestry, + flip_alleles = args$flip_alleles +)) diff --git a/snakemake/PIPELINES.md b/snakemake/PIPELINES.md index 533c241..2cdd71f 100644 --- a/snakemake/PIPELINES.md +++ b/snakemake/PIPELINES.md @@ -1,62 +1,176 @@ -# Pipeline Input +# Pipeline input (`input.yaml`) -All piplines will require a GWAS object or list of objects, they have the following properties +**Execution profiles:** Snakemake **`--profile`** directories live under **`snakemake/profiles/`** from the repo root (e.g. **`snakemake/profiles/local/`**, **`snakemake/profiles/slurm/`**), selected via **`SNAKEMAKE_PROFILE`** when using **`./run_pipeline.sh`**. Details: [PLATFORM_SETUP.md](../PLATFORM_SETUP.md), [README.md](../README.md). -### GWAS Object: -* `file`: Full path to file. Mandatory -* `ancestry`: one of `AFR, AMR, EAS, EUR, SAS`. Mandatory for some pipelines -* `build`: one of `GRCh36, GRCh37, GRCh38`. Default: `GRCh37` -* `columns`: object of column name maps (or string of predefined map). Explained below -* `populate_rsid`: boolean value. Populates RSID column if it doesn't exist. Default `false` +--- -**GWAS Columns:** +All pipelines consume a **YAML** file. Use **`./run_pipeline.sh .smk [path/to/input.yaml]`** — the second argument is optional and defaults to **`input.yaml`** in the working directory. If you invoke **`snakemake` directly**, pass **`--config genehackman_input=path/to/input.yaml`** (or omit for default **`input.yaml`**). Copy an example from [`input_templates/`](input_templates/). -With each GWAS file, you can specify column names ex. `{"P":"pval", ...}`, if you do not specify header names it will assume your GWAS has default names +Indented nesting defines hierarchy (`gwases:` lists `- item` blocks). Use `true` / `false` for booleans. Quote a string if it could be parsed as a YAML 1.1 keyword (e.g. column names `yes`, `no`). Validate syntax with [yamllint](https://www.yamllint.com/) if needed. -Default names : `SNP, CHR, BP, EA, OA, EAF, BETA, SE, OR, P, LOG_P, Z, OR_LB, OR_UB, RSID, N, N_CASES, ENSEMBL_ID, GENE_NAME` +Set **`PROJECT_DIR`** in **`.env`**: the pipeline uses **`PROJECT_DIR/data/`** (GWAS, clumps, …) and **`PROJECT_DIR/results/`** (finemap, coloc, plots, …). -* Mandatory Columns: `CHR, BP, EA, OA` -Effect Column Options. One of these sets are mandatory: - * `BETA, SE` - * `OR, OR_LB, OR_UB` - * `Z` -* P-value Column: `P or LOG_P` +See also: [GWAS harmonisation README](../README.md#how-it-works). -Alternatively, `columns` accepts a string of some of the more common output formats (ex: `metal`, `gwama`). There are also a list of [predefined common column maps](../inst/extdata/predefined_column_maps.csv) here. +--- -### standardise_gwas +## GWAS objects (`gwases` array) -All pipelines will standardise each GWAS before running the subsequent steps. The `SNP` field will be recalculated as `CHR:POS_EA_OA`, where EA and OA are ordered alphabetically, and the subsequent BETA and EAF will be adjusted accordingly +Each element describes one GWAS file. -* `n GWAS objects`: See above for GWAS Object explanation -* `output`: - * `build`: one of the supported reference builds. Default: `GRCh37` - * `columns`: same format as input columns. Either a object {} or predefined map string -## disease_progression +| Field | Required | Notes | +| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `file` | **Yes** | Absolute path (recommended) or path relative to the working directory from which Snakemake runs. | +| `columns` | No | Maps logical names (`CHR`, `BETA`, …) to header names in your file, or a preset string (e.g. `metal`, `gwama`, `opengwas`). Omit to use defaults. See [predefined_column_maps.csv](../inst/extdata/predefined_column_maps.csv). | +| `N` | Varies | Sample size; **required** for compare_gwases (LDSC). Used where finemapping/coloc need `N`. | +| `ancestry` | Varies | One of `AFR`, `AMR`, `EAS`, `EUR`, `SAS`. **Required** for pipelines that use LD (clumping, finemap, coloc, qtl_mr) and whenever `populate_eaf` is enabled for that GWAS. | +| `build` | No | `GRCh36`, `GRCh37`, `GRCh38`. Default `GRCh37`. | +| `populate_rsid` | No | Per-GWAS override: `true` / `false`. With clumping and `false` at both levels, RSID population is **partial** (see below). | +| `populate_eaf` | No | Per-GWAS: if `true`, fills missing `EAF` from the 1000 Genomes reference (`/genomic_data/1000genomes/b37_dbsnp156/` … `.bim` + `.frq`). Requires `ancestry`. | +| `flip_alleles` | No | If omitted, inherited from root (default **`true`**). **`false`** is **allowed only for [`standardise_gwas.smk`](standardise_gwas.smk)** (other pipelines fail at YAML load). With `false`, EA/OA order is preserved; **partial** RSID + `flip_alleles: false` remains invalid (use **`none`** or **`full`** RSID). | +| `study_type` | No | `continuous` (default) or `categorical`. Used by **qtl_mr** coloc step (LBF conversion from QTL z-scores); stored on the GWAS object for future pipeline steps that need it. | +| `remove_extra_columns` | No | Default `false`. | -* `2 GWAS objects`: Incident and Subsequent GWAS. See above for GWAS Object explanation -* `plink_clump_arguments`: arguments that are fed into the `plink --clump` call. [Options here](https://zzz.bwh.harvard.edu/plink/clump.shtml) -* `output`: 2 fields in `adjusted_gwas`, where `type` can be `slopehunter`, `cwls`, or `mr_ivw`, and `p_val` can be any of `0.1, 0.01, 0.001, 1e-5` -## compare_gwases +**Effect / P-value columns** (see also top-level `output`): -* `n GWAS objects`: See above for GWAS Object explanation - * Please also explicitly include `N` in the GWAS object, for use in the LDSC tool -* `plink_clump_arguments`: arguments that are fed into the `plink --clump` call. [Options here](https://zzz.bwh.harvard.edu/plink/clump.shtml) +- Required: `CHR`, `BP`, `EA`, `OA` +- One of: `BETA`+`SE`, or `OR`+`OR_LB`+`OR_UB`, or `Z` +- P-value: `P` or `LOG_P` -## qtl_mr +Default column names: `SNP, CHR, BP, EA, OA, EAF, BETA, SE, OR, P, LOG_P, Z, OR_LB, OR_UB, RSID, N, N_CASES, ENSEMBL_ID, GENE_NAME`. -* `1 GWAS object`: See above for GWAS Object explanation -* `qtl`: - * `dataset`: see below for options - * `subcategories`: list of subcategories to run, see below for options - * If left empty, all subcategories will be run - * `exposures`: list of specific exposures to be run. If left empty all exposures will be run in the dataset - * example: `["IL6", "CCL2"]` +--- + +## Root-level fields (shared) + +These may appear at the **root** of the YAML document (alongside `gwases`). + + +| Field | Default | Notes | +| ------------------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `is_test` | `false` | Used by tests / logging. | +| `populate_rsid` | `false` | Pipeline-wide default; each GWAS can override. **Clumping + false** here and on each GWAS ⇒ RSID population is **partial** (chrpos-based) unless you set `populate_rsid: true` where needed. | +| `populate_eaf` | `false` | Pipeline-wide default; each GWAS can override. Any GWAS with `populate_eaf: true` must include `ancestry`. | +| `flip_alleles` | `true` | Pipeline-wide default. Root or per-GWAS **`false`** only for **`standardise_gwas.smk`**; downstream pipelines require default harmonised alleles (`true`). | +| `output` | `build: GRCh37` + default column map | Only `standardise_gwas` / harmonisation: target `build` and/or output column names. | +| `finemap` | See below | Present in **finemap**, **coloc**, and **qtl_mr** inputs. Omitted keys get defaults from the parser (e.g. `window_kb` defaults to **500** unless you set it under `finemap`). | +| `coloc` | See below | Only **coloc** pipeline. | +| `plink_clump_arguments` | — | String passed to `plink --clump` for pipelines that clump. [PLINK clump options](https://zzz.bwh.harvard.edu/plink/clump.shtml). | +| `qtl` | — | Only **qtl_mr** (see below). | +| `output` (disease_progression) | — | `adjusted_gwas`: `type` (`slopehunter`, `cwls`, `mr_ivw`) and `p_val`. | + + +### `finemap` block (optional) + +Used by **finemap**, **coloc**, and **qtl_mr**. If the block is missing, defaults are filled by `parse_pipeline_input` (e.g. `window_kb` **500** unless set). + + +| Key | Meaning | Default (if missing) | +| -------------- | -------------------------------------------------------------------- | -------------------- | +| `window_kb` | Half-width of the fine-mapping window around each clumped lead (kb). | `500` | +| `max_causal` | SuSiE `L` (max causal signals per locus). | `10` | +| `coverage` | Credible set coverage. | `0.95` | +| `min_abs_corr` | Minimum |r| for credible-set purity. | `0.5` | + + +### `coloc` block (optional, **coloc** pipeline only) + + +| Key | Meaning | Default | +| ------------ | ---------------------------------------------------------------------------------------------------------------- | ------- | +| `overlap_kb` | Two finemapped signals on the same chromosome are “overlapping” if lead positions are within this distance (kb). | `1000` | +| `p1` | Coloc prior: SNP associated with trait 1. | `1e-4` | +| `p2` | Coloc prior: SNP associated with trait 2. | `1e-4` | +| `p12` | Coloc prior: SNP associated with both traits. | `5e-6` | + + +--- + +## Pipeline overview + + +| Snakemake file | Example template | One-line role | +| ------------------------------------------------------- | -------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| [`standardise_gwas.smk`](standardise_gwas.smk) | [`input_templates/standardise_gwases.yaml`](input_templates/standardise_gwases.yaml) | Harmonise GWAS columns and builds only (no PLINK). | +| [`compare_gwases.smk`](compare_gwases.smk) | [`input_templates/compare_gwases.yaml`](input_templates/compare_gwases.yaml) | Standardise → clump → heterogeneity / LDSC / expected-observed plots. | +| [`disease_progression.smk`](disease_progression.smk) | [`input_templates/disease_progression.yaml`](input_templates/disease_progression.yaml) | Incident vs subsequent GWAS: collider bias, plots, comparisons. | +| [`finemap.smk`](finemap.smk) | [`input_templates/finemap.yaml`](input_templates/finemap.yaml) | Standardise → clump → SuSiE RSS fine-mapping per GWAS (no pairwise coloc). | +| [`coloc.smk`](coloc.smk) | [`input_templates/coloc.yaml`](input_templates/coloc.yaml) | Same as finemap prep for ≥2 GWAS, then pairwise `coloc::coloc.bf_bf` + HTML report. | +| [`qtl_mr.smk`](qtl_mr.smk) | [`input_templates/qtl_mr.yaml`](input_templates/qtl_mr.yaml) | One GWAS: MR vs a QTL panel, volcano plot, BF-BF coloc for MR hits. | + + +--- + +## `standardise_gwas` + +Harmonisation only (`snakemake/standardise_gwas.smk`). + +- **GWAS**: one or more objects (see above). +- **Root**: optional `populate_rsid`, `populate_eaf`, `flip_alleles`, `output.build` / `output.columns`, `is_test`. This is the **only** pipeline that allows **`flip_alleles: false`**; when combined with **`populate_eaf`**, EAF merge runs **before** the SNP id is rebuilt without allele flipping. +- **Outputs**: `{DATA_DIR}/gwas/_std.tsv.gz` per input file. + +--- + +## `compare_gwases` + +- **GWAS**: two or more; include `**N`** on each for LDSC. +- **Root**: `plink_clump_arguments`, optional `populate_rsid` / `populate_eaf`. +- **Behaviour**: same harmonisation as other pipelines, then PLINK clumping, heterogeneity, LDSC *h*² / *r*g, expected vs observed, HTML report. + +--- + +## `disease_progression` + +- **GWAS**: exactly **two** (incident and subsequent). +- **Root**: `plink_clump_arguments`, optional `populate_rsid`, and `**output.adjusted_gwas`**: `type` ∈ `slopehunter`, `cwls`, `mr_ivw`; `p_val` ∈ `0.1`, `0.01`, `0.001`, `1e-5`. + +--- + +## `finemap` + +SuSiE fine-mapping per GWAS — no MR, no trait-trait coloc. + +- **GWAS**: one or more; each needs `**ancestry`** (LD reference for clumping and LD matrix for `susieR::susie_rss`). +- **Root**: `plink_clump_arguments`, optional `populate_rsid` / `populate_eaf`, optional `**finemap`** block (see above). Example template sets `window_kb` to `1000`. +- **Outputs** (under `RESULTS_DIR`): `finemap//` per GWAS (per-locus `*_finemap.tsv.gz`, plus `finemap_complete.txt` with one line = expected lead count `n`). Snakemake registers only that completion file as the finemap rule output (`FINEMAP_COMPLETE_TXT_PATTERN` in `snakemake/util/constants.smk`). + +--- + +## `coloc` + +Standardise → clump → finemap for **all** input GWAS, then **pairwise** BF-BF colocalisation between overlapping finemapped signals (`coloc::coloc.bf_bf`). + +- **GWAS**: **at least two**; each needs `**ancestry`**. +- **Root**: `plink_clump_arguments`, optional `populate_rsid` / `populate_eaf`, optional `**finemap`** and `**coloc**` blocks (see tables above). +- **Outputs** (under `RESULTS_DIR`): `coloc/coloc_results.tsv`, `coloc/result_coloc.html` (summary table sorted by **PP.H4.abf**; includes a **cross-ancestry disclaimer** when more than one ancestry code is used across GWAS). + +--- + +## `qtl_mr` + +- **GWAS**: exactly **one** outcome GWAS; `**ancestry`** required. +- **Environment**: QTL summary data from `**QTL_DATA_DIR`** (see `.env` / `.env_example`); if unset, defaults follow `PIPELINE_DATA_DIR/qtl_datasets` layout (`pqtl`, `metabrain`, `eqtlgen`, …). +- **Root**: `plink_clump_arguments`, optional `**finemap`** (same keys as finemap pipeline), and `**qtl**`: + - `dataset` — e.g. `metabrain`, `eqtlgen` + - `subcategory` — one sub-run, e.g. `cortex` (MetaBrain) or `cis` (eQTLGen); combined with `dataset` in output names + - `exposures` — optional list of exposure names; if empty, all exposures in that dataset run +- **`study_type`** on the GWAS object (`continuous` or `categorical`) controls LBF conversion in the coloc step. + +**Dataset / subcategory examples** (see also repository docs): + +- **metabrain**: `basalganglia`, `cerebellum`, `cortex`, `hippocampus`, `spinalcord` +- **eqtlgen**: `cis` (trans may be added later) + +Pipeline flow: standardise → clump → finemap → MR → volcano → BF-BF coloc on significant MR results (GWAS LBF vs QTL LBF from `convert_z_to_lbf`). + +--- + +## References + +- Expected vs observed: doi:10.1038/nature17671 +- LDSC: [github.com/bulik/ldsc](https://github.com/bulik/ldsc) +- `coloc`: [chr1swallace.github.io/coloc](https://chr1swallace.github.io/coloc/) +- Pipeline DOI: [10.5281/zenodo.10624713](https://doi.org/10.5281/zenodo.10624713) -Available Datasets and Subcategories: -* dataset: `metabrain` - * subcategories: `basalganglia, cerebellum, cortex, hippocampus, spinalcord` -* dataset: `eqtlgen` - * subcategories: `cis` (future: `trans`) diff --git a/snakemake/change_reference_build.smk b/snakemake/change_reference_build.smk new file mode 100644 index 0000000..362dfc1 --- /dev/null +++ b/snakemake/change_reference_build.smk @@ -0,0 +1,48 @@ +include: "util/common.smk" +singularity: get_docker_container() + +pipeline_name = "standardise_gwas" +pipeline = parse_pipeline_input() + +onstart: + print("##### GWAS Standardisation Pipeline #####") + +std_file_pattern = standardised_gwas_name("{prefix}") + +rule all: + input: expand(std_file_pattern, prefix=[g.prefix for g in pipeline.gwases]) + +rule change_reference_build: + params: + input_gwas = lambda wildcards: getattr(pipeline, wildcards.prefix).file, + input_build = lambda wildcards: getattr(pipeline, wildcards.prefix).build, + output_build = lambda wildcards: getattr(pipeline, wildcards.prefix).build, + threads: 8 + resources: + mem = lambda wildcards: f"{getattr(pipeline, wildcards.prefix).standardised_memory}G", + time = '04:00:00' + output: std_file_pattern + shell: + """ + INPUT_GWAS={params.input_gwas} + if [[ {params.input_gwas} =~ .vcf ]]; then + INPUT_GWAS=$DATA_DIR/gwas/$(basename "{params.input_gwas}" | sed s/.vcf.*/\.tsv/g) + ./vcf_to_tsv.sh {params.input_gwas} {params.vcf_columns} $INPUT_GWAS + fi + + Rscript change_snp_identifiers.R \ + --input_gwas $INPUT_GWAS \ + --output_gwas {output} \ + --input_build {params.input_build} \ + --output_build {params.output_build} \ + --input_columns {params.input_columns} \ + --populate_rsid {params.populate_rsid} + """ + + + +onsuccess: + onsuccess(pipeline_name, is_test=pipeline.is_test) + +onerror: + onerror_message(pipeline_name, is_test=pipeline.is_test) diff --git a/snakemake/coloc.smk b/snakemake/coloc.smk new file mode 100644 index 0000000..a489e2a --- /dev/null +++ b/snakemake/coloc.smk @@ -0,0 +1,149 @@ +include: "util/common.smk" +singularity: get_docker_container() + +pipeline_name = "coloc" +pipeline = parse_pipeline_input(pipeline_includes_clumping=True) + +onstart: + print("##### Colocalization Pipeline #####") + +ancestries = list([g.ancestry for g in pipeline.gwases]) +validate_ancestries(ancestries) + +unique_ancestries = list(set(ancestries)) +if len(unique_ancestries) > 1: + raise ValueError( + "Colocalization is not supported for multiple ancestries. " + f"Found {len(unique_ancestries)} unique ancestries: {unique_ancestries}. " + "Please use the same ancestry for all GWAS inputs in the coloc pipeline, " + "or use the finemap pipeline for multi-ancestry fine-mapping with MultiSuSiE." + ) + +finemap_opts = getattr(pipeline, "finemap", SimpleNamespace()) +finemap_window_kb = getattr(finemap_opts, "window_kb", 1000) +finemap_max_causal = getattr(finemap_opts, "max_causal", 10) +finemap_coverage = getattr(finemap_opts, "coverage", 0.95) +finemap_min_abs_corr = getattr(finemap_opts, "min_abs_corr", 0.5) + +coloc_opts = getattr(pipeline, "coloc", SimpleNamespace()) +coloc_overlap_kb = getattr(coloc_opts, "overlap_kb", 1000) +coloc_p1 = getattr(coloc_opts, "p1", 1e-4) +coloc_p2 = getattr(coloc_opts, "p2", 1e-4) +coloc_p12 = getattr(coloc_opts, "p12", 5e-6) + +std_file_pattern = standardised_gwas_name("{prefix}") + +for g in pipeline.gwases: + g.finemap_dir = RESULTS_DIR + "finemap/" + g.prefix + +trait_names = [g.prefix for g in pipeline.gwases] +finemap_dirs_str = " ".join([g.finemap_dir for g in pipeline.gwases]) +trait_names_str = " ".join(trait_names) + +coloc_results = RESULTS_DIR + "coloc/coloc_results.tsv" +results_file = RESULTS_DIR + "coloc/result_coloc.html" +locus_zoom_dir = RESULTS_DIR + "coloc/locus_zoom/" +locus_zoom_done = RESULTS_DIR + "coloc/locus_zoom/locus_zoom_complete.txt" + +gwas_files_str = " ".join([g.standardised_gwas for g in pipeline.gwases]) +first_ancestry = pipeline.gwases[0].ancestry + +trait_bar = "|".join(trait_names) +ancestry_bar = "|".join([g.ancestry for g in pipeline.gwases]) + +rule all: + input: + expand(std_file_pattern, prefix=trait_names), + expand(FINEMAP_COMPLETE_TXT_PATTERN, prefix=trait_names), + coloc_results, + results_file, + locus_zoom_done + +include: "rules/standardise_rule.smk" +include: "rules/clumping_rule.smk" +include: "rules/finemap_rule.smk" + +rule run_coloc: + resources: + mem = "16G", + time = "02:00:00" + input: + finemap_complete = expand(FINEMAP_COMPLETE_TXT_PATTERN, prefix=trait_names) + params: + finemap_dirs = finemap_dirs_str, + trait_names = trait_names_str, + overlap_kb = coloc_overlap_kb, + p1 = coloc_p1, + p2 = coloc_p2, + p12 = coloc_p12 + output: + coloc_results = coloc_results + shell: + """ + Rscript run_coloc.R \ + --finemap_dirs {params.finemap_dirs} \ + --trait_names {params.trait_names} \ + --overlap_kb {params.overlap_kb} \ + --p1 {params.p1} \ + --p2 {params.p2} \ + --p12 {params.p12} \ + --output_file {output.coloc_results} + """ + +rmd_report_params = { + "coloc_results": coloc_results, + "traits_ordered": trait_bar, + "ancestries_ordered": ancestry_bar, + "overlap_kb": str(coloc_overlap_kb), + "p1": str(coloc_p1), + "p2": str(coloc_p2), + "p12": str(coloc_p12), +} +rmd_report_param_string = turn_dict_into_cli_string(rmd_report_params) + +rule create_results_file: + threads: 4 + resources: + mem = "8G", + input: + coloc_results + output: results_file + shell: + """ + Rscript create_results_file.R \ + --rmd_file /home/R/markdown/coloc.Rmd \ + --params '{rmd_report_param_string}' \ + --output_file "{output}" + """ + +rule run_locus_zoom: + resources: + mem = "12G", + time = "01:00:00" + input: + coloc_results = coloc_results, + gwas_files = [g.standardised_gwas for g in pipeline.gwases] + params: + gwas_files = gwas_files_str, + trait_names = trait_names_str, + ancestry = first_ancestry, + output_dir = locus_zoom_dir + output: + done = locus_zoom_done + shell: + """ + Rscript run_locus_zoom.R \ + --coloc_results {input.coloc_results} \ + --gwas_files {params.gwas_files} \ + --trait_names {params.trait_names} \ + --ancestry {params.ancestry} \ + --output_dir {params.output_dir} \ + --completion_file {output.done} + """ + +onsuccess: + files = [g.finemap_dir for g in pipeline.gwases] + [coloc_results, results_file, locus_zoom_dir] + onsuccess(pipeline_name, files, results_file, is_test=pipeline.is_test) + +onerror: + onerror_message(pipeline_name, is_test=pipeline.is_test) diff --git a/snakemake/compare_gwases.smk b/snakemake/compare_gwases.smk index d380ccc..da5679e 100644 --- a/snakemake/compare_gwases.smk +++ b/snakemake/compare_gwases.smk @@ -16,8 +16,8 @@ validate_ancestries(ancestries) expected_vs_observed_results = RESULTS_DIR + "gwas_comparison/expected_vs_observed_outcomes.tsv" expected_vs_observed_variants = RESULTS_DIR + "gwas_comparison/expected_vs_observed_variants.tsv" heterogeneity_scores = RESULTS_DIR + "gwas_comparison/heterogeneity_scores.tsv" -heterogeneity_plot = RESULTS_DIR + "plots/ancestry_heterogeneity_plot.png" -heterogeneity_snp_comparison = RESULTS_DIR + "plots/ancestry_heterogeneity_snp_comparison.png" +heterogeneity_plot = RESULTS_DIR + "plots/snp_heterogeneity_plot.png" +heterogeneity_snp_comparison = RESULTS_DIR + "plots/snps_with_heterogeneity_forest_plot.png" results_file = RESULTS_DIR + "gwas_comparison/result_compare_gwases.html" std_file_pattern = standardised_gwas_name("{prefix}") diff --git a/snakemake/disease_progression.smk b/snakemake/disease_progression.smk index 9e95014..8d405c5 100644 --- a/snakemake/disease_progression.smk +++ b/snakemake/disease_progression.smk @@ -1,5 +1,5 @@ include: "util/common.smk" -singularity: get_docker_container() +container: get_docker_container() pipeline_name = "disease_progression" pipeline = parse_pipeline_input(pipeline_includes_clumping=True) @@ -65,14 +65,14 @@ rule collider_bias_correction: --collider_bias_adjusted_output {output.adjusted_results} """ -rule unadjusted_miami_plot: +rule incident_vs_adjusted_miami_plot: threads: 4 resources: mem = "16G", time = "02:00:00" input: first_gwas = incident.standardised_gwas, - second_gwas = subsequent.standardised_gwas, + second_gwas = adjusted_results output: unadjusted_miami_plot shell: """ @@ -80,16 +80,16 @@ rule unadjusted_miami_plot: --first_gwas {input.first_gwas} \ --second_gwas {input.second_gwas} \ --miami_filename {output} \ - --title "Comparing Incidence and Subsequent GWAS" + --title "Comparing Incident GWAS and Collider Bias Adjusted GWAS" """ -rule collider_bias_adjusted_miami_plot: +rule subsequent_vs_adjusted_miami_plot: threads: 4 resources: mem = "16G", time = "02:00:00" input: - first_gwas = incident.standardised_gwas, + first_gwas = subsequent.standardised_gwas, second_gwas = adjusted_results output: collider_bias_adjusted_miami_plot shell: @@ -98,7 +98,7 @@ rule collider_bias_adjusted_miami_plot: --first_gwas {input.first_gwas} \ --second_gwas {input.second_gwas} \ --miami_filename {output} \ - --title "Comparing Incidence and Collider Bias Adjusted Subsequent GWAS" + --title "Comparing Subsequent GWAS and Collider Bias Adjusted GWAS" """ rule compare_original_observed_vs_expected_gwas: diff --git a/snakemake/finemap.smk b/snakemake/finemap.smk new file mode 100644 index 0000000..f54d29a --- /dev/null +++ b/snakemake/finemap.smk @@ -0,0 +1,61 @@ +include: "util/common.smk" +singularity: get_docker_container() + +pipeline_name = "finemap" +pipeline = parse_pipeline_input(pipeline_includes_clumping=True) + +onstart: + print("##### SuSiE Fine-Mapping Pipeline #####") + +ancestries = list([g.ancestry for g in pipeline.gwases]) +validate_ancestries(ancestries) +unique_ancestries = list(set(ancestries)) +if len(unique_ancestries) > 1 and len(unique_ancestries) < len(ancestries): + raise ValueError( + "Finemap requires either the same ancestry for all GWAS inputs, " + "or a distinct ancestry per GWAS (no duplicates). " + f"Found ancestries: {ancestries}" + ) +is_multi_ancestry = len(unique_ancestries) > 1 + +finemap_opts = getattr(pipeline, "finemap", SimpleNamespace()) +finemap_window_kb = getattr(finemap_opts, "window_kb", 1000) +finemap_max_causal = getattr(finemap_opts, "max_causal", 10) +finemap_coverage = getattr(finemap_opts, "coverage", 0.95) +finemap_min_abs_corr = getattr(finemap_opts, "min_abs_corr", 0.5) + +std_file_pattern = standardised_gwas_name("{prefix}") + +for g in pipeline.gwases: + g.finemap_dir = RESULTS_DIR + "finemap/" + g.prefix + +MULTI_FINEMAP_COMPLETE = RESULTS_DIR + "finemap/multi_ancestry/finemap_complete.txt" + +if is_multi_ancestry: + rule all: + input: + expand(std_file_pattern, prefix=[g.prefix for g in pipeline.gwases]), + MULTI_FINEMAP_COMPLETE +else: + rule all: + input: + expand(std_file_pattern, prefix=[g.prefix for g in pipeline.gwases]), + expand(FINEMAP_COMPLETE_TXT_PATTERN, prefix=[g.prefix for g in pipeline.gwases]) + +include: "rules/standardise_rule.smk" +include: "rules/clumping_rule.smk" + +if is_multi_ancestry: + include: "rules/finemap_multi_ancestry_rule.smk" +else: + include: "rules/finemap_rule.smk" + +onsuccess: + if is_multi_ancestry: + files = [RESULTS_DIR + "finemap/multi_ancestry"] + else: + files = [g.finemap_dir for g in pipeline.gwases] + onsuccess(pipeline_name, files, is_test=pipeline.is_test) + +onerror: + onerror_message(pipeline_name, is_test=pipeline.is_test) diff --git a/snakemake/input_templates/coloc.yaml b/snakemake/input_templates/coloc.yaml new file mode 100644 index 0000000..8d11116 --- /dev/null +++ b/snakemake/input_templates/coloc.yaml @@ -0,0 +1,21 @@ +gwases: + - file: "" + columns: {} + N: 0 + ancestry: EUR + - file: "" + columns: {} + N: 0 + ancestry: EUR +plink_clump_arguments: "--clump-p1 5e-8 --clump-r2 0.01 --clump-kb 1000" +finemap: + window_kb: 1000 + max_causal: 10 + coverage: 0.95 + min_abs_corr: 0.5 +coloc: + overlap_kb: 1000 + p1: 1e-4 + p2: 1e-4 + p12: 5e-6 +populate_rsid: false diff --git a/snakemake/input_templates/compare_gwases.json b/snakemake/input_templates/compare_gwases.json deleted file mode 100644 index 4edd40f..0000000 --- a/snakemake/input_templates/compare_gwases.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "gwases": [ - { - "file": "", - "columns": {"SNP":"", "CHR":"", "BP":"", "EA":"", "OA":"", "EAF":"", "P":"", "BETA":"", "SE":""}, - "N": 10000, - "ancestry": "EUR" - }, - { - "file": "", - "columns": {}, - "N": 10000, - "ancestry": "AFR" - }, - { - "file": "", - "columns": {}, - "N": 10000, - "ancestry": "EAS" - } - ], - "plink_clump_arguments": "--clump-p1 0.00000005", - "populate_rsid": false -} diff --git a/snakemake/input_templates/compare_gwases.yaml b/snakemake/input_templates/compare_gwases.yaml new file mode 100644 index 0000000..9f3dfdc --- /dev/null +++ b/snakemake/input_templates/compare_gwases.yaml @@ -0,0 +1,24 @@ +gwases: + - file: "" + columns: + SNP: "" + CHR: "" + BP: "" + EA: "" + OA: "" + EAF: "" + P: "" + BETA: "" + SE: "" + N: 10000 + ancestry: EUR + - file: "" + columns: {} + N: 10000 + ancestry: AFR + - file: "" + columns: {} + N: 10000 + ancestry: EAS +plink_clump_arguments: "--clump-p1 0.00000005" +populate_rsid: false diff --git a/snakemake/input_templates/disease_progression.json b/snakemake/input_templates/disease_progression.json deleted file mode 100644 index 13badb9..0000000 --- a/snakemake/input_templates/disease_progression.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "gwases": [ - { - "file": "", - "columns": {"CHR":"", "BP":"", "EA":"", "OA":"", "EAF":"", "P":"", "BETA":"", "SE":""}, - "ancestry": "EUR" - }, - { - "file": "", - "columns": {}, - "ancestry": "EAS" - } - ], - "plink_clump_arguments": "--clump-p2 0.001 --clump-r2 0.3 --clump-kb 500", - "populate_rsid": false, - "output": { - "adjusted_gwas": - { - "type" : "cwls", - "p_val": 0.001 - } - } -} diff --git a/snakemake/input_templates/disease_progression.yaml b/snakemake/input_templates/disease_progression.yaml new file mode 100644 index 0000000..add6d80 --- /dev/null +++ b/snakemake/input_templates/disease_progression.yaml @@ -0,0 +1,21 @@ +gwases: + - file: "" + columns: + CHR: "" + BP: "" + EA: "" + OA: "" + EAF: "" + P: "" + BETA: "" + SE: "" + ancestry: EUR + - file: "" + columns: {} + ancestry: EAS +plink_clump_arguments: "--clump-p2 0.001 --clump-r2 0.3 --clump-kb 500" +populate_rsid: false +output: + adjusted_gwas: + type: cwls + p_val: 0.001 diff --git a/snakemake/input_templates/finemap.yaml b/snakemake/input_templates/finemap.yaml new file mode 100644 index 0000000..c679239 --- /dev/null +++ b/snakemake/input_templates/finemap.yaml @@ -0,0 +1,12 @@ +gwases: + - file: "" + columns: {} + N: 0 + ancestry: EUR +plink_clump_arguments: "--clump-p1 5e-8 --clump-r2 0.01 --clump-kb 1000" +finemap: + window_kb: 1000 + max_causal: 10 + coverage: 0.95 + min_abs_corr: 0.5 +populate_rsid: false diff --git a/snakemake/input_templates/qtl_mr.json b/snakemake/input_templates/qtl_mr.json deleted file mode 100644 index a600e15..0000000 --- a/snakemake/input_templates/qtl_mr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "gwases": [ - { - "file": "", - "columns": {"CHR":"", "BP":"", "EA":"", "OA":"", "EAF":"", "P":"", "BETA":"", "SE":""}, - "ancestry": "EUR", - "N": 0 - } - ], - "qtl": { - "dataset": "", - "subcategory": "", - "exposures": [] - }, - "populate_rsid": false -} diff --git a/snakemake/input_templates/qtl_mr.yaml b/snakemake/input_templates/qtl_mr.yaml new file mode 100644 index 0000000..16021b9 --- /dev/null +++ b/snakemake/input_templates/qtl_mr.yaml @@ -0,0 +1,25 @@ +gwases: + - file: "" + columns: + CHR: "" + BP: "" + EA: "" + OA: "" + EAF: "" + P: "" + BETA: "" + SE: "" + ancestry: EUR + N: 0 + study_type: continuous +plink_clump_arguments: "--clump-p1 5e-8 --clump-r2 0.01 --clump-kb 1000" +finemap: + window_kb: 1000 + max_causal: 10 + coverage: 0.95 + min_abs_corr: 0.5 +qtl: + dataset: "" + subcategory: "" + exposures: [] +populate_rsid: false diff --git a/snakemake/input_templates/standardise_gwases.json b/snakemake/input_templates/standardise_gwases.json deleted file mode 100644 index 914a1fc..0000000 --- a/snakemake/input_templates/standardise_gwases.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "gwases": [ - { - "file": "", - "columns": {"CHR":"", "BP":"", "EA":"", "OA":"", "EAF":"", "P":"", "BETA":"", "SE":""}, - "build": "GRCh37" - }, - { - "file": "", - "columns": {} - } - ], - "output": { - "build": "GRCh37", - "columns": {} - }, - "populate_rsid": false -} diff --git a/snakemake/input_templates/standardise_gwases.yaml b/snakemake/input_templates/standardise_gwases.yaml new file mode 100644 index 0000000..ba16dfc --- /dev/null +++ b/snakemake/input_templates/standardise_gwases.yaml @@ -0,0 +1,17 @@ +gwases: + - file: /Users/wt23152/Documents/Projects/scratch/020/data/gwas_with_lbfs.tsv.gz + columns: + CHR: CHR + BP: BP + EA: EA + OA: OA + EAF: EAF + P: P + BETA: BETA + SE: SE + build: GRCh38 + ancestry: EUR + populate_eaf: false +output: + build: GRCh37 +populate_rsid: false diff --git a/snakemake/profiles/local/config.yaml b/snakemake/profiles/local/config.yaml new file mode 100644 index 0000000..5844098 --- /dev/null +++ b/snakemake/profiles/local/config.yaml @@ -0,0 +1,27 @@ +# Local profile: Snakemake 7 expects a `singularity` executable; Apptainer provides `apptainer`. +# run_pipeline.sh prepends a tiny shim when only apptainer exists. +use-singularity: True +singularity-args: >- + -B $(pwd)/R:/home/R + -B $(pwd)/scripts:/home/scripts + -B $(pwd)/inst:/home/inst + -B ${HOME}:${HOME} + ${WORK:+-B ${WORK}:${WORK}} + -B ${DATA_DIR}:${DATA_DIR} + -B ${RESULTS_DIR}:${RESULTS_DIR} + -B ${PIPELINE_DATA_DIR}:${PIPELINE_DATA_DIR} + -B ${QTL_DATA_DIR}:${QTL_DATA_DIR} + -B /tmp:/tmp + --env-file .env + --pwd /home/scripts/ + +jobs: 1 +default-resources: + - mem_mb=16000 + - threads=8 +local-cores: 1 +latency-wait: 10 +keep-going: True +rerun-incomplete: True +printshellcmds: True +scheduler: greedy diff --git a/snakemake/bc4/config.yaml b/snakemake/profiles/slurm/config.yaml similarity index 53% rename from snakemake/bc4/config.yaml rename to snakemake/profiles/slurm/config.yaml index ca528ca..41439c5 100644 --- a/snakemake/bc4/config.yaml +++ b/snakemake/profiles/slurm/config.yaml @@ -1,9 +1,8 @@ cluster: - export USER=$(whoami) && - export ACCOUNT_ID=$(sacctmgr show user withassoc format=account where user=$USER | grep '[0-9]' | head -n1) && - mkdir -p /user/work/$USER/slurm_logs && + export PARTITION="$SLURM_PARTITION" && + export ACCOUNT_ID="$SLURM_ACCOUNT" && sbatch - --partition={resources.partition} + --partition=$PARTITION --account=$ACCOUNT_ID --nodes={resources.nodes} --ntasks-per-node={resources.ntasks_per_node} @@ -12,10 +11,9 @@ cluster: --cpus-per-task={threads} --mem={resources.mem} --job-name={rule}-{wildcards} - --output=/user/work/$USER/slurm_logs/{rule}_%j.out + --output="$PIPELINE_LOG_DIR/{rule}_%j.out" default-resources: - time='01:00:00' - - partition=mrcieu - mem='1G' - ntasks_per_node=1 - nodes=1 @@ -28,6 +26,18 @@ jobs: 10 keep-going: True rerun-incomplete: True use-singularity: True -singularity-args: '-B $(pwd)/R:/home/R -B $(pwd)/scripts:/home/scripts -B $(pwd)/inst:/home/inst -B /user/work/$(whoami) -B /user/home/$(whoami) -B /mnt/storage/private/mrcieu/data/ --env-file .env --pwd /home/scripts/' +singularity-args: >- + -B $(pwd)/R:/home/R + -B $(pwd)/scripts:/home/scripts + -B $(pwd)/inst:/home/inst + -B ${HOME}:${HOME} + ${WORK:+-B ${WORK}:${WORK}} + -B ${DATA_DIR}:${DATA_DIR} + -B ${RESULTS_DIR}:${RESULTS_DIR} + -B ${PIPELINE_DATA_DIR}:${PIPELINE_DATA_DIR} + ${GENEHACKMAN_EXTRA_SINGULARITY_BINDS} + -B /tmp:/tmp + --env-file .env + --pwd /home/scripts/ printshellcmds: True scheduler: greedy diff --git a/snakemake/bp1/config.yaml b/snakemake/profiles/uob-bp1/config.yaml similarity index 90% rename from snakemake/bp1/config.yaml rename to snakemake/profiles/uob-bp1/config.yaml index d7348a2..bce7c53 100644 --- a/snakemake/bp1/config.yaml +++ b/snakemake/profiles/uob-bp1/config.yaml @@ -28,6 +28,6 @@ jobs: 10 keep-going: True rerun-incomplete: True use-singularity: True -singularity-args: '-B $(pwd)/R:/home/R -B $(pwd)/scripts:/home/scripts -B $(pwd)/inst:/home/inst -B /user/work/$(whoami) -B /user/home/$(whoami) -B /bp1/mrcieu1/data/ --env-file .env --pwd /home/scripts/' +singularity-args: '-B $(pwd)/R:/home/R -B $(pwd)/scripts:/home/scripts -B $(pwd)/inst:/home/inst -B /user/work/$(whoami) -B /user/home/$(whoami) -B /bp1/mrcieu1/data/ -B ${QTL_DATA_DIR}:${QTL_DATA_DIR} --env-file .env --pwd /home/scripts/' printshellcmds: True scheduler: greedy diff --git a/snakemake/qtl_mr.smk b/snakemake/qtl_mr.smk index 311c45b..bc6c765 100644 --- a/snakemake/qtl_mr.smk +++ b/snakemake/qtl_mr.smk @@ -1,18 +1,33 @@ include: "util/common.smk" -singularity: get_docker_container() +container: get_docker_container() pipeline_name = "qtl_mr" -pipeline = parse_pipeline_input() +pipeline = parse_pipeline_input(pipeline_includes_clumping=True) gwas = pipeline.gwases[0] onstart: print("##### QTL MR Pipeline #####") +ancestries = list([g.ancestry for g in pipeline.gwases]) +validate_ancestries(ancestries) + +finemap_opts = getattr(pipeline, "finemap", SimpleNamespace()) +finemap_window_kb = getattr(finemap_opts, "window_kb", 1000) +finemap_max_causal = getattr(finemap_opts, "max_causal", 10) +finemap_coverage = getattr(finemap_opts, "coverage", 0.95) +finemap_min_abs_corr = getattr(finemap_opts, "min_abs_corr", 0.5) + +qtl_opts = getattr(pipeline, "qtl", SimpleNamespace()) +study_type = gwas.study_type + qtl_name = pipeline.qtl.dataset + "_" + pipeline.qtl.subcategory if not hasattr(pipeline.qtl, "exposures"): pipeline.qtl.exposures = [] exposures_string = " ".join(pipeline.qtl.exposures) gwas_prefix = file_prefix(gwas.file) +gwas.finemap_dir = RESULTS_DIR + "finemap/" + gwas_prefix +finemap_complete_file = expand(FINEMAP_COMPLETE_TXT_PATTERN, prefix=[gwas_prefix])[0] + mr_results = RESULTS_DIR + "mr/" + gwas_prefix + "_" + qtl_name + ".tsv.gz" coloc_results = RESULTS_DIR + "mr/coloc_" + gwas_prefix + "_" + qtl_name + ".tsv" volcano_plot = RESULTS_DIR + "plots/volcano_plot_" + gwas_prefix + "_" + qtl_name + ".png" @@ -21,9 +36,17 @@ results_file = RESULTS_DIR + "mr/result_" + qtl_name + "_" + gwas_prefix + ".ht std_file_pattern = standardised_gwas_name("{prefix}") rule all: - input: gwas.standardised_gwas, mr_results, volcano_plot, coloc_results, results_file + input: + gwas.standardised_gwas, + expand(FINEMAP_COMPLETE_TXT_PATTERN, prefix=[gwas_prefix]), + mr_results, + volcano_plot, + coloc_results, + results_file include: "rules/standardise_rule.smk" +include: "rules/clumping_rule.smk" +include: "rules/finemap_rule.smk" rule run_mr_against_qtl_datasets: threads: 4 @@ -64,24 +87,28 @@ rule run_coloc_analysis_of_significant_mr_results: resources: mem = "16G" input: - mr_results = mr_results + mr_results = mr_results, + finemap_complete = expand(FINEMAP_COMPLETE_TXT_PATTERN, prefix=[gwas_prefix]) params: - gwas = gwas.standardised_gwas, + finemap_dir = gwas.finemap_dir, N = gwas.N, + study_type = study_type, exposures = f"--exposures {exposures_string}" if exposures_string else "" output: coloc_results shell: """ Rscript coloc_of_mr_results.R \ --mr_results_filename {input.mr_results} \ - --gwas_filename {params.gwas} \ + --finemap_dir {params.finemap_dir} \ --N {params.N} \ + --study_type {params.study_type} \ --dataset {pipeline.qtl.dataset} {params.exposures} \ - --output_file {output} + --output_file {output} """ files_created = { "gwas": gwas.standardised_gwas, + "finemap_complete": finemap_complete_file, "mr_results": mr_results, "volcano_plot": volcano_plot, "coloc_results": coloc_results diff --git a/snakemake/rules/clumping_rule.smk b/snakemake/rules/clumping_rule.smk index ed7e0c4..55a3a5e 100644 --- a/snakemake/rules/clumping_rule.smk +++ b/snakemake/rules/clumping_rule.smk @@ -7,27 +7,25 @@ for g in pipeline.gwases: g.clumped_file = g.clumped_snp_prefix + ".clumped" setattr(pipeline, g.prefix, g) -rule clump_gwases: +CLUMPED_FILE_PATTERN = clump_dir + "{prefix}.clumped" + +rule clump_gwas: + threads: 4 resources: - mem = "4G" + mem = "8G" input: - gwases = [g.standardised_gwas for g in pipeline.gwases] - output: [g.clumped_file for g in pipeline.gwases] + gwas = lambda wildcards: getattr(pipeline, wildcards.prefix).standardised_gwas params: - clumped_snp_prefixes = list([g.clumped_snp_prefix for g in pipeline.gwases]) + ancestry = lambda wildcards: getattr(pipeline, wildcards.prefix).ancestry, + out_prefix = lambda wildcards: getattr(pipeline, wildcards.prefix).clumped_snp_prefix + output: + clumped = CLUMPED_FILE_PATTERN shell: """ - gwases=({input.gwases}) - ancestries=({ancestries}) - clumped_snp_prefixes=({params.clumped_snp_prefixes}) - - for i in "${{!gwases[@]}}" - do - ancestry=${{ancestries[$i]}} - plink1.9 --bfile {THOUSAND_GENOMES_DIR}/$ancestry \ - --clump ${{gwases[$i]}} \ - --clump-snp-field RSID \ - {pipeline.plink_clump_arguments} \ - --out ${{clumped_snp_prefixes[$i]}} || echo "{default_clump_headers}" > {output} - done + plink1.9 --bfile {THOUSAND_GENOMES_DIR}/{params.ancestry} \ + --threads {AVAILABLE_CPUS} \ + --clump {input.gwas} \ + --clump-snp-field RSID \ + {pipeline.plink_clump_arguments} \ + --out {params.out_prefix} || echo "{default_clump_headers}" > {output.clumped} """ diff --git a/snakemake/rules/finemap_multi_ancestry_rule.smk b/snakemake/rules/finemap_multi_ancestry_rule.smk new file mode 100644 index 0000000..3aef908 --- /dev/null +++ b/snakemake/rules/finemap_multi_ancestry_rule.smk @@ -0,0 +1,35 @@ +rule run_multi_ancestry_finemapping: + threads: 8 + resources: + mem = "64G", + time = "48:00:00" + input: + gwas_files = [g.standardised_gwas for g in pipeline.gwases], + clumped_files = [g.clumped_file for g in pipeline.gwases] + params: + gwas_files = " ".join([g.standardised_gwas for g in pipeline.gwases]), + clumped_files = " ".join([g.clumped_file for g in pipeline.gwases]), + ancestries = " ".join([g.ancestry for g in pipeline.gwases]), + sample_sizes = " ".join([str(g.N) for g in pipeline.gwases]), + output_dir = RESULTS_DIR + "finemap/multi_ancestry", + window_kb = finemap_window_kb, + max_causal = finemap_max_causal, + coverage = finemap_coverage, + min_abs_corr = finemap_min_abs_corr + output: + finemap_complete = MULTI_FINEMAP_COMPLETE + shell: + """ + python run_multisusie.py \ + --gwas_files {params.gwas_files} \ + --clumped_files {params.clumped_files} \ + --ancestries {params.ancestries} \ + --sample_sizes {params.sample_sizes} \ + --output_dir {params.output_dir} \ + --window_kb {params.window_kb} \ + --max_causal {params.max_causal} \ + --coverage {params.coverage} \ + --min_abs_corr {params.min_abs_corr} \ + --thousand_genomes_dir {THOUSAND_GENOMES_DIR} \ + --completion_file {output.finemap_complete} + """ diff --git a/snakemake/rules/finemap_rule.smk b/snakemake/rules/finemap_rule.smk new file mode 100644 index 0000000..a2c6287 --- /dev/null +++ b/snakemake/rules/finemap_rule.smk @@ -0,0 +1,32 @@ +rule run_finemapping: + threads: 8 + resources: + mem = "48G", + time = "24:00:00" + input: + gwas = lambda wildcards: getattr(pipeline, wildcards.prefix).standardised_gwas, + clumped_file = lambda wildcards: getattr(pipeline, wildcards.prefix).clumped_file + params: + ancestry = lambda wildcards: getattr(pipeline, wildcards.prefix).ancestry, + n = lambda wildcards: getattr(pipeline, wildcards.prefix).N, + finemap_dir = lambda wildcards: getattr(pipeline, wildcards.prefix).finemap_dir, + window_kb = finemap_window_kb, + max_causal = finemap_max_causal, + coverage = finemap_coverage, + min_abs_corr = finemap_min_abs_corr + output: + finemap_complete = FINEMAP_COMPLETE_TXT_PATTERN + shell: + """ + Rscript run_finemap.R \ + --gwas_filename {input.gwas} \ + --clumped_filename {input.clumped_file} \ + --ancestry {params.ancestry} \ + --N {params.n} \ + --output_finemap_dir {params.finemap_dir} \ + --completion_file {output.finemap_complete} \ + --window_kb {params.window_kb} \ + --max_causal {params.max_causal} \ + --coverage {params.coverage} \ + --min_abs_corr {params.min_abs_corr} + """ diff --git a/snakemake/rules/standardise_rule.smk b/snakemake/rules/standardise_rule.smk index e7141df..df7d956 100644 --- a/snakemake/rules/standardise_rule.smk +++ b/snakemake/rules/standardise_rule.smk @@ -7,7 +7,10 @@ rule standardise_gwases: input_build = lambda wildcards: getattr(pipeline, wildcards.prefix).build, input_columns = lambda wildcards: getattr(pipeline, wildcards.prefix).input_columns, output_columns = lambda wildcards: getattr(pipeline, wildcards.prefix).output_columns, - populate_rsid = lambda wildcards: getattr(pipeline, wildcards.prefix).populate_rsid.value + populate_rsid = lambda wildcards: getattr(pipeline, wildcards.prefix).populate_rsid.value, + ancestry = lambda wildcards: getattr(pipeline, wildcards.prefix).ancestry, + populate_eaf_flag = lambda wildcards: "TRUE" if getattr(pipeline, wildcards.prefix).populate_eaf else "FALSE", + flip_alleles_flag = lambda wildcards: "TRUE" if getattr(pipeline, wildcards.prefix).flip_alleles else "FALSE" threads: 8 resources: mem = lambda wildcards: f"{getattr(pipeline, wildcards.prefix).standardised_memory}G", @@ -30,6 +33,9 @@ rule standardise_gwases: --output_build {pipeline.output.build} \ --input_columns {params.input_columns} \ --output_columns {params.output_columns} \ - --populate_rsid {params.populate_rsid} + --populate_rsid {params.populate_rsid} \ + --populate_eaf {params.populate_eaf_flag} \ + --ancestry '{params.ancestry}' \ + --flip_alleles {params.flip_alleles_flag} """ diff --git a/snakemake/standardise_gwas.smk b/snakemake/standardise_gwas.smk index 3b3d426..b7843f0 100644 --- a/snakemake/standardise_gwas.smk +++ b/snakemake/standardise_gwas.smk @@ -2,7 +2,7 @@ include: "util/common.smk" singularity: get_docker_container() pipeline_name = "standardise_gwas" -pipeline = parse_pipeline_input() +pipeline = parse_pipeline_input(allow_flip_false=True) onstart: print("##### GWAS Standardisation Pipeline #####") @@ -15,7 +15,11 @@ rule all: include: "rules/standardise_rule.smk" onsuccess: - onsuccess(pipeline_name, is_test=pipeline.is_test) + onsuccess( + pipeline_name, + is_test=pipeline.is_test, + files_created=expand(std_file_pattern, prefix=[g.prefix for g in pipeline.gwases]) + ) onerror: onerror_message(pipeline_name, is_test=pipeline.is_test) diff --git a/snakemake/util/common.smk b/snakemake/util/common.smk index 3e59269..5121e9c 100644 --- a/snakemake/util/common.smk +++ b/snakemake/util/common.smk @@ -1,9 +1,10 @@ import csv import zipfile import gzip -import json +import yaml import os import re +import subprocess from datetime import datetime from dateutil.relativedelta import relativedelta from pathlib import Path @@ -12,25 +13,38 @@ from types import SimpleNamespace include: "constants.smk" include: "log_results.smk" -def get_docker_container(): - version = DOCKER_VERSION if DOCKER_VERSION else None - with open("DESCRIPTION") as file: - for line in file: - match = re.match(r"^Version: ([\w\.]+)", line) - if match: - version = match.group(1) - break - - sif_file = f"{PIPELINE_DATA_DIR}/genehackman_{version}.sif" - if os.path.isfile(sif_file): - return sif_file - elif version: - return f"{docker_repo}:{version}" - else: - return f"{docker_repo}:latest" + +def dict_to_namespace(obj): + """Recursively convert dict/list from YAML to SimpleNamespace (same shapes as former JSON parsing).""" + if isinstance(obj, dict): + return SimpleNamespace(**{k: dict_to_namespace(v) for k, v in obj.items()}) + if isinstance(obj, list): + return [dict_to_namespace(item) for item in obj] + return obj -def parse_pipeline_input(pipeline_includes_clumping=False): +def get_docker_container(): + version = DOCKER_VERSION + if not version: + raise ValueError("Error: DOCKER_VERSION must be set in .env") + pipeline_genomic_dir = os.path.join(PIPELINE_DATA_DIR.rstrip("/"), "genomic_data", "pipeline") + sif_path = os.path.join(pipeline_genomic_dir, f"genehackman_{version}.sif") + if not os.path.isfile(sif_path): + raise ValueError(f"Error: SIF file not found: {sif_path}") + return sif_path + + +def parse_pipeline_input(pipeline_includes_clumping=False, allow_flip_false=False): + global input_file + env_in = os.environ.get("GENEHACKMAN_INPUT") or os.environ.get("GENEHACKMAN_INPUT_FILE") + try: + cfg_in = config.get("genehackman_input") + except NameError: + cfg_in = None + if cfg_in is not None and str(cfg_in).startswith("-"): + cfg_in = None + input_file = env_in or cfg_in or "input.yaml" + if not os.path.isfile(".env"): raise ValueError("Error: .env file doesn't exist") if not os.path.isfile(input_file): @@ -38,10 +52,15 @@ def parse_pipeline_input(pipeline_includes_clumping=False): with open(input_file) as pipeline_input: try: - pipeline = json.load(pipeline_input,object_hook=lambda data: SimpleNamespace(**data)) + raw = yaml.safe_load(pipeline_input) + if raw is None: + raw = {} + pipeline = dict_to_namespace(raw) except Exception as e: - raise Exception('ERROR: There is an error with the JSON file, ' - + 'please ensure it is valid JSON: https://jsonlint.com/') from e + raise Exception( + "ERROR: There is an error with the pipeline YAML file, " + "please ensure it is valid YAML: https://www.yamllint.com/" + ) from e if not hasattr(pipeline, "is_test"): pipeline.is_test = False if not hasattr(pipeline, "output"): pipeline.output = default_output_options @@ -52,15 +71,46 @@ def parse_pipeline_input(pipeline_includes_clumping=False): if not hasattr(pipeline.output, "columns"): pipeline.output.columns = default_output_options.columns if not hasattr(pipeline, "populate_rsid"): pipeline.populate_rsid = False + if not hasattr(pipeline, "populate_eaf"): pipeline.populate_eaf = False + if not hasattr(pipeline, "flip_alleles"): pipeline.flip_alleles = True + + if not hasattr(pipeline, "finemap"): + pipeline.finemap = SimpleNamespace() + fm = pipeline.finemap + if not hasattr(fm, "window_kb"): fm.window_kb = 500 + if not hasattr(fm, "max_causal"): fm.max_causal = 10 + if not hasattr(fm, "coverage"): fm.coverage = 0.95 + if not hasattr(fm, "min_abs_corr"): fm.min_abs_corr = 0.5 + + if not hasattr(pipeline, "coloc"): + pipeline.coloc = SimpleNamespace() + cl = pipeline.coloc + if not hasattr(cl, "overlap_kb"): cl.overlap_kb = 1000 + if not hasattr(cl, "p1"): cl.p1 = 1e-4 + if not hasattr(cl, "p2"): cl.p2 = 1e-4 + if not hasattr(cl, "p12"): cl.p12 = 5e-6 for g in pipeline.gwases: if not hasattr(g, "N"): g.N = 0 if not hasattr(g, "columns"): g.columns = SimpleNamespace() if not hasattr(g, "remove_extra_columns"): g.remove_extra_columns = False + if not hasattr(g, "flip_alleles"): g.flip_alleles = getattr(pipeline, "flip_alleles", True) if not hasattr(g, "build"): g.build = "GRCh37" + if not hasattr(g, "study_type"): g.study_type = "continuous" if not hasattr(g, "populate_rsid"): g.populate_rsid = False + if not hasattr(g, "populate_eaf"): g.populate_eaf = False g.file = os.path.abspath(g.file) g.populate_rsid = resolve_rsid_population(pipeline_includes_clumping, g.populate_rsid or pipeline.populate_rsid) + g.populate_eaf = bool(g.populate_eaf or pipeline.populate_eaf) + if hasattr(g, "ancestry") and g.ancestry is not None and str(g.ancestry).strip(): + g.ancestry = str(g.ancestry).strip() + else: + g.ancestry = "" + if g.populate_eaf and not g.ancestry: + raise ValueError( + 'When populate_eaf is true, each GWAS must include "ancestry" in the input JSON ' + '(same as compare_gwases: AFR, AMR, EAS, EUR, or SAS), e.g. "ancestry": "EUR".' + ) g.standardised_memory = estimate_memory_needed_for_standardisation(g.file, g.populate_rsid) g.prefix = file_prefix(g.file) g.vcf_columns = get_columns_for_vcf_parsing(g.columns) @@ -68,6 +118,13 @@ def parse_pipeline_input(pipeline_includes_clumping=False): g.output_columns = resolve_gwas_columns(g.file, pipeline.output.columns, check_input_columns=False) g.standardised_gwas = standardised_gwas_name(g.file) setattr(pipeline,g.prefix,g) + eaf_ancestries = [g.ancestry for g in pipeline.gwases if g.populate_eaf] + if eaf_ancestries: + validate_ancestries(eaf_ancestries) + if not allow_flip_false: + for g in pipeline.gwases: + if getattr(g, "flip_alleles", True) is False: + raise ValueError("flip_alleles = FALSE is only available for standardise_gwas.smk") return pipeline def resolve_rsid_population(pipeline_includes_clumping, populate_rsid): @@ -198,14 +255,13 @@ def standardised_gwas_name(gwas_name): return DATA_DIR + "gwas/" + file_prefix(gwas_name) + "_std.tsv.gz" def cleanup_old_slurm_logs(): - if not os.path.isdir(slurm_log_directory): return + if not os.path.isdir(pipeline_log_directory): return one_month_ago = datetime.now() - relativedelta(months=1) - files = [f for f in os.listdir(slurm_log_directory) if os.path.isfile(f)] - print("deleting old logs") + files = [f for f in os.listdir(pipeline_log_directory) if os.path.isfile(os.path.join(pipeline_log_directory, f))] for filename in files: - file = os.path.join(slurm_log_directory, filename) + file = os.path.join(pipeline_log_directory, filename) file_timestamp = datetime.utcfromtimestamp(os.stat(file).st_mtime) if file_timestamp < one_month_ago: os.remove(file) @@ -223,20 +279,6 @@ def turn_dict_into_cli_string(results_dict): return ','.join(['%s=%s' % (key, value) for (key, value) in results_dict.items()]) -def copy_data_to_rdfs(files_created): - if RDFS_DIR is not None: - for file_created in files_created: - rdfs_file = None - if file_created.startswith(DATA_DIR): - rdfs_file = file_created.replace(DATA_DIR, RDFS_DIR + "/data/") - elif file_created.startswith(RESULTS_DIR): - rdfs_file = file_created.replace(RESULTS_DIR, RDFS_DIR + "/results/") - - if rdfs_file: - os.system(f"install -D {file_created} {rdfs_file}") - print(f"Files successfully copied to {RDFS_DIR}") - - def onsuccess(pipeline_name, files_created=list(), results_file=None, is_test=False): print("\nPipeline finished, no errors. List of created files:") print(*files_created, sep='\n') @@ -244,21 +286,33 @@ def onsuccess(pipeline_name, files_created=list(), results_file=None, is_test=Fa if results_file: print("\n---------------------") print("PLEASE VIEW THIS HTML FILE FOR A SUMMARY OF RESULTS:") - print(f"scp {user}@bc4login1.acrc.bris.ac.uk:{results_file} .") + print(f"{results_file}.") print(f"\n\n\033[1;35;40m Please do us a favour and cite this pipeline: https://doi.org/10.5281/zenodo.10624713 \033[0m\n") - copy_data_to_rdfs(files_created) - update_google_sheet(pipeline_name, succeeded=True, is_test=is_test) def onerror_message(pipeline_name, is_test=False): - last_log = subprocess.check_output(f"ls -t {slurm_log_directory} | head -n1", shell=True, universal_newlines=True) - log_full_path = slurm_log_directory + last_log print("\n---------------------") print("There are some documented common errors here: https://github.com/MRCIEU/GeneHackman/wiki") - print("There was an error in the pipeline, please check the last written slurm log to see the error:") - print(log_full_path) + log_full_path = None + if os.path.isdir(pipeline_log_directory): + try: + entries = [ + os.path.join(pipeline_log_directory, f) + for f in os.listdir(pipeline_log_directory) + if os.path.isfile(os.path.join(pipeline_log_directory, f)) + ] + if entries: + log_full_path = max(entries, key=lambda p: os.stat(p).st_mtime) + except OSError: + pass + if log_full_path: + print("There was an error in the pipeline. If the job ran under Slurm, check the batch log:") + print(log_full_path) + else: + print("There was an error in the pipeline. Check the Snakemake output above,") + print(f"and logs under: {os.path.join(os.getcwd(), '.snakemake', 'log')}") update_google_sheet(pipeline_name, succeeded=False, error_file=log_full_path, is_test=is_test) diff --git a/snakemake/util/constants.smk b/snakemake/util/constants.smk index b499d1f..364a655 100644 --- a/snakemake/util/constants.smk +++ b/snakemake/util/constants.smk @@ -9,12 +9,41 @@ def format_dir_string(directory): if not directory: return None return directory + "/" if not directory.endswith('/') else directory + +def _available_cpus(): + """CPUs on the allocated Slurm node if Slurm sets them, else logical CPUs on this host.""" + slurm = os.getenv("SLURM_CPUS_ON_NODE") + if slurm is not None and str(slurm).strip() != "": + return max(1, int(slurm)) + n = os.cpu_count() + return max(1, n if n is not None else 1) + + +def _strip_project_path(s): + return (s or "").strip().rstrip("/\\") + + load_dotenv() docker_repo = "docker://mrcieu/genehackman" user = os.getenv('USER') -input_file = os.getenv('INPUT_FILE') or "input.json" +input_file = "input.yaml" start_time = datetime.now() -slurm_log_directory = f"/user/work/{user}/slurm_logs/" + +_project = _strip_project_path(os.getenv("PROJECT_DIR", "")) + +DATA_DIR = format_dir_string(os.path.join(_project, "data")) +RESULTS_DIR = format_dir_string(os.path.join(_project, "results")) + +_pipeline_log_override = os.getenv("PIPELINE_LOG_DIR") +_local_flag = os.getenv("GENEHACKMAN_LOCAL", "").strip().lower() in ("1", "true", "yes") +if _pipeline_log_override: + pipeline_log_directory = format_dir_string(_pipeline_log_override) +elif _local_flag or not (DATA_DIR and os.path.isdir(DATA_DIR)): + pipeline_log_directory = format_dir_string(os.path.join(os.getcwd(), "snakemake_logs")) +else: + pipeline_log_directory = format_dir_string(f"{DATA_DIR}/snakemake_logs") + +slurm_log_directory = pipeline_log_directory default_clump_headers = "CHR F SNP BP P TOTAL NSIG S05 S01 S001 S0001 SP2" #TODO: this should be read from predefined_column_map.csv @@ -43,21 +72,17 @@ default_output_options = SimpleNamespace(**{ "columns": SimpleNamespace(**default_columns) }) -if not os.getenv("DATA_DIR") or not os.getenv("RESULTS_DIR"): - raise ValueError("Please populate DATA_DIR and RESULTS_DIR in the .env file provided") -if not os.getenv("RDFS_DIR"): - print("Please populate RDFS_DIR in .env if you want the generated files to be automatically copied to RDFS") - DOCKER_VERSION = os.getenv('DOCKER_VERSION') -DATA_DIR = format_dir_string(os.getenv('DATA_DIR')) -RESULTS_DIR = format_dir_string(os.getenv('RESULTS_DIR')) -RDFS_DIR = format_dir_string(os.getenv('RDFS_DIR')) -LDSC_DIR = format_dir_string(os.getenv('LDSC_DIR')) -GENOMIC_DATA_DIR = format_dir_string(os.getenv('GENOMIC_DATA_DIR')) -THOUSAND_GENOMES_DIR = format_dir_string(os.getenv('THOUSAND_GENOMES_DIR')) -PIPELINE_DATA_DIR = GENOMIC_DATA_DIR + "/pipeline" - -if RDFS_DIR and not RDFS_DIR.endswith("working/"): - raise ValueError("Please ensure RDFS_DIR ends with working/ to ensure the data gets copied to the correct place") -if DATA_DIR == RESULTS_DIR: - raise ValueError("DATA_DIR and RESULTS_DIR must be different directories") + +# Single Snakemake output path for successful finemap runs (written last by finemap_gwas). +FINEMAP_COMPLETE_TXT_PATTERN = RESULTS_DIR + "finemap/{prefix}/finemap_complete.txt" + +PIPELINE_DATA_DIR = format_dir_string(os.getenv('PIPELINE_DATA_DIR')) +QTL_DATA_DIR = os.getenv("QTL_DATA_DIR", "").strip() + +LDSC_DIR = format_dir_string(PIPELINE_DATA_DIR + "/LDSCORE/b37_dbsnp156") +GENOMIC_DATA_DIR = format_dir_string(PIPELINE_DATA_DIR + "/genomic_data") +THOUSAND_GENOMES_DIR = format_dir_string(GENOMIC_DATA_DIR + "1000genomes/b37_dbsnp156") +QTL_DIRECTORY = format_dir_string(QTL_DATA_DIR) + +AVAILABLE_CPUS = _available_cpus() diff --git a/snakemake/util/log_results.smk b/snakemake/util/log_results.smk index 088ee55..8e42d3c 100644 --- a/snakemake/util/log_results.smk +++ b/snakemake/util/log_results.smk @@ -7,8 +7,11 @@ import os def update_google_sheet(pipeline_name, succeeded=True, error_file=None, is_test=False): if is_test: return + creds_file = GENOMIC_DATA_DIR + "/googlesheets/google_sheets_creds.json" try: - gc = pygsheets.authorize(service_file=GENOMIC_DATA_DIR + "/googlesheets/google_sheets_creds.json") + if not os.path.isfile(creds_file): + return + gc = pygsheets.authorize(service_file=creds_file) spreadsheet = gc.open('GeneHackman') worksheet = spreadsheet[0] @@ -23,8 +26,11 @@ def update_google_sheet(pipeline_name, succeeded=True, error_file=None, is_test= if not succeeded: with open(input_file,'r') as file: input_data = file.read() - with open(error_file.rstrip(), 'r') as file: - error_message = file.read() + if error_file and os.path.isfile(error_file): + with open(error_file, 'r') as file: + error_message = file.read() + else: + error_message = "(no batch log file; see Snakemake console or .snakemake/log/)" values = [user, hostname, pipeline_name, time_submitted_str, time_taken, result, input_data, error_message] worksheet.append_table(values=[values]) diff --git a/tests/e2e_tests/run_test_pipelines.sh b/tests/e2e_tests/run_test_pipelines.sh index 1903282..84a139f 100755 --- a/tests/e2e_tests/run_test_pipelines.sh +++ b/tests/e2e_tests/run_test_pipelines.sh @@ -1,7 +1,17 @@ #!/bin/bash set -e -./run_pipeline.sh snakemake/standardise_gwas.smk tests/testthat/data/snakemake_inputs/standardise_gwas.json -F -./run_pipeline.sh snakemake/disease_progression.smk tests/testthat/data/snakemake_inputs/disease_progression.json -F -./run_pipeline.sh snakemake/compare_gwases.smk tests/testthat/data/snakemake_inputs/compare_gwases.json -F -./run_pipeline.sh snakemake/qtl_mr.smk tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.json -F +./run_pipeline.sh snakemake/standardise_gwas.smk tests/testthat/data/snakemake_inputs/standardise_gwas.yaml -F +./run_pipeline.sh snakemake/disease_progression.smk tests/testthat/data/snakemake_inputs/disease_progression.yaml -F +./run_pipeline.sh snakemake/compare_gwases.smk tests/testthat/data/snakemake_inputs/compare_gwases.yaml -F +./run_pipeline.sh snakemake/finemap.smk tests/testthat/data/snakemake_inputs/finemap.yaml -F +./run_pipeline.sh snakemake/finemap.smk tests/testthat/data/snakemake_inputs/finemap_multi_ancestry.yaml -F +./run_pipeline.sh snakemake/coloc.smk tests/testthat/data/snakemake_inputs/coloc.yaml -F + +if [[ -n "${QTL_DATA_DIR}" ]]; then + ./run_pipeline.sh snakemake/qtl_mr.smk tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.yaml -F +fi + +branch_name=$(git rev-parse --abbrev-ref HEAD) +status_message="SUCCESS: All tests passed on branch: $branch_name" +echo "$status_message" > tests/testing_complete.txt \ No newline at end of file diff --git a/tests/testing_complete.txt b/tests/testing_complete.txt new file mode 100644 index 0000000..a32788d --- /dev/null +++ b/tests/testing_complete.txt @@ -0,0 +1 @@ +SUCCESS: All tests passed on branch: feature/run-locally diff --git a/tests/testthat/data/snakemake_inputs/coloc.yaml b/tests/testthat/data/snakemake_inputs/coloc.yaml new file mode 100644 index 0000000..0f01f28 --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/coloc.yaml @@ -0,0 +1,11 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR + - file: tests/testthat/data/test_data_small_hg38.tsv.gz + columns: {} + build: GRCh38 + ancestry: EUR +plink_clump_arguments: "--clump-p1 0.00000005" +populate_rsid: false +is_test: true diff --git a/tests/testthat/data/snakemake_inputs/compare_gwases.json b/tests/testthat/data/snakemake_inputs/compare_gwases.json deleted file mode 100644 index 69bda64..0000000 --- a/tests/testthat/data/snakemake_inputs/compare_gwases.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "gwases": [ - { - "file": "tests/testthat/data/test_data_small.tsv.gz", - "columns": {}, - "ancestry": "EUR" - }, - { - "file": "tests/testthat/data/test_data_small_hg38.tsv.gz", - "columns": {}, - "build": "GRCh38", - "ancestry": "EUR" - } - ], - "plink_clump_arguments": "--clump-p1 0.00000005", - "populate_rsid": false, - "is_test": true -} diff --git a/tests/testthat/data/snakemake_inputs/compare_gwases.yaml b/tests/testthat/data/snakemake_inputs/compare_gwases.yaml new file mode 100644 index 0000000..0f01f28 --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/compare_gwases.yaml @@ -0,0 +1,11 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR + - file: tests/testthat/data/test_data_small_hg38.tsv.gz + columns: {} + build: GRCh38 + ancestry: EUR +plink_clump_arguments: "--clump-p1 0.00000005" +populate_rsid: false +is_test: true diff --git a/tests/testthat/data/snakemake_inputs/disease_progression.json b/tests/testthat/data/snakemake_inputs/disease_progression.json deleted file mode 100644 index 81a62e8..0000000 --- a/tests/testthat/data/snakemake_inputs/disease_progression.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "gwases": [ - { - "file": "tests/testthat/data/test_data_small.tsv.gz", - "columns": {}, - "ancestry": "EUR" - }, - { - "file": "tests/testthat/data/test_data_small_hg38.tsv.gz", - "columns": {}, - "build": "GRCh38", - "ancestry": "EUR" - } - ], - "plink_clump_arguments": "--clump-p1 0.00000005", - "populate_rsid": false, - "output": { - "adjusted_gwas": - { - "type" : "slopehunter", - "p_val": 0.001 - } - }, - "is_test": true -} diff --git a/tests/testthat/data/snakemake_inputs/disease_progression.yaml b/tests/testthat/data/snakemake_inputs/disease_progression.yaml new file mode 100644 index 0000000..3a0cabc --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/disease_progression.yaml @@ -0,0 +1,15 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR + - file: tests/testthat/data/test_data_small_hg38.tsv.gz + columns: {} + build: GRCh38 + ancestry: EUR +plink_clump_arguments: "--clump-p1 0.00000005" +populate_rsid: false +output: + adjusted_gwas: + type: slopehunter + p_val: 0.001 +is_test: true diff --git a/tests/testthat/data/snakemake_inputs/finemap.yaml b/tests/testthat/data/snakemake_inputs/finemap.yaml new file mode 100644 index 0000000..0f01f28 --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/finemap.yaml @@ -0,0 +1,11 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR + - file: tests/testthat/data/test_data_small_hg38.tsv.gz + columns: {} + build: GRCh38 + ancestry: EUR +plink_clump_arguments: "--clump-p1 0.00000005" +populate_rsid: false +is_test: true diff --git a/tests/testthat/data/snakemake_inputs/finemap_multi_ancestry.yaml b/tests/testthat/data/snakemake_inputs/finemap_multi_ancestry.yaml new file mode 100644 index 0000000..300ebbb --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/finemap_multi_ancestry.yaml @@ -0,0 +1,10 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR + - file: tests/testthat/data/test_data_small_eas.tsv.gz + columns: {} + ancestry: EAS +plink_clump_arguments: "--clump-p1 0.00000005" +populate_rsid: false +is_test: true \ No newline at end of file diff --git a/tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.json b/tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.json deleted file mode 100644 index 8868001..0000000 --- a/tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "gwases": [ - { - "file": "tests/testthat/data/test_data_small.tsv.gz", - "columns": {}, - "ancestry": "EUR" - } - ], - "qtl": { - "dataset": "eqtlgen", - "subcategory": "cis" - }, - "is_test": true -} diff --git a/tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.yaml b/tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.yaml new file mode 100644 index 0000000..6567ce8 --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/qtl_mr_eqtlgen.yaml @@ -0,0 +1,10 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR +plink_clump_arguments: "--clump-p1 0.00000005" +qtl: + dataset: eqtlgen + subcategory: cis +populate_rsid: false +is_test: true diff --git a/tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.json b/tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.json deleted file mode 100644 index 173674e..0000000 --- a/tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "gwases": [ - { - "file": "tests/testthat/data/test_data_small.tsv.gz", - "columns": {}, - "ancestry": "EUR" - } - ], - "qtl": { - "dataset": "metabrain", - "subcategory": "cortex", - "exposures": [] - }, - "is_test": true -} diff --git a/tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.yaml b/tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.yaml new file mode 100644 index 0000000..b927779 --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/qtl_mr_metabrain.yaml @@ -0,0 +1,11 @@ +gwases: + - file: tests/testthat/data/test_data_small.tsv.gz + columns: {} + ancestry: EUR +plink_clump_arguments: "--clump-p1 0.00000005" +qtl: + dataset: metabrain + subcategory: cortex + exposures: [] +populate_rsid: false +is_test: true diff --git a/tests/testthat/data/snakemake_inputs/standardise_gwas.json b/tests/testthat/data/snakemake_inputs/standardise_gwas.json deleted file mode 100644 index d7fae61..0000000 --- a/tests/testthat/data/snakemake_inputs/standardise_gwas.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "gwases": [ - { - "file": "tests/testthat/data/some_eqtlgen_hits.tsv.gz", - "columns": {} - }, - { - "file": "tests/testthat/data/opengwas_example.vcf.gz", - "columns": "opengwas" - }, - { - "file": "tests/testthat/data/metal_output.txt", - "columns": "metal", - "populate_rsid": true - }, - { - "file": "tests/testthat/data/test_data_small_hg38.tsv.gz", - "columns": {}, - "build": "GRCh38" - } - ], - "populate_rsid": false, - "is_test": true -} diff --git a/tests/testthat/data/snakemake_inputs/standardise_gwas.yaml b/tests/testthat/data/snakemake_inputs/standardise_gwas.yaml new file mode 100644 index 0000000..ce302fa --- /dev/null +++ b/tests/testthat/data/snakemake_inputs/standardise_gwas.yaml @@ -0,0 +1,18 @@ +gwases: + - file: tests/testthat/data/some_eqtlgen_hits.tsv.gz + columns: {} + - file: tests/testthat/data/opengwas_example.vcf.gz + columns: opengwas + - file: tests/testthat/data/metal_output.txt + columns: metal + - file: tests/testthat/data/test_data_small_hg38.tsv.gz + columns: {} + build: GRCh38 + - file: tests/testthat/data/test_data_small_hg38_no_eaf.tsv.gz + ancestry: EUR + populate_eaf: true + columns: {} + build: GRCh38 + +populate_rsid: false +is_test: true diff --git a/tests/testthat/data/test_data_small_eas.tsv.gz b/tests/testthat/data/test_data_small_eas.tsv.gz new file mode 100644 index 0000000..9335c69 Binary files /dev/null and b/tests/testthat/data/test_data_small_eas.tsv.gz differ diff --git a/tests/testthat/data/test_data_small_hg38_no_eaf.tsv.gz b/tests/testthat/data/test_data_small_hg38_no_eaf.tsv.gz new file mode 100644 index 0000000..cb75366 Binary files /dev/null and b/tests/testthat/data/test_data_small_hg38_no_eaf.tsv.gz differ diff --git a/tests/testthat/test_coloc_bf.R b/tests/testthat/test_coloc_bf.R new file mode 100644 index 0000000..979d21d --- /dev/null +++ b/tests/testthat/test_coloc_bf.R @@ -0,0 +1,212 @@ +# Tests for R/coloc_bf.R (BF-BF colocalization on finemapped loci) + +min_locus_tbl <- function(snps, lbf1, lbf2 = NULL, cs = 1L) { + n <- length(snps) + if (is.null(lbf2)) { + lbf2 <- rep(0.1, n) + } + tibble::tibble( + SNP = snps, + CS = rep(cs, length.out = n), + LBF_1 = lbf1, + LBF_2 = lbf2 + ) +} + +test_that("coloc_bf.build_lbf_matrix is rows=signals, cols=SNPs", { + d <- min_locus_tbl( + snps = c("1:10_A_G", "1:20_T_C"), + lbf1 = c(0.5, 0.25), + lbf2 = c(0.1, 0.2) + ) + m <- GeneHackman:::build_lbf_matrix(d, c("LBF_1", "LBF_2")) + expect_equal(dim(m), c(2L, 2L)) + expect_equal(colnames(m), d$SNP) + expect_equal(rownames(m), c("LBF_1", "LBF_2")) + expect_equal(as.numeric(m[1, ]), c(0.5, 0.25)) +}) + +test_that("coloc_bf.build_lbf_matrix returns NULL without LBF columns", { + d <- tibble::tibble(SNP = "a", CS = 1L) + expect_null(GeneHackman:::build_lbf_matrix(d, c("LBF_1"))) +}) + +test_that("coloc_bf.build_lbf_matrix returns NULL when all CS are NA", { + d <- tibble::tibble( + SNP = c("s1", "s2"), + CS = c(NA_integer_, NA_integer_), + LBF_1 = c(1, 1) + ) + expect_null(GeneHackman:::build_lbf_matrix(d, "LBF_1")) +}) + +test_that("coloc_bf.parse_pairwise_bf_bf_result reads coloc summary rows", { + sm <- tibble::tibble( + nsnps = 3L, + hit1 = "LBF_1", + hit2 = "LBF_1", + PP.H0.abf = 0.1, + PP.H1.abf = 0.2, + PP.H2.abf = 0.15, + PP.H3.abf = 0.15, + PP.H4.abf = 0.4 + ) + out <- GeneHackman:::parse_pairwise_bf_bf_result( + list(summary = sm), + trait1 = "A", + trait2 = "B", + locus1 = "1_1", + locus2 = "1_1", + n_snps = 3L + ) + expect_equal(nrow(out), 1L) + expect_equal(out$trait1, "A") + expect_equal(out$PP.H4.abf, 0.4) +}) + +test_that("coloc_bf.parse_pairwise_bf_bf_result fallback when no summary", { + out <- GeneHackman:::parse_pairwise_bf_bf_result( + list(), + "A", "B", "1_1", "1_1", + n_snps = 5L + ) + expect_equal(nrow(out), 1L) + expect_equal(out$n_snps, 5L) + expect_true(is.na(out$PP.H4.abf)) +}) + +test_that("coloc_bf.load_all_finemap_loci loads LBF and chr/bp from filename", { + d1 <- min_locus_tbl(c("1:1_A_G", "1:2_T_C"), c(0.1, 0.2), c(0, 0)) + dir1 <- tempfile("fm_") + dir.create(dir1) + vroom::vroom_write( + d1, + file.path(dir1, "1_100000_finemap.tsv.gz") + ) + fd <- c(trait1 = dir1) + loci <- GeneHackman:::load_all_finemap_loci(fd) + expect_length(loci, 1L) + expect_length(loci$trait1, 1L) + expect_equal(loci$trait1[[1]]$chr, 1) + expect_equal(loci$trait1[[1]]$lead_bp, 100000) + expect_true("LBF_1" %in% loci$trait1[[1]]$lbf_cols) +}) + +test_that("coloc_bf.load_all_finemap_loci skips files without LBF columns", { + d <- tibble::tibble(SNP = "x", BETA = 1) + dir1 <- tempfile("fm_") + dir.create(dir1) + vroom::vroom_write(d, file.path(dir1, "1_100000_finemap.tsv.gz")) + expect_equal(length(GeneHackman:::load_all_finemap_loci(c(t = dir1))$t), 0L) +}) + +test_that("coloc_bf.coloc_bf_bf_for_locus_pair returns NULL for <2 shared SNPs", { + l1 <- list( + data = tibble::tibble(SNP = "a", LBF_1 = 1, CS = 1L), + lbf_cols = "LBF_1", + chr = 1L, + lead_bp = 100L, + locus_name = "1_100" + ) + l2 <- list( + data = tibble::tibble(SNP = "b", LBF_1 = 1, CS = 1L), + lbf_cols = "LBF_1", + chr = 1L, + lead_bp = 100L, + locus_name = "1_100" + ) + expect_null( + GeneHackman:::coloc_bf_bf_for_locus_pair(l1, l2, "A", "B", 1e-4, 1e-4, 5e-6) + ) +}) + +test_that("coloc_bf.coloc_bf_bf_for_locus_pair runs coloc on two aligned loci", { + snps <- c("1:1_A_G", "1:2_T_C", "1:3_G_T") + d <- min_locus_tbl( + snps = snps, + lbf1 = c(0.5, 0.3, 0.2), + lbf2 = c(0.1, 0.4, 0.1) + ) + l1 <- list( + data = d, + lbf_cols = c("LBF_1", "LBF_2"), + chr = 1L, + lead_bp = 100000L, + locus_name = "1_100000" + ) + l2 <- list( + data = d, + lbf_cols = c("LBF_1", "LBF_2"), + chr = 1L, + lead_bp = 100500L, + locus_name = "1_100500" + ) + res <- GeneHackman:::coloc_bf_bf_for_locus_pair( + l1, l2, "trait1", "trait2", 1e-4, 1e-4, 5e-6 + ) + expect_true(!is.null(res)) + expect_true(all(c("trait1", "trait2", "locus1", "PP.H4.abf") %in% names(res))) + expect_true(all(is.finite(res$PP.H0.abf + res$PP.H4.abf))) +}) + +write_pair_finemap_dirs <- function() { + snps <- c("1:1_A_G", "1:2_T_C", "1:3_G_T") + d <- min_locus_tbl( + snps = snps, + lbf1 = c(0.4, 0.35, 0.25), + lbf2 = c(0.2, 0.2, 0.2) + ) + dir1 <- tempfile("coc_a_") + dir2 <- tempfile("coc_b_") + dir.create(dir1) + dir.create(dir2) + vroom::vroom_write(d, file.path(dir1, "1_100000_finemap.tsv.gz")) + vroom::vroom_write(d, file.path(dir2, "1_100200_finemap.tsv.gz")) + c(a = dir1, b = dir2) +} + +test_that("coloc_bf.run_bf_bf_coloc needs two traits with loci", { + d <- tempfile("one_") + dir.create(d) + d2 <- min_locus_tbl(c("1:1_A_G", "1:2_T_C"), c(0.1, 0.2), c(0, 0)) + vroom::vroom_write(d2, file.path(d, "1_1_finemap.tsv.gz")) + out <- run_bf_bf_coloc(c(only = d), overlap_kb = 1000) + expect_equal(nrow(out), 0L) +}) + +test_that("coloc_bf.run_bf_bf_coloc pairs overlapping loci and writes file", { + dirs <- write_pair_finemap_dirs() + out_path <- tempfile(fileext = ".tsv") + res <- run_bf_bf_coloc( + finemap_dirs = dirs, + overlap_kb = 1000, + output_file = out_path + ) + expect_true(nrow(res) >= 1L) + expect_true(file.exists(out_path)) + disk <- vroom::vroom(out_path, show_col_types = FALSE) + expect_equal(nrow(disk), nrow(res)) + expect_equal(disk$trait1[1], "a") + expect_equal(disk$trait2[1], "b") +}) + +test_that("coloc_bf.run_bf_bf_coloc uses basename when finemap_dirs unnamed", { + dirs <- write_pair_finemap_dirs() + names(dirs) <- NULL + res <- run_bf_bf_coloc(dirs, overlap_kb = 1000) + expect_true(all(c(basename(dirs[1]), basename(dirs[2])) %in% c(res$trait1, res$trait2))) +}) + +test_that("coloc_bf.run_bf_bf_coloc finds no pair when loci on different chromosomes", { + snps <- c("1:1_A_G", "1:2_T_C", "1:3_G_T") + d1 <- min_locus_tbl(snps, c(0.4, 0.35, 0.25), c(0.2, 0.2, 0.2)) + d2 <- min_locus_tbl(snps, c(0.4, 0.35, 0.25), c(0.2, 0.2, 0.2)) + dir1 <- tempfile("coc_chr1_") + dir2 <- tempfile("coc_chr2_") + dir.create(dir1) + dir.create(dir2) + vroom::vroom_write(d1, file.path(dir1, "1_100000_finemap.tsv.gz")) + vroom::vroom_write(d2, file.path(dir2, "2_100000_finemap.tsv.gz")) + res <- run_bf_bf_coloc(c(x = dir1, y = dir2), overlap_kb = 1000) + expect_equal(nrow(res), 0L) +}) diff --git a/tests/testthat/test_finemap.R b/tests/testthat/test_finemap.R new file mode 100644 index 0000000..148c63a --- /dev/null +++ b/tests/testthat/test_finemap.R @@ -0,0 +1,200 @@ +test_that("finemap.finemap_gwas stops when sample size cannot be resolved", { + # Mock LD so we reach the N check; otherwise plink failure skips the locus + # before default_n is validated. + local_mocked_bindings( + compute_ld_matrix = function(rsids, chr, ancestry) { + n <- length(rsids) + list(matrix = diag(n), snps = rsids) + } + ) + + gwas <- tibble::tibble( + SNP = c("1:100000_A_G", "1:100150_T_C"), + CHR = c(1, 1), + BP = c(100000, 100150), + RSID = c("rs1", "rs2"), + BETA = c(0.1, 0.05), + SE = c(0.02, 0.02), + P = c(0.01, 0.05), + EA = c("A", "T"), + OA = c("G", "C"), + EAF = c(0.3, 0.4) + ) + gwas_file <- tempfile(fileext = ".tsv.gz") + vroom::vroom_write(gwas, gwas_file) + clump_file <- tempfile(fileext = ".clumped") + writeLines(c("SNP\tCHR\tBP", "rs1\t1\t100000"), clump_file) + out_dir <- tempfile("finemap_") + expect_error( + finemap_gwas( + gwas_file, + clump_file, + ancestry = "EUR", + default_n = NA, + output_finemap_dir = out_dir + ), + regexp = "Fine-mapping requires GWAS sample size" + ) +}) + +test_that("finemap.finemap_gwas writes no locus files when clump has no rows", { + gwas <- tibble::tibble( + SNP = c("1:100000_A_G", "1:100150_T_C"), + CHR = c(1, 1), + BP = c(100000, 100150), + RSID = c("rs1", "rs2"), + BETA = c(0.1, 0.05), + SE = c(0.02, 0.02), + P = c(0.01, 0.05), + EA = c("A", "T"), + OA = c("G", "C"), + EAF = c(0.3, 0.4) + ) + gwas_file <- tempfile(fileext = ".tsv.gz") + vroom::vroom_write(gwas, gwas_file) + + clump_file <- tempfile(fileext = ".clumped") + writeLines("SNP\tCHR\tBP", clump_file) + + out_dir <- tempfile("finemap_") + dir.create(out_dir) + + res <- finemap_gwas( + gwas_file, + clump_file, + ancestry = "EUR", + default_n = 10000L, + output_finemap_dir = out_dir + ) + + expect_equal(nrow(res), 0) + expect_equal(length(list.files(out_dir, pattern = "_finemap\\.tsv\\.gz$")), 0) + expect_true(file.exists(file.path(out_dir, "finemap_complete.txt"))) + expect_identical(readLines(file.path(out_dir, "finemap_complete.txt")), "0") +}) + +test_that("finemap.compute_ld_matrix returns NULL when plink fails", { + local_mocked_bindings( + run_system = function(command, wait = TRUE, intern = FALSE, + ignore.stdout = FALSE, ignore.stderr = FALSE) { + 1L + } + ) + expect_null(compute_ld_matrix(c("rs1", "rs2"), 1L, "EUR")) +}) + +test_that("finemap.run_susie_for_locus maps per-CS LBF_k and credible sets", { + local_mocked_bindings( + run_susie_rss_impl = function(z, R, n, L, coverage, min_abs_corr, verbose = FALSE) { + p <- length(z) + list( + lbf_variable = matrix(c(0.5, 0.25, 0.5, 0.75), nrow = 2, ncol = p), + sets = list(cs = list(c(1L, 2L))) + ) + } + ) + + snp_info <- tibble::tibble( + SNP = c("1:100000_A_G", "1:100150_T_C"), + CHR = c(1, 1), + BP = c(100000, 100150), + RSID = c("rs1", "rs2") + ) + z <- c(1, 2) + R <- diag(2) + + out <- run_susie_for_locus( + z_scores = z, + ld_matrix = R, + snp_info = snp_info, + n = 1000L, + lead_snp = "rs1" + ) + + expect_equal(nrow(out), 2) + expect_equal(out$LBF_1, c(0.5, 0.5)) + expect_equal(out$CS, c(1L, 1L)) +}) + +test_that("finemap.finemap_gwas runs end-to-end with mocked LD and SuSiE", { + local_mocked_bindings( + compute_ld_matrix = function(rsids, chr, ancestry) { + n <- length(rsids) + list(matrix = diag(n), snps = rsids) + }, + run_susie_rss_impl = function(z, R, n, L, coverage, min_abs_corr, verbose = FALSE) { + p <- length(z) + list( + lbf_variable = matrix(1, nrow = 2, ncol = p), + sets = list(cs = list(seq_len(p))) + ) + } + ) + + gwas <- tibble::tibble( + SNP = c("1:100000_A_G", "1:100150_T_C"), + CHR = c(1, 1), + BP = c(100000, 100150), + RSID = c("rs1", "rs2"), + BETA = c(0.1, 0.05), + SE = c(0.02, 0.02), + P = c(0.01, 0.05), + EA = c("A", "T"), + OA = c("G", "C"), + EAF = c(0.3, 0.4) + ) + gwas_file <- tempfile(fileext = ".tsv.gz") + vroom::vroom_write(gwas, gwas_file) + + clump_file <- tempfile(fileext = ".clumped") + writeLines(c("SNP\tCHR\tBP", "rs1\t1\t100000"), clump_file) + + out_dir <- tempfile("finemap_") + dir.create(out_dir) + + res <- finemap_gwas( + gwas_file, + clump_file, + ancestry = "EUR", + default_n = 10000L, + output_finemap_dir = out_dir, + window_kb = 500 + ) + + expect_equal(nrow(res), 2) + expect_setequal(res$RSID, c("rs1", "rs2")) + expect_true(all(res$CS == 1L)) + + locus_file <- file.path(out_dir, "1_100000_finemap.tsv.gz") + expect_true(file.exists(locus_file)) + locus_read <- vroom::vroom(locus_file, show_col_types = FALSE) + expect_equal(nrow(locus_read), 2) + expect_true("LBF_1" %in% names(locus_read)) + expect_setequal(locus_read$SNP, gwas$SNP) + expect_false(any(grepl("^LEAD_", names(locus_read)))) + expect_true(file.exists(file.path(out_dir, "finemap_complete.txt"))) + expect_identical(readLines(file.path(out_dir, "finemap_complete.txt")), "1") +}) + +test_that("finemap.run_susie_for_locus returns empty tibble when SuSiE errors", { + local_mocked_bindings( + run_susie_rss_impl = function(z, R, n, L, coverage, min_abs_corr, verbose = FALSE) { + stop("mock failure") + } + ) + + snp_info <- tibble::tibble( + SNP = "1:100000_A_G", + CHR = 1, + BP = 100000, + RSID = "rs1" + ) + out <- run_susie_for_locus( + z_scores = 1, + ld_matrix = matrix(1, 1, 1), + snp_info = snp_info, + n = 1000L, + lead_snp = "rs1" + ) + expect_equal(nrow(out), 0) +}) diff --git a/tests/testthat/test_gwas_formatting.R b/tests/testthat/test_gwas_formatting.R index 5d97245..481be1b 100644 --- a/tests/testthat/test_gwas_formatting.R +++ b/tests/testthat/test_gwas_formatting.R @@ -115,6 +115,148 @@ test_that("gwas_formatting.standardise_gwas standardises an IEU ukb pipeline out expect_true(all(grep("\\d+:\\d+_\\w+_\\w+", result$SNP))) }) +test_that("standardise_gwas flip_alleles=FALSE preserves EA/OA order and effect direction", { + tmp <- tempfile(fileext = ".tsv.gz") + # Use G/C (not T/A): vroom reads bare "T" in a file as logical TRUE. + g <- tibble::tibble( + CHR = 1L, + BP = 100000L, + EA = "G", + OA = "C", + P = 0.05, + BETA = 0.15, + SE = 0.03, + EAF = 0.42 + ) + vroom::vroom_write(g, tmp) + + out_no_flip <- standardise_gwas( + tmp, + tempfile(fileext = ".tsv.gz"), + input_reference_build = reference_builds$GRCh37, + output_reference_build = reference_builds$GRCh37, + flip_alleles = FALSE + ) + expect_equal(out_no_flip$EA[1], "G") + expect_equal(out_no_flip$OA[1], "C") + expect_equal(out_no_flip$BETA[1], 0.15) + expect_equal(out_no_flip$EAF[1], 0.42) + expect_match(out_no_flip$SNP[1], "^1:100000_G_C$") + + out_flip <- standardise_gwas( + tmp, + tempfile(fileext = ".tsv.gz"), + input_reference_build = reference_builds$GRCh37, + output_reference_build = reference_builds$GRCh37, + flip_alleles = TRUE + ) + expect_equal(out_flip$EA[1], "C") + expect_equal(out_flip$OA[1], "G") + expect_equal(out_flip$BETA[1], -0.15) + expect_equal(out_flip$EAF[1], 1 - 0.42) + expect_match(out_flip$SNP[1], "^1:100000_C_G$") +}) + +test_that("standardise_gwas errors when populate_eaf is TRUE and ancestry is missing", { + expect_error( + standardise_gwas( + "data/test_data_tiny.tsv.gz", + tempfile(fileext = ".tsv.gz"), + populate_eaf = TRUE, + ancestry = NULL + ), + "populate_eaf is TRUE but ancestry is missing" + ) + expect_error( + standardise_gwas( + "data/test_data_tiny.tsv.gz", + tempfile(fileext = ".tsv.gz"), + populate_eaf = TRUE, + ancestry = "" + ), + "populate_eaf is TRUE but ancestry is missing" + ) +}) + +test_that("standardise_gwas flip_alleles=FALSE with partial RSID succeeds (internal flip handles it)", { + out <- standardise_gwas( + "data/test_data_tiny.tsv.gz", + tempfile(fileext = ".tsv.gz"), + input_columns = "SNP=MARKER,CHR=CHR,BP=BP,EA=A0,OA=A1,EAF=A0FREQ,P=P,BETA=BETA,SE=SE,OR=OR,OR_LB=OR_LB,OR_UB=OR_UB,RSID=RSID", + flip_alleles = FALSE, + populate_rsid_option = populate_rsid_options$partial + ) + expect_equal(nrow(out), 12L) +}) + +test_that("standardise_gwas allows flip_alleles FALSE with populate_rsid none", { + out_none <- standardise_gwas( + "data/test_data_tiny.tsv.gz", + tempfile(fileext = ".tsv.gz"), + input_columns = "SNP=MARKER,CHR=CHR,BP=BP,EA=A0,OA=A1,EAF=A0FREQ,P=P,BETA=BETA,SE=SE,OR=OR,OR_LB=OR_LB,OR_UB=OR_UB,RSID=RSID", + flip_alleles = FALSE, + populate_rsid_option = populate_rsid_options$none + ) + expect_equal(nrow(out_none), 12L) +}) + +test_that("standardise_alleles always flips and unflip_alleles restores original", { + g <- tibble::tibble( + CHR = 1L, + BP = 100L, + EA = "T", + OA = "A", + BETA = 0.2, + EAF = 0.1, + P = 0.5, + SE = 0.1 + ) + flipped <- GeneHackman:::standardise_alleles(g) + expect_equal(flipped$BETA, -0.2) + expect_equal(flipped$EA, "A") + expect_equal(flipped$OA, "T") + expect_equal(flipped$EAF, 1 - 0.1) + expect_true(flipped$.ALLELES_FLIPPED[1]) + + restored <- GeneHackman:::unflip_alleles(flipped) + expect_equal(restored$BETA, 0.2) + expect_equal(restored$EA, "T") + expect_equal(restored$OA, "A") + expect_equal(restored$EAF, 0.1) + expect_false(".ALLELES_FLIPPED" %in% names(restored)) +}) + +test_that("filter_incomplete_rows stops when all rows are incomplete", { + bad <- tibble::tibble( + CHR = 1L, + BP = 100000L, + EA = NA_character_, + OA = "A", + P = 0.05, + BETA = 0.1, + SE = 0.02, + EAF = 0.3 + ) + tmp <- tempfile(fileext = ".tsv.gz") + vroom::vroom_write(bad, tmp) + expect_error( + standardise_gwas( + tmp, + tempfile(fileext = ".tsv.gz"), + input_reference_build = reference_builds$GRCh37, + output_reference_build = reference_builds$GRCh37 + ), + "all rows have been filtered from GWAS" + ) +}) + +test_that("resolve_column_map errors on unresolvable column_map value", { + expect_error( + GeneHackman:::resolve_column_map(1L), + "Error resolving column map" + ) +}) + test_that("gwas_formatting.convert_beta_to_or and back returns the same results", { original_gwas <- vroom::vroom("data/test_data_small.tsv.gz", show_col_types=F) diff --git a/tests/testthat/test_liftover.R b/tests/testthat/test_liftover.R index 7ece891..0b57d59 100644 --- a/tests/testthat/test_liftover.R +++ b/tests/testthat/test_liftover.R @@ -1,13 +1,31 @@ +mock_liftover_bed_output <- function(bed_file_output, unmapped = NULL) { + fixture <- testthat::test_path("data/test_data_small_hg38.bed.gz") + if (!file.exists(fixture)) { + testthat::skip(paste("missing fixture:", fixture)) + } + if (grepl("\\.gz$", fixture, ignore.case = TRUE) && + !grepl("\\.gz$", bed_file_output, ignore.case = TRUE)) { + con <- gzfile(fixture, "rt") + on.exit(close(con), add = TRUE) + writeLines(readLines(con), bed_file_output) + } else { + file.copy(fixture, bed_file_output, overwrite = TRUE) + } + if (!is.null(unmapped) && nzchar(unmapped)) { + file.create(unmapped) + } +} + test_that("liftover.convert_reference_build_via_liftover returns updated data frame", { - local_mock( - system = function(command, wait, intern) { - bed_output_file <- unlist(strsplit(command, " "))[4] - file.copy("data/test_data_small_hg38.bed.gz", bed_output_file) - } + local_mocked_bindings( + run_liftover = function(bed_file_input, bed_file_output, input_build, output_build, unmapped) { + mock_liftover_bed_output(bed_file_output) + } ) - result <- convert_reference_build_via_liftover("data/test_data_small.tsv.gz", - reference_builds$GRCh37, - reference_builds$GRCh38 + result <- convert_reference_build_via_liftover( + "data/test_data_small.tsv.gz", + reference_builds$GRCh37, + reference_builds$GRCh38 ) expect_equal(nrow(result), 75048) @@ -17,25 +35,23 @@ test_that("liftover.convert_reference_build_via_liftover returns updated data fr }) test_that("liftover.convert_reference_build_via_liftover saved output and unmapped", { - local_mock( - system = function(command, wait) { - bed_output_file <- unlist(strsplit(command, " "))[4] - unmapped_file <- unlist(strsplit(command, " "))[5] - file.copy("data/test_data_small_hg38.bed.gz", bed_output_file) - file.create(unmapped_file) + local_mocked_bindings( + run_liftover = function(bed_file_input, bed_file_output, input_build, output_build, unmapped) { + mock_liftover_bed_output(bed_file_output, unmapped = unmapped) } ) output_file <- tempfile(fileext = ".tsv.gz") - convert_reference_build_via_liftover("data/test_data_small.tsv.gz", - reference_builds$GRCh37, - reference_builds$GRCh38, - output_file + convert_reference_build_via_liftover( + "data/test_data_small.tsv.gz", + reference_builds$GRCh37, + reference_builds$GRCh38, + output_file ) expect_true(file.exists(output_file)) expect_true(file.exists(paste0(dirname(output_file), file_prefix(output_file), ".unmapped"))) - result <- vroom::vroom(output_file, show_col_types=F) + result <- vroom::vroom(output_file, show_col_types = FALSE) expect_equal(nrow(result), 75048) expect_equal(nrow(result[is.na(result$BP), ]), 0) expect_false("NEW_BP" %in% colnames(result)) diff --git a/tests/testthat/test_mr.R b/tests/testthat/test_mr.R index e3dcfaa..3f58391 100644 --- a/tests/testthat/test_mr.R +++ b/tests/testthat/test_mr.R @@ -1,8 +1,8 @@ mr_results <- tempfile(fileext = ".tsv") test_that("mr.perform_mr_on_pqtl_datasets runs cis only mr against a gwas", { - local_mock( - list.files = function(dir, pattern, full.names) { + local_mocked_bindings( + mr_list_files = function(dir, pattern, full.names) { return("data/some_eqtlgen_hits.tsv.gz") } )