feat: persist per-runner cache volumes with surgical prune

Dockerfile

@@ -78,9 +78,12 @@ ENV ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk
 ENV ANDROID_HOME=$ANDROID_ROOT/sdk
 LABEL maintainer="ernstjason1@gmail.com"
 
-# Post-job cleanup hook to prevent unbounded cache growth
+# Job hooks: JOB_STARTED is the poison sweep (covers OOM / killed runner);
+# JOB_COMPLETED is the tidy-up. Neither wipes package / Gradle caches.
+COPY pre-job.sh /usr/local/bin/pre-job.sh
 COPY cleanup.sh /usr/local/bin/cleanup.sh
-RUN chmod +x /usr/local/bin/cleanup.sh
+RUN chmod +x /usr/local/bin/pre-job.sh /usr/local/bin/cleanup.sh
+ENV ACTIONS_RUNNER_HOOK_JOB_STARTED=/usr/local/bin/pre-job.sh
 ENV ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/usr/local/bin/cleanup.sh
 
 # Preflight entrypoint: validates the persisted runner registration against

README.md

@@ -17,8 +17,55 @@ Linux hosts run natively without emulation.
 To run the container, check out the environment variables from the base image:
 https://github.com/myoung34/docker-github-actions-runner#environment-variables
 
-There is also an example docker-compose file which uses .env file to set the
-variables.
+[`docker-compose.yml`](docker-compose.yml) is the X64 beelink layout — three
+persistent runners, each with its own work **and** cache volumes — not a
+side file. After pulling a new image on the host:
+
+```
+docker compose pull && docker compose up -d
+```
+
+Existing named `runner-work*` volumes keep their data; new cache volumes
+start empty and warm on first jobs. Do not share one bun/gradle volume
+across the three containers: concurrent jobs would corrupt the same tree.
+
+## X64 cache volumes
+
+This compose is for the **X64** class (`beelink`, `docker`, `large`). System
+labels (`Linux`, `X64`) are still applied by the runner binary; set
+`LABELS` in `.env` to `beelink,docker,large` (do not add a `build` label).
+`EPHEMERAL` stays `false` so the volumes survive between jobs.
+
+Each runner gets its own:
+
+| Mount | Volume (runner / runner-2 / runner-3) | Why |
+|-------|----------------------------------------|-----|
+| `/tmp/runner/work` | `runner-work*` | Job workspace. Cleanup spares `$RUNNER_WORKDIR`. |
+| `/root/.bun` | `runner-bun*` | Bun install cache |
+| `/root/.npm` | `runner-npm*` | npm cache |
+| `/root/.cache/pnpm` | `runner-pnpm*` | pnpm store |
+| `/root/.gradle` | `runner-gradle*` | Gradle modules, build-cache, transforms |
+| `/root/.cargo` | `runner-cargo*` | Cargo registry |
+
+Budget ~20 GB per runner (enforced by the post-job hook with LRU, not a
+wipe), ~60 GB per beelink. ARM64 / Pi hosts should **not** mount these
+cache volumes — those runners isolate to `RUNNER_TEMP`.
+
+Do **not** tmpfs `/tmp`. `RUNNER_WORKDIR` is `/tmp/runner/work`; a tmpfs
+or a blanket `find /tmp -delete` hides or wipes the work volume.
+
+### Workflows must use the volume homes
+
+Named volumes are unused if the job relocates the caches. On these X64
+self-hosted runners, workflows must **not** isolate `GRADLE_USER_HOME` or
+`BUN_INSTALL_CACHE_DIR` (or `npm_config_cache` / `CARGO_HOME`) to
+`$RUNNER_TEMP`. That is the `setup-stack` local-home mode in
+`dodi-smart/.github` — `deps-verify` stays isolated; `pr-checks` uses the
+default homes so the volumes are actually read.
+
+The other half of cache poison is GitHub `actions/cache` with
+`restore-keys` re-importing a partial bun tarball. Use an exact key, or
+do not upload the bun install cache at all.
 
 ## Image tags
 

cleanup.sh

@@ -1,28 +1,104 @@
 #!/bin/bash
 # Post-job cleanup script for GitHub Actions runner.
 # Invoked automatically via ACTIONS_RUNNER_HOOK_JOB_COMPLETED after each job.
-# Removes rebuildable caches that grow unbounded between jobs.
+#
+# Persist package and Gradle/Cargo caches on the named volumes. Cap each
+# runner's gradle-home and bun-home at ~20 GB with LRU eviction — do not
+# wholesale wipe. JOB_STARTED (pre-job.sh) is the poison sweep for a killed
+# runner process; this hook is the tidy-up when the job finishes normally.
 #
 # Intentionally not using set -e: individual cleanup failures should not
 # prevent other cleanups from running.
+#
+# KEEP: /root/.bun, /root/.npm, /root/.cache/pnpm, /root/.cache/yarn,
+#       Gradle modules-2 / build-cache-* / transforms-*, Cargo registry.
+# DELETE: /tmp contents except $RUNNER_WORKDIR (and $RUNNER_TEMP if under
+#         /tmp), Gradle daemon/journals/locks, runner _diag logs.
+
+CACHE_CAP_BYTES="${CACHE_CAP_BYTES:-21474836480}" # 20 GiB
 
 echo "[cleanup] Running post-job cleanup..."
 BEFORE=$(df -h / | awk 'NR==2 {print $4}')
 
-# Temp files from builds
-find /tmp -mindepth 1 -delete 2>/dev/null || true
+# True when $1 is $2, a descendant of $2, or an ancestor of $2.
+_spared() {
+  local path="$1"
+  local keep="$2"
+  [ -z "$keep" ] && return 1
+  case "$path" in
+    "$keep"|"$keep"/*) return 0 ;;
+  esac
+  case "$keep" in
+    "$path"/*) return 0 ;;
+  esac
+  return 1
+}
 
-# Gradle build caches (rebuilt each job from source)
-rm -rf /root/.gradle/caches/build-cache-* 2>/dev/null || true
-rm -rf /root/.gradle/caches/transforms-* 2>/dev/null || true
-rm -rf /root/.gradle/caches/journal-* 2>/dev/null || true
+# RUNNER_WORKDIR is /tmp/runner/work on the beelinks. A blanket
+# `find /tmp -delete` wipes the work volume. Spare the workdir (and
+# RUNNER_TEMP if it sits under /tmp), plus their ancestors under /tmp.
+clean_tmp() {
+  local workdir="${RUNNER_WORKDIR:-/tmp/runner/work}"
+  local temp="${RUNNER_TEMP:-}"
+  workdir="${workdir%/}"
+  temp="${temp%/}"
+
+  if [ -n "$temp" ] && [ "$temp" != "/tmp" ] && [ "${temp#/tmp/}" = "$temp" ]; then
+    temp=""
+  fi
+  if [ "$temp" = "/tmp" ]; then
+    temp=""
+  fi
 
-# Gradle daemon logs and state
+  local path
+  while IFS= read -r -d '' path; do
+    _spared "$path" "$workdir" && continue
+    _spared "$path" "$temp" && continue
+    rm -rf "$path" 2>/dev/null || true
+  done < <(find /tmp -mindepth 1 -depth -print0 2>/dev/null)
+}
+
+# LRU-evict oldest regular files until $1 is at or under CACHE_CAP_BYTES.
+# Does not wholesale wipe the tree.
+lru_cap() {
+  local dir="$1"
+  [ -d "$dir" ] || return 0
+  local size
+  size=$(du -sb "$dir" 2>/dev/null | awk '{print $1}')
+  [ -n "$size" ] || return 0
+  if [ "$size" -le "$CACHE_CAP_BYTES" ]; then
+    return 0
+  fi
+  local excess=$((size - CACHE_CAP_BYTES))
+  local freed=0
+  echo "[cleanup] $dir is ${size} bytes (cap ${CACHE_CAP_BYTES}); LRU-evicting oldest files"
+  local rec fsize path
+  while IFS= read -r -d '' rec; do
+    [ "$freed" -ge "$excess" ] && break
+    rec="${rec#*	}"
+    fsize="${rec%%	*}"
+    path="${rec#*	}"
+    [ -n "$path" ] && [ -n "$fsize" ] || continue
+    rm -f "$path" 2>/dev/null || true
+    freed=$((freed + fsize))
+  done < <(find "$dir" -type f -printf '%T@\t%s\t%p\0' 2>/dev/null | sort -z -n)
+  find "$dir" -type d -empty -delete 2>/dev/null || true
+}
+
+clean_tmp
+
+# Gradle daemon / journals / locks only. Keep modules-2, build-cache-*,
+# transforms-*.
 rm -rf /root/.gradle/daemon/ 2>/dev/null || true
+rm -rf /root/.gradle/caches/journal-* 2>/dev/null || true
+find /root/.gradle -name '*.lock' -delete 2>/dev/null || true
 
 # Runner diagnostic logs
 find /actions-runner/_diag -name '*.log' -delete 2>/dev/null || true
 find /runner-data/_diag -name '*.log' -delete 2>/dev/null || true
 
+lru_cap /root/.gradle
+lru_cap /root/.bun
+
 AFTER=$(df -h / | awk 'NR==2 {print $4}')
 echo "[cleanup] Post-job cleanup complete. Disk free: ${BEFORE} -> ${AFTER}"

docker-compose.yml

@@ -1,16 +1,75 @@
+# Beelink X64 layout: three persistent runners, each with its own work and
+# cache volumes. Do not share bun/gradle/npm/cargo volumes across services —
+# concurrent jobs would corrupt the same tree. YAML merge of volumes:
+# replaces the list, so docker.sock is repeated on each service.
+x-runner: &runner
+  image: ghcr.io/dodi-smart/github-runner
+  restart: unless-stopped
+  environment:
+    APP_ID: ${APP_ID}
+    APP_PRIVATE_KEY: ${APP_PRIVATE_KEY}
+    APP_LOGIN: ${APP_LOGIN}
+    RUNNER_NAME_PREFIX: ${RUNNER_NAME_PREFIX:-runner}
+    RUNNER_WORKDIR: /tmp/runner/work
+    RUNNER_SCOPE: org
+    ORG_NAME: ${APP_LOGIN}
+    LABELS: ${LABELS:-self-hosted,Linux,X64,beelink,docker,large}
+    RUNNER_GROUP: ${RUNNER_GROUP:-Default}
+    DISABLE_AUTO_UPDATE: ${DISABLE_AUTO_UPDATE:-false}
+    EPHEMERAL: "false"
+  volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+  security_opt:
+    - label:disable
+
 services:
-  gha:
-    build:
-      context: .
-    # if you don't want to build and instead just use the pre-built image
-    # comment out the build block above and uncomment below
-    # image: ghcr.io/dodi-smart/github-runner
-    container_name: gha
-    env_file:
-      - .env
-    restart: always
-    # uncomment below for interactive debugging, this will override the normal entrypoint and prevent
-    # gh actions runner from starting up, but is good for debugging path issues, etc.
-#    entrypoint: /bin/bash
-#    tty: true
-#    stdin_open: true
+  runner:
+    <<: *runner
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - runner-work:/tmp/runner/work
+      - runner-bun:/root/.bun
+      - runner-npm:/root/.npm
+      - runner-pnpm:/root/.cache/pnpm
+      - runner-gradle:/root/.gradle
+      - runner-cargo:/root/.cargo
+  runner-2:
+    <<: *runner
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - runner-work-2:/tmp/runner/work
+      - runner-bun-2:/root/.bun
+      - runner-npm-2:/root/.npm
+      - runner-pnpm-2:/root/.cache/pnpm
+      - runner-gradle-2:/root/.gradle
+      - runner-cargo-2:/root/.cargo
+  runner-3:
+    <<: *runner
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - runner-work-3:/tmp/runner/work
+      - runner-bun-3:/root/.bun
+      - runner-npm-3:/root/.npm
+      - runner-pnpm-3:/root/.cache/pnpm
+      - runner-gradle-3:/root/.gradle
+      - runner-cargo-3:/root/.cargo
+
+volumes:
+  runner-work: {}
+  runner-work-2: {}
+  runner-work-3: {}
+  runner-bun: {}
+  runner-bun-2: {}
+  runner-bun-3: {}
+  runner-npm: {}
+  runner-npm-2: {}
+  runner-npm-3: {}
+  runner-pnpm: {}
+  runner-pnpm-2: {}
+  runner-pnpm-3: {}
+  runner-gradle: {}
+  runner-gradle-2: {}
+  runner-gradle-3: {}
+  runner-cargo: {}
+  runner-cargo-2: {}
+  runner-cargo-3: {}

pre-job.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Pre-job hook for GitHub Actions runner.
+# Invoked via ACTIONS_RUNNER_HOOK_JOB_STARTED before each job.
+#
+# JOB_COMPLETED does not run when the runner process dies (OOM). This sweep
+# covers that case: delete incomplete/poison entries only. Do not rm -rf
+# cache trees — named volumes are the warm cache.
+#
+# Intentionally not using set -e: individual prune failures should not
+# prevent the job from starting.
+#
+# NOTE: this is only half the fix. Workflows that restore ~/.bun/install/cache
+# via actions/cache with restore-keys can re-import a poisoned entry from the
+# GitHub cache, which outlives this container. Those workflows should use an
+# exact cache key (no restore-keys) so a corrupt entry cannot be carried
+# forward across lockfile changes.
+
+echo "[pre-job] Surgical prune of incomplete cache entries..."
+
+# Bun: a killed job leaves partial tarballs (*.tmp / *.part). The next job
+# then fails with "Fail extracting tarball for ..." and it looks like a
+# dependency problem rather than a runner problem.
+find /root/.bun/install/cache \( -name '*.tmp' -o -name '*.part' \) -delete 2>/dev/null || true
+
+# Gradle: leftover lock / journal / daemon state from a killed compile.
+# Keep modules-2, build-cache-*, and transforms-*.
+rm -f /root/.gradle/caches/modules-2/modules-2.lock 2>/dev/null || true
+rm -rf /root/.gradle/caches/journal-* 2>/dev/null || true
+rm -rf /root/.gradle/daemon/ 2>/dev/null || true
+
+echo "[pre-job] Prune complete."