From 110fa26d41b2ef9641c00f89d5320b73ce8494c3 Mon Sep 17 00:00:00 2001 From: Mateusz Kubuszok Date: Fri, 16 Oct 2026 20:07:33 +0200 Subject: [PATCH] CI: retry sbt runs that failed on a dependency download, cache the pinned snapshots Coursier does not retry a download that is cut off partway ("download error: Caught java.io.IOException (Premature EOF)"). Between 2026-10-14 and 10-16, 5 ssg jobs failed that way on the 8 MB balticporter-engine_3 snapshot jar while the project definition loaded; each of the ~20 jobs of a run downloads it afresh. In the same window one generate job got 404s from repo1.maven.org for 18 long-published poms and one JVM job a 502 fetching the sbt launcher's jars. All 7 were download failures, not build or test failures; a rerun passed. - every sbt run (sbt-guarded, the generator command, format, compile, the tree-sitter update, the re-scale install, the release) goes through .github/scripts/sbt-download-retry.sh: when the command failed, its output names a download failure and no test ran and nothing was published, it deletes coursier's recorded download errors and retries up to 3 times after 20s, 60s, 120s. sbt-guarded no longer reports such a run as a crashed test process. - setup-scala caches only the hand-pinned snapshot repository directory, per OS, keyed on project/plugins.sbt and project/Versions.scala (about 15 MB per entry, inside the quota that keeps the general dependency cache off). - the Codecov upload runs only when the tests passed, so a failed JVM job shows one error. --- .github/actions/generated-port/action.yml | 5 ++- .github/actions/sbt-guarded/action.yml | 11 +++-- .github/actions/setup-scala/action.yml | 15 ++++++- .github/scripts/sbt-download-retry.sh | 52 +++++++++++++++++++++++ .github/workflows/ci.yml | 18 +++++--- .github/workflows/release.yml | 3 +- 6 files changed, 90 insertions(+), 14 deletions(-) create mode 100755 .github/scripts/sbt-download-retry.sh diff --git a/.github/actions/generated-port/action.yml b/.github/actions/generated-port/action.yml index 09b6e5911..608d7bb28 100644 --- a/.github/actions/generated-port/action.yml +++ b/.github/actions/generated-port/action.yml @@ -10,7 +10,8 @@ description: > and injected sources under `ssg-liquid/port` and `ssg-md/port`, and the liqp parser step) are in the key too: a change to any of them must regenerate, and no other job can (it has no submodule). liqp's parser is ANTLR output the upstream does not track, so `command` generates it with liqp's - own maven wrapper before the generator runs. + own maven wrapper before the generator runs. The generator's sbt run is retried when it failed on + a dependency download (.github/scripts/sbt-download-retry.sh). The `uses:` line below is the ONE place the shared action's version is pinned. inputs: @@ -71,5 +72,5 @@ runs: git submodule update --init --depth=1 original-src/liqp original-src/katex original-src/roughjs original-src/path-data-parser original-src/points-on-curve original-src/points-on-path original-src/hachure-fill original-src/terser && (cd original-src/liqp && chmod +x mvnw && ./mvnw -q generate-sources -pl .) && (cd original-src/katex && npm install --no-save 2>/dev/null || true) - && sbt generatePort + && bash "$GITHUB_WORKSPACE/.github/scripts/sbt-download-retry.sh" "$RUNNER_TEMP/generate-port.log" "sbt generatePort" key-extra: ${{ steps.upstreams.outputs.extra }} diff --git a/.github/actions/sbt-guarded/action.yml b/.github/actions/sbt-guarded/action.yml index 4f03bcbec..04c028799 100644 --- a/.github/actions/sbt-guarded/action.yml +++ b/.github/actions/sbt-guarded/action.yml @@ -4,7 +4,8 @@ description: > test binary that dies uncleanly can let the task exit 0 while hiding failures, and an alias that only compiles exits 0 having tested nothing. The run passes only when (a) sbt exits 0, (b) the output proves tests ran to their end — a line matching `require` — and (c) no munit failure - marker appears in the output. + marker appears in the output. A run that failed on a dependency download before any test ran is + retried (.github/scripts/sbt-download-retry.sh) and is never reported as a crash. inputs: command: @@ -27,9 +28,13 @@ runs: # ('Cannot run program "C:/Program Files"'); sbt.bat — what `sbt` means in PowerShell — can. if [ "$RUNNER_OS" = "Windows" ]; then GUARDED_COMMAND="${GUARDED_COMMAND/#sbt /sbt.bat }"; fi set +e - bash -c "$GUARDED_COMMAND" 2>&1 | tee "$log" - code=${PIPESTATUS[0]} + bash "$GITHUB_WORKSPACE/.github/scripts/sbt-download-retry.sh" "$log" "$GUARDED_COMMAND" + code=$? set -e + if [ -e "$log.download-failed" ]; then + # the retry script reported the download failure; no test ran, so nothing crashed + exit 1 + fi fail=0 if [ "$code" -ne 0 ]; then echo "::error::the command exited with code $code" diff --git a/.github/actions/setup-scala/action.yml b/.github/actions/setup-scala/action.yml index e6ec4c96e..b966fae2d 100644 --- a/.github/actions/setup-scala/action.yml +++ b/.github/actions/setup-scala/action.yml @@ -26,7 +26,7 @@ runs: java-version: ${{ inputs.java-version }} architecture: ${{ inputs.architecture }} - # Deliberately NO dependency cache here (coursier/cache-action, setup-java's `cache: sbt`): each + # Deliberately NO general dependency cache here (coursier/cache-action, setup-java's `cache: sbt`): each # job saves its own 1-2 GB entry, which fills the repository's 10 GB Actions-cache quota within # one run and evicts the generated-port entry every other job depends on. Resolving from # Maven Central on a runner costs far less than those entries do. @@ -35,6 +35,19 @@ runs: jvm: ${{ inputs.architecture == '' && inputs.jvm || '' }} apps: sbt + # The ONE dependency cache: the hand-pinned snapshots (the Baltic Porter engine, lls) and + # nothing else, about 15 MB per OS, so it stays far inside the quota above. They are the + # downloads that failed: Central's snapshot repository cut the 8 MB engine jar off partway + # (`Premature EOF`), coursier does not retry that, and every job downloaded it afresh. + # One entry per OS (the jars are the same on every architecture) and per pin. + - uses: actions/cache@v4 + with: + path: | + ~/.cache/coursier/v1/https/central.sonatype.com/repository/maven-snapshots + ~/Library/Caches/Coursier/v1/https/central.sonatype.com/repository/maven-snapshots + ~/AppData/Local/Coursier/Cache/v1/https/central.sonatype.com/repository/maven-snapshots + key: coursier-snapshots-${{ runner.os }}-${{ hashFiles('project/plugins.sbt', 'project/Versions.scala') }} + # For an sbt 2 build the `sbt` launcher defaults to a THIN CLIENT that starts a background # server. On a runner that is wrong three times over: the server keeps the environment of the # step that started it (a publish step's credentials never reach the build), in bash on Windows diff --git a/.github/scripts/sbt-download-retry.sh b/.github/scripts/sbt-download-retry.sh new file mode 100755 index 000000000..199b9b574 --- /dev/null +++ b/.github/scripts/sbt-download-retry.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Run an sbt command line, retrying it (up to 3 times, with backoff) when it failed on a dependency +# download before any test ran or anything was published. Coursier does not retry a download cut +# off partway (`Premature EOF`), and Maven Central sometimes answers 404/502 for artifacts it has +# long held; a rerun fixes both. A failure with no download error, or after a test ran, is final. +# +# usage: sbt-download-retry.sh +# The log file holds the LAST attempt's output. When every attempt failed on a download, +# `.download-failed` is created, so a caller can tell that apart from a crash. +set -uo pipefail + +log="$1" +cmd="$2" +download='Premature EOF|Error fetching artifacts|Error downloading|ResolveException|could not retrieve sbt' +# output proving the command got past dependency resolution and did work that must not be redone +progress='Passed: Total|Failed: Total|==> X |published ' +delays=(20 60 120) + +# the coursier caches of the three runner OSes; a `.*.error` file records a failed download, and +# coursier would answer from it instead of trying again +caches=("${COURSIER_CACHE:-}" "$HOME/.cache/coursier" "$HOME/Library/Caches/Coursier") +if [ -n "${LOCALAPPDATA:-}" ]; then + if command -v cygpath > /dev/null; then + caches+=("$(cygpath -u "$LOCALAPPDATA")/Coursier/Cache") + else + caches+=("$LOCALAPPDATA/Coursier/Cache") + fi +fi + +rm -f "$log.download-failed" +attempt=0 +while :; do + bash -c "$cmd" 2>&1 | tee "$log" + code=${PIPESTATUS[0]} + if [ "$code" -eq 0 ] || ! grep -qE "$download" "$log" || grep -qE "$progress" "$log"; then + exit "$code" + fi + if [ "$attempt" -ge "${#delays[@]}" ]; then + echo "::error::dependency download failed ${#delays[@]} retries in a row (a network or repository failure, not a build or test failure)" + touch "$log.download-failed" + exit "$code" + fi + delay=${delays[$attempt]} + attempt=$((attempt + 1)) + echo "::warning::dependency download failed, retrying ($attempt/${#delays[@]}) in ${delay}s" + for dir in "${caches[@]}"; do + if [ -n "$dir" ] && [ -d "$dir" ]; then + find "$dir" -type f -name '.*.error' -delete + fi + done + sleep "$delay" +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0d84daa2..a02339354 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,10 @@ env: # and the build leaves the remote cache OFF. Entries are shared between jobs of the same # os × arch only, so the linux-x64 rows (compile + the JS shards + one JVM and one Native row) # are the main win; the other rows compile for themselves. - # Deliberately NO actions/cache for ~/.cache/sbt/v2 or for dependencies: the remote cache - # supersedes the first and the second evicts the generated port (see setup-scala). + # Deliberately NO actions/cache for ~/.cache/sbt/v2 or for dependencies beyond the hand-pinned + # snapshots: the remote cache supersedes the first and the second evicts the generated port (see + # setup-scala). Every sbt run retries a failed dependency download + # (.github/scripts/sbt-download-retry.sh, directly or through sbt-guarded). BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} jobs: @@ -58,7 +60,7 @@ jobs: - uses: ./.github/actions/setup-scala - name: Check formatting - run: sbt "scalafmtCheckAll; scalafmtSbtCheck" + run: bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/format.log" 'sbt "scalafmtCheckAll; scalafmtSbtCheck"' - name: Lint the workflows run: | @@ -92,7 +94,7 @@ jobs: set -euo pipefail git clone --depth=1 https://github.com/kubuszok/re-scale.git "$RUNNER_TEMP/re-scale" cd "$RUNNER_TEMP/re-scale" - sbt install + bash "$GITHUB_WORKSPACE/.github/scripts/sbt-download-retry.sh" "$RUNNER_TEMP/re-scale-install.log" "sbt install" echo "$RUNNER_TEMP/re-scale/bin" >> "$GITHUB_PATH" - name: Print re-scale version @@ -169,7 +171,7 @@ jobs: key: ${{ needs.generate.outputs.key }} - name: Test/compile all modules - run: sbt 'testCompile-${{ matrix.row }}-3' + run: bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/compile.log" 'sbt testCompile-${{ matrix.row }}-3' # ─── JVM tests (6 desktop platforms) ─────────────────────────────── @@ -220,8 +222,10 @@ jobs: with: command: sbt ci-jvm-3 + # Only after the tests passed: with no report fail_ci_if_error fails the job a second time, + # and a failed job should show one error, the test run's. - name: Upload coverage to Codecov - if: always() + if: success() uses: codecov/codecov-action@v7 with: # sbt 2.0's target/out layout: `sbt ci-jvm-3` runs `coverageAggregate`, which writes the @@ -278,7 +282,7 @@ jobs: - name: Extract WASM and query files for tree-sitter if: matrix.shard == 'engines' run: | - sbt 'ssg-highlightJS/update' + bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/update.log" "sbt ssg-highlightJS/update" WASM_JAR=$(find ~/.cache/coursier ~/.ivy2 \ -name "wasm-provider-tree-sitter-*.jar" \ -not -name "*-sources.jar" -not -name "*-javadoc.jar" 2>/dev/null | head -1) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e99595ffa..d699a1f03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,8 @@ jobs: # setup-scala makes every sbt call its own foreground JVM, so the credentials below reach # the build even though the generation step ran sbt first. - name: Publish all projects - run: sbt ci-release + # retried only on a dependency download failure before anything was published + run: bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/release.log" "sbt ci-release" env: SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}