CI: retry sbt runs that failed on a dependency download, cache the pinned snapshots

.github/actions/generated-port/action.yml

@@ -10,7 +10,8 @@ description: >
   and injected sources under `ssg-liquid/port` and `ssg-md/port`, and the liqp parser step) are in
   the key too: a change to any of them must regenerate, and no other job can (it has no submodule).
   liqp's parser is ANTLR output the upstream does not track, so `command` generates it with liqp's
-  own maven wrapper before the generator runs.
+  own maven wrapper before the generator runs. The generator's sbt run is retried when it failed on
+  a dependency download (.github/scripts/sbt-download-retry.sh).
   The `uses:` line below is the ONE place the shared action's version is pinned.
 
 inputs:
@@ -71,5 +72,5 @@ runs:
           git submodule update --init --depth=1 original-src/liqp original-src/katex original-src/roughjs original-src/path-data-parser original-src/points-on-curve original-src/points-on-path original-src/hachure-fill original-src/terser
           && (cd original-src/liqp && chmod +x mvnw && ./mvnw -q generate-sources -pl .)
           && (cd original-src/katex && npm install --no-save 2>/dev/null || true)
-          && sbt generatePort
+          && bash "$GITHUB_WORKSPACE/.github/scripts/sbt-download-retry.sh" "$RUNNER_TEMP/generate-port.log" "sbt generatePort"
         key-extra: ${{ steps.upstreams.outputs.extra }}

.github/actions/sbt-guarded/action.yml

@@ -4,7 +4,8 @@ description: >
   test binary that dies uncleanly can let the task exit 0 while hiding failures, and an alias that
   only compiles exits 0 having tested nothing. The run passes only when (a) sbt exits 0, (b) the
   output proves tests ran to their end — a line matching `require` — and (c) no munit failure
-  marker appears in the output.
+  marker appears in the output. A run that failed on a dependency download before any test ran is
+  retried (.github/scripts/sbt-download-retry.sh) and is never reported as a crash.
 
 inputs:
   command:
@@ -27,9 +28,13 @@ runs:
         # ('Cannot run program "C:/Program Files"'); sbt.bat — what `sbt` means in PowerShell — can.
         if [ "$RUNNER_OS" = "Windows" ]; then GUARDED_COMMAND="${GUARDED_COMMAND/#sbt /sbt.bat }"; fi
         set +e
-        bash -c "$GUARDED_COMMAND" 2>&1 | tee "$log"
-        code=${PIPESTATUS[0]}
+        bash "$GITHUB_WORKSPACE/.github/scripts/sbt-download-retry.sh" "$log" "$GUARDED_COMMAND"
+        code=$?
         set -e
+        if [ -e "$log.download-failed" ]; then
+          # the retry script reported the download failure; no test ran, so nothing crashed
+          exit 1
+        fi
         fail=0
         if [ "$code" -ne 0 ]; then
           echo "::error::the command exited with code $code"

.github/actions/setup-scala/action.yml

@@ -26,7 +26,7 @@ runs:
         java-version: ${{ inputs.java-version }}
         architecture: ${{ inputs.architecture }}
 
-    # Deliberately NO dependency cache here (coursier/cache-action, setup-java's `cache: sbt`): each
+    # Deliberately NO general dependency cache here (coursier/cache-action, setup-java's `cache: sbt`): each
     # job saves its own 1-2 GB entry, which fills the repository's 10 GB Actions-cache quota within
     # one run and evicts the generated-port entry every other job depends on. Resolving from
     # Maven Central on a runner costs far less than those entries do.
@@ -35,6 +35,19 @@ runs:
         jvm: ${{ inputs.architecture == '' && inputs.jvm || '' }}
         apps: sbt
 
+    # The ONE dependency cache: the hand-pinned snapshots (the Baltic Porter engine, lls) and
+    # nothing else, about 15 MB per OS, so it stays far inside the quota above. They are the
+    # downloads that failed: Central's snapshot repository cut the 8 MB engine jar off partway
+    # (`Premature EOF`), coursier does not retry that, and every job downloaded it afresh.
+    # One entry per OS (the jars are the same on every architecture) and per pin.
+    - uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/coursier/v1/https/central.sonatype.com/repository/maven-snapshots
+          ~/Library/Caches/Coursier/v1/https/central.sonatype.com/repository/maven-snapshots
+          ~/AppData/Local/Coursier/Cache/v1/https/central.sonatype.com/repository/maven-snapshots
+        key: coursier-snapshots-${{ runner.os }}-${{ hashFiles('project/plugins.sbt', 'project/Versions.scala') }}
+
     # For an sbt 2 build the `sbt` launcher defaults to a THIN CLIENT that starts a background
     # server. On a runner that is wrong three times over: the server keeps the environment of the
     # step that started it (a publish step's credentials never reach the build), in bash on Windows

.github/scripts/sbt-download-retry.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Run an sbt command line, retrying it (up to 3 times, with backoff) when it failed on a dependency
+# download before any test ran or anything was published. Coursier does not retry a download cut
+# off partway (`Premature EOF`), and Maven Central sometimes answers 404/502 for artifacts it has
+# long held; a rerun fixes both. A failure with no download error, or after a test ran, is final.
+#
+# usage: sbt-download-retry.sh <log file> <command line>
+# The log file holds the LAST attempt's output. When every attempt failed on a download,
+# `<log file>.download-failed` is created, so a caller can tell that apart from a crash.
+set -uo pipefail
+
+log="$1"
+cmd="$2"
+download='Premature EOF|Error fetching artifacts|Error downloading|ResolveException|could not retrieve sbt'
+# output proving the command got past dependency resolution and did work that must not be redone
+progress='Passed: Total|Failed: Total|==> X |published '
+delays=(20 60 120)
+
+# the coursier caches of the three runner OSes; a `.*.error` file records a failed download, and
+# coursier would answer from it instead of trying again
+caches=("${COURSIER_CACHE:-}" "$HOME/.cache/coursier" "$HOME/Library/Caches/Coursier")
+if [ -n "${LOCALAPPDATA:-}" ]; then
+  if command -v cygpath > /dev/null; then
+    caches+=("$(cygpath -u "$LOCALAPPDATA")/Coursier/Cache")
+  else
+    caches+=("$LOCALAPPDATA/Coursier/Cache")
+  fi
+fi
+
+rm -f "$log.download-failed"
+attempt=0
+while :; do
+  bash -c "$cmd" 2>&1 | tee "$log"
+  code=${PIPESTATUS[0]}
+  if [ "$code" -eq 0 ] || ! grep -qE "$download" "$log" || grep -qE "$progress" "$log"; then
+    exit "$code"
+  fi
+  if [ "$attempt" -ge "${#delays[@]}" ]; then
+    echo "::error::dependency download failed ${#delays[@]} retries in a row (a network or repository failure, not a build or test failure)"
+    touch "$log.download-failed"
+    exit "$code"
+  fi
+  delay=${delays[$attempt]}
+  attempt=$((attempt + 1))
+  echo "::warning::dependency download failed, retrying ($attempt/${#delays[@]}) in ${delay}s"
+  for dir in "${caches[@]}"; do
+    if [ -n "$dir" ] && [ -d "$dir" ]; then
+      find "$dir" -type f -name '.*.error' -delete
+    fi
+  done
+  sleep "$delay"
+done

.github/workflows/ci.yml

@@ -39,8 +39,10 @@ env:
   # and the build leaves the remote cache OFF. Entries are shared between jobs of the same
   # os × arch only, so the linux-x64 rows (compile + the JS shards + one JVM and one Native row)
   # are the main win; the other rows compile for themselves.
-  # Deliberately NO actions/cache for ~/.cache/sbt/v2 or for dependencies: the remote cache
-  # supersedes the first and the second evicts the generated port (see setup-scala).
+  # Deliberately NO actions/cache for ~/.cache/sbt/v2 or for dependencies beyond the hand-pinned
+  # snapshots: the remote cache supersedes the first and the second evicts the generated port (see
+  # setup-scala). Every sbt run retries a failed dependency download
+  # (.github/scripts/sbt-download-retry.sh, directly or through sbt-guarded).
   BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
 
 jobs:
@@ -58,7 +60,7 @@ jobs:
       - uses: ./.github/actions/setup-scala
 
       - name: Check formatting
-        run: sbt "scalafmtCheckAll; scalafmtSbtCheck"
+        run: bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/format.log" 'sbt "scalafmtCheckAll; scalafmtSbtCheck"'
 
       - name: Lint the workflows
         run: |
@@ -92,7 +94,7 @@ jobs:
           set -euo pipefail
           git clone --depth=1 https://github.com/kubuszok/re-scale.git "$RUNNER_TEMP/re-scale"
           cd "$RUNNER_TEMP/re-scale"
-          sbt install
+          bash "$GITHUB_WORKSPACE/.github/scripts/sbt-download-retry.sh" "$RUNNER_TEMP/re-scale-install.log" "sbt install"
           echo "$RUNNER_TEMP/re-scale/bin" >> "$GITHUB_PATH"
 
       - name: Print re-scale version
@@ -169,7 +171,7 @@ jobs:
           key: ${{ needs.generate.outputs.key }}
 
       - name: Test/compile all modules
-        run: sbt 'testCompile-${{ matrix.row }}-3'
+        run: bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/compile.log" 'sbt testCompile-${{ matrix.row }}-3'
 
   # ─── JVM tests (6 desktop platforms) ───────────────────────────────
 
@@ -220,8 +222,10 @@ jobs:
         with:
           command: sbt ci-jvm-3
 
+      # Only after the tests passed: with no report fail_ci_if_error fails the job a second time,
+      # and a failed job should show one error, the test run's.
       - name: Upload coverage to Codecov
-        if: always()
+        if: success()
         uses: codecov/codecov-action@v7
         with:
           # sbt 2.0's target/out layout: `sbt ci-jvm-3` runs `coverageAggregate`, which writes the
@@ -278,7 +282,7 @@ jobs:
       - name: Extract WASM and query files for tree-sitter
         if: matrix.shard == 'engines'
         run: |
-          sbt 'ssg-highlightJS/update'
+          bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/update.log" "sbt ssg-highlightJS/update"
           WASM_JAR=$(find ~/.cache/coursier ~/.ivy2 \
             -name "wasm-provider-tree-sitter-*.jar" \
             -not -name "*-sources.jar" -not -name "*-javadoc.jar" 2>/dev/null | head -1)

.github/workflows/release.yml

@@ -48,7 +48,8 @@ jobs:
       # setup-scala makes every sbt call its own foreground JVM, so the credentials below reach
       # the build even though the generation step ran sbt first.
       - name: Publish all projects
-        run: sbt ci-release
+        # retried only on a dependency download failure before anything was published
+        run: bash .github/scripts/sbt-download-retry.sh "$RUNNER_TEMP/release.log" "sbt ci-release"
         env:
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}