From 40b631fec8904a8706b11506c43ca583847b2071 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 23:23:57 +0000 Subject: [PATCH 1/2] ci: pin the Android emulator to 37.1.11 for the crash harness The API 34 job has hung since 2026-10-02. The emulator-runner action installs the latest stable emulator when it is given no build, and 37.2.12.0 (build 16428233) became stable on 2026-10-01. On API 34, runs on 37.1.11.0 (build 15917651) passed 19 of 20 times, the one red being the known first-native "received []" flake. Of the runs on 37.2.12.0 that reached the repeat stage, 10 of 10 hung: QEMU stalls about twelve minutes after boot and the harness dies when adb goes offline. The runner image and the system image are the same on the green and the red runs. Pin the emulator to build 15917651 through EMULATOR_BUILD and the action's emulator-build input, fail the step when the running emulator is another build, and log the system image's revision. Sample the host's memory and disk every 15 seconds, and keep dmesg on failure, in the harness-logs artifact, so a further stall is read from evidence. Nothing is skipped, retried or split: API 34 still runs, still repeats native twenty times and still boots once. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01XLy7TiPAxtVjPKEwQ9iwBa --- .github/workflows/crash-harness.yml | 47 +++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/.github/workflows/crash-harness.yml b/.github/workflows/crash-harness.yml index 6cf4a91..ef4d07f 100644 --- a/.github/workflows/crash-harness.yml +++ b/.github/workflows/crash-harness.yml @@ -40,6 +40,13 @@ env: # The example is built to export here, and the harness listens on it. The # emulator reaches the runner's loopback through `adb reverse`. OTEL_PORT: "4421" + # The Android emulator build the harness runs on: 37.1.11.0. It is the last + # build that passed. 37.2.12.0 (build 16428233, stable since 2026-10-01) + # stalls QEMU about twelve minutes into the API 34 run and takes adb with it, + # on every run that reached the repeat stage. Bump this only after the + # harness has passed repeatedly on the new build; the first script line below + # fails the step if the emulator that came up is not this one. + EMULATOR_BUILD: "15917651" jobs: android: @@ -112,12 +119,36 @@ jobs: working-directory: example run: flutter build apk --release --dart-define=OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${{ env.OTEL_PORT }} + # The host's memory and disk every 15 seconds, for the emulator step + # below, whose failures have looked like a host problem (QEMU stalls, adb + # offline). It keeps running after this step and is stopped with the job. + - name: Sample the host + run: | + mkdir -p "$GITHUB_WORKSPACE/harness-logs" + ( + set +e + while true; do + date -u +%FT%TZ + free -m + df -h "$HOME" + ps -eo pid,rss,etimes,args --sort=-rss | head -n 6 | cut -c1-160 + echo + sleep 15 + done + ) > "$GITHUB_WORKSPACE/harness-logs/host.txt" 2>&1 < /dev/null & + # One emulator boot for everything: the host-driven crashes, the # release-build refusal, and last the in-app channel test — last because # `flutter test` rebuilds the debug APK without the endpoint define, and # the harness's `--no-build` runs must use the ones built above. Each line # runs as its own command, and the step fails on the first one that does. # + # The emulator is pinned (`EMULATOR_BUILD`, above) because the action + # installs the latest stable one otherwise, and 37.2.12.0 hangs QEMU on + # API 34. The first line fails the step if the one running is another + # build, and the second logs the system image's revision, so a change in + # either is read from the log rather than guessed. + # # No run is retried: a run that needs a retry is a defect (see #36, where # a retry hid a crash exported twice). Two lines exist to measure that # rather than hope: `native` twenty times over on API 34, and the @@ -126,13 +157,18 @@ jobs: # the window they probe moves with the machine. - name: Crash, relaunch, drain uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + env: + API_LEVEL: ${{ matrix.api-level }} with: api-level: ${{ matrix.api-level }} target: google_apis arch: x86_64 profile: pixel_6 disable-animations: true + emulator-build: ${{ env.EMULATOR_BUILD }} script: | + "$ANDROID_HOME/emulator/emulator" -version | grep -F "build_id $EMULATOR_BUILD" + grep Pkg.Revision "$ANDROID_HOME/system-images/android-$API_LEVEL/google_apis/x86_64/source.properties" dart run tool/android-crash-harness.dart --no-build --kinds jvm,native,anr --log-dir ${{ github.workspace }}/harness-logs/kinds --port ${{ env.OTEL_PORT }} dart run tool/android-crash-harness.dart --no-build --kinds jvm,native --upgrade-apk ${{ runner.temp }}/app-1.0.1-2.apk --log-dir ${{ github.workspace }}/harness-logs/upgrade --port ${{ env.OTEL_PORT }} if [ "${{ matrix.api-level }}" = "34" ]; then dart run tool/android-crash-harness.dart --no-build --kinds native --repeat 20 --log-dir ${{ github.workspace }}/harness-logs/repeat --port ${{ env.OTEL_PORT }}; fi @@ -140,9 +176,16 @@ jobs: dart run tool/android-crash-harness.dart --no-build --release-refusal --port ${{ env.OTEL_PORT }} cd example && flutter test integration_test -d emulator-5554 + - name: Keep the host's kernel log + if: failure() + run: | + mkdir -p "$GITHUB_WORKSPACE/harness-logs" + sudo dmesg -T 2>&1 | tee "$GITHUB_WORKSPACE/harness-logs/host-dmesg.txt" > /dev/null || true + # What the runs saw, kept when one fails: each run's whole logcat and the - # OS's exit records. A failing run also prints the lines that decide it, - # but a flake is read here, after the fact, not re-run. + # OS's exit records, and the host's memory, disk and kernel log. A failing + # run also prints the lines that decide it, but a flake is read here, + # after the fact, not re-run. - name: Keep the harness logs if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 From c890d41cd927aec789955136155ea67507fe392a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 23:34:20 +0000 Subject: [PATCH 2/2] ci: read the emulator build from the SDK record, not from the emulator The first run on this branch showed that the pin works and the guard did not. The emulator downloaded and booted as 37.1.11.0, build 15917651, but the guard ran emulator -version, which loads QEMU, and the runner has no libpulse for it, so the step failed before the harness started. The boot itself does not take that path. The guard now matches Pkg.BuildId in the emulator's source.properties as a whole line. The pinned zip carries that file with the build id in it, and the step still fails if the installed emulator is any other build. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01XLy7TiPAxtVjPKEwQ9iwBa --- .github/workflows/crash-harness.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/crash-harness.yml b/.github/workflows/crash-harness.yml index ef4d07f..e223d32 100644 --- a/.github/workflows/crash-harness.yml +++ b/.github/workflows/crash-harness.yml @@ -45,7 +45,7 @@ env: # stalls QEMU about twelve minutes into the API 34 run and takes adb with it, # on every run that reached the repeat stage. Bump this only after the # harness has passed repeatedly on the new build; the first script line below - # fails the step if the emulator that came up is not this one. + # fails the step if the emulator installed is not this one. EMULATOR_BUILD: "15917651" jobs: @@ -145,9 +145,13 @@ jobs: # # The emulator is pinned (`EMULATOR_BUILD`, above) because the action # installs the latest stable one otherwise, and 37.2.12.0 hangs QEMU on - # API 34. The first line fails the step if the one running is another - # build, and the second logs the system image's revision, so a change in - # either is read from the log rather than guessed. + # API 34. The first line fails the step if the installed emulator is + # another build, and the second logs the system image's revision, so a + # change in either is read from the log rather than guessed. The first + # reads the SDK's own record of the build instead of running `emulator + # -version`, which loads QEMU and fails on this runner for want of + # libpulse; the build that boots is the one the emulator's first log line + # names. # # No run is retried: a run that needs a retry is a defect (see #36, where # a retry hid a crash exported twice). Two lines exist to measure that @@ -167,7 +171,7 @@ jobs: disable-animations: true emulator-build: ${{ env.EMULATOR_BUILD }} script: | - "$ANDROID_HOME/emulator/emulator" -version | grep -F "build_id $EMULATOR_BUILD" + grep -Fx "Pkg.BuildId=$EMULATOR_BUILD" "$ANDROID_HOME/emulator/source.properties" grep Pkg.Revision "$ANDROID_HOME/system-images/android-$API_LEVEL/google_apis/x86_64/source.properties" dart run tool/android-crash-harness.dart --no-build --kinds jvm,native,anr --log-dir ${{ github.workspace }}/harness-logs/kinds --port ${{ env.OTEL_PORT }} dart run tool/android-crash-harness.dart --no-build --kinds jvm,native --upgrade-apk ${{ runner.temp }}/app-1.0.1-2.apk --log-dir ${{ github.workspace }}/harness-logs/upgrade --port ${{ env.OTEL_PORT }}