distro.yml

name: Distro

# The whole distribution built and tested on GitHub's runners, in jobs that hand work on as artifacts, all under /mnt/kryptik.

on:
  push:
    branches: [main]
    # A tag builds that release, tests it whole and drafts its Releases page.
    tags: ['v*']
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '.github/workflows/ci.yml'
  # Weekly, main from nothing as a tag builds it: a step that only an empty cache breaks shows within a week.
  schedule:
    - cron: '23 4 * * 0'
  workflow_dispatch:
    inputs:
      role:
        description: "Build and test the release as a development build, or as a production one signed with a throwaway key medium: the release flow, rehearsed"
        type: choice
        options: [development, production]
        default: development
      cold:
        description: "Build every stage from its sources, as a tag does, restoring and saving no stage cache"
        type: boolean
        default: false

# Set here, not in a repository setting; the cache and the artifacts use the runner's own credential.
permissions:
  contents: read

concurrency:
  # A cold run has a group of its own, so a push to main does not cancel it.
  group: distro-${{ github.ref }}${{ (github.event_name == 'schedule' || inputs.cold == true) && '-cold' || '' }}
  cancel-in-progress: true

env:
  # A scheduled or cold run restores and saves no stage cache: it builds what a tag would.
  KRYPTIK_COLD: ${{ github.event_name == 'schedule' || inputs.cold == true }}
  KRYPTIK_WORK: /mnt/kryptik/work
  KRYPTIK_SOURCES: /mnt/kryptik/sources
  KRYPTIK_OUT: /mnt/kryptik/out
  # A step whose inputs moved rebuilds instead of stopping the build.
  KRYPTIK_STALE: rebuild
  NO_COLOR: "1"
  HOST_PACKAGES: >-
    build-essential bison flex texinfo gawk m4 patch perl python3 python3-pip
    xz-utils bzip2 zstd cpio rsync bc curl git openssl libssl-dev libelf-dev
    e2fsprogs dosfstools mtools exfatprogs xorriso util-linux cryptsetup-bin sbsigntool libasound2-dev
    qemu-system-x86 ovmf musl-tools file

# Caches are keyed on the stamps' digest and saved after a failure too; the stage 05 tree's key starts with the toolchain's.

jobs:
  toolchain:
    name: Stages 01-02 (cross toolchain, temporary tools)
    # actions: read, for the tag preflight to read CI's verdict on the commit.
    permissions:
      contents: read
      actions: read
    runs-on: ubuntu-24.04
    timeout-minutes: 360
    outputs:
      ttfp: ${{ steps.progress02.outputs.ttfp }}
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
      # A release is MAJOR.MINOR.PATCH, and 0.0.0 is the build the update suite starts from.
      - name: The tag names the version
        if: github.ref_type == 'tag'
        run: |
          v="${GITHUB_REF_NAME#v}"
          if [[ "$GITHUB_REF_NAME" != v* || ! "$v" =~ ^(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*)){2}$ || "$v" == 0.0.0 ]]; then
            echo "tag ${GITHUB_REF_NAME} does not name a release: tags are v<MAJOR>.<MINOR>.<PATCH>, digits without leading zeros, above 0.0.0"
            exit 1
          fi
          echo "release ${v}"
      # A release takes no held pin.
      - name: The release's pins are reviewed, and none is held
        if: github.ref_type == 'tag'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ./tools/check-source-currency.sh --tsv > pin-survey.tsv
          ./tools/check-pin-reviews.sh --survey pin-survey.tsv --no-held
      - uses: ./.github/actions/prepare
        with:
          sources: fetch
          image-sources: ${{ github.ref_type == 'tag' && 'fetch' || '' }}

      # CI's gates once more, so a tag never outruns them.
      - name: The release's sources are signed, and their provenance holds
        if: github.ref_type == 'tag'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ./tools/verify-signatures.sh --strict
          ./tools/verify-provenance.sh --strict
      - name: CI passed on the tagged commit
        if: github.ref_type == 'tag'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          sha="$(git rev-parse 'HEAD^{commit}')"
          status=none; conclusion=-
          for try in $(seq 1 40); do
            read -r status conclusion < <(gh api "repos/${GITHUB_REPOSITORY}/actions/workflows/ci.yml/runs?head_sha=${sha}&per_page=1" \
              --jq '.workflow_runs[0] | "\(.status // "none") \(.conclusion // "-")"')
            case "$status" in
              completed) break ;;
              none) echo "no CI run for ${sha}: a release is cut from a commit CI has tested"; exit 1 ;;
              *) echo "CI is ${status} on ${sha}; waiting (${try}/40)"; sleep 60 ;;
            esac
          done
          [ "$status" = completed ] || { echo "CI did not finish on ${sha} in time"; exit 1; }
          [ "$conclusion" = success ] || { echo "CI concluded ${conclusion} on ${sha}: a release is cut only from a commit CI passed"; exit 1; }
          echo "CI passed on ${sha}"

      # A tag or a cold run restores nothing: a resumed tree keeps files a step no longer installs.
      - name: Stage 01-02 sysroot (cached)
        id: work02
        if: github.ref_type != 'tag' && env.KRYPTIK_COLD != 'true'
        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: |
            /mnt/kryptik/work/sysroot
            /mnt/kryptik/work/.stamps
          key: work02-
          restore-keys: |
            work02-

      - name: Stage 01, the cross toolchain (unprivileged)
        timeout-minutes: 200
        # The root's sources alone: make's own sources would fetch every image's from upstream too,
        # which no stage here reads and whose cache only the images' jobs restore.
        run: |
          ./tools/fetch-sources.sh --root
          make -o sources toolchain

      - name: Stage 02, the temporary tools (unprivileged)
        id: stage02
        timeout-minutes: 120
        # -o: the step before ran stage 01, which temp-tools would walk again.
        run: make -o toolchain temp-tools

      # A step past its time limit keeps building, so everything stops before the cache reads the tree.
      - name: Stop what the stages left running
        if: ${{ !cancelled() }}
        run: |
          pkill -KILL -f 'build/stages/0[12]-' || true
          pkill -KILL -x make || true

      - name: Progress of stages 01-02
        id: progress02
        if: ${{ !cancelled() }}
        env:
          STAGE02: ${{ steps.stage02.outcome }}
        # Progress is the stamps' fingerprint lines, not their times.
        run: |
          d="$( { cd "$KRYPTIK_WORK/.stamps" && find . -path ./legacy -prune -o -type f ! -name '*.tmp.*' -print0 | LC_ALL=C sort -z | xargs -0r grep -H '^fingerprint: ' | sha256sum | cut -c1-16; } 2>/dev/null || true)"
          if [ -n "$d" ]; then echo "prefix=work02-$d-" >> "$GITHUB_OUTPUT"; fi
          ttfp="$(awk '$1 == "fingerprint:" { print $2; exit }' "$KRYPTIK_WORK/.stamps/tt-gcc2" 2>/dev/null || true)"
          # Without it every toolchain's stage 05 tree would share one key.
          if [ "$STAGE02" = success ] && [ -z "$ttfp" ]; then echo "stage 02 passed and tt-gcc2 has no fingerprint"; exit 1; fi
          echo "ttfp=$ttfp" >> "$GITHUB_OUTPUT"

      # Saved from main alone, which every branch's run restores: a branch's own copy would be read
      # before main's, so main's went unread until the cache's size limit dropped it.
      - name: Save the stage 01-02 sysroot
        if: ${{ !cancelled() && github.ref == 'refs/heads/main' && env.KRYPTIK_COLD != 'true' && steps.progress02.outputs.prefix != '' && !startsWith(steps.work02.outputs.cache-matched-key, steps.progress02.outputs.prefix) }}
        continue-on-error: true
        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: |
            /mnt/kryptik/work/sysroot
            /mnt/kryptik/work/.stamps
          key: ${{ steps.progress02.outputs.prefix }}${{ github.run_id }}-${{ github.run_attempt }}

      - name: Pack the sysroot for the next job
        run: |
          cd /mnt/kryptik
          rm -rf work/build
          members=(sysroot .stamps)
          [ -d work/logs ] && members+=(logs)
          tar -I 'zstd -T0' -cf work-after-02.tar.zst -C work "${members[@]}"
          ls -la work-after-02.tar.zst

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: work-after-02
          path: /mnt/kryptik/work-after-02.tar.zst
          retention-days: 3
          compression-level: 0

      - name: Stage logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: logs-toolchain
          path: /mnt/kryptik/work/logs
          retention-days: 7
          if-no-files-found: ignore

  base:
    name: Stages 04-05 (base system, kernel)
    needs: toolchain
    runs-on: ubuntu-24.04
    timeout-minutes: 360
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
      - uses: ./.github/actions/prepare
        with:
          sources: fetch

      # One root-made tarball, keyed on the toolchain, the pins and stage 03; a change to any restarts stage 04 (about 45 minutes).
      - name: Stage 04-05 sysroot and kernel (cached)
        id: work05
        if: github.ref_type != 'tag' && env.KRYPTIK_COLD != 'true'
        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: /mnt/kryptik/work-after-05.tar.zst
          key: work05-${{ needs.toolchain.outputs.ttfp }}-${{ hashFiles('build/config/versions.env', 'sources.lock', 'build/stages/03-chroot-prep.sh') }}-
          restore-keys: |
            work05-${{ needs.toolchain.outputs.ttfp }}-${{ hashFiles('build/config/versions.env', 'sources.lock', 'build/stages/03-chroot-prep.sh') }}-

      - name: Unpack the stage 04-05 tree
        if: steps.work05.outputs.cache-matched-key != ''
        run: |
          cd /mnt/kryptik
          echo "restored ${{ steps.work05.outputs.cache-matched-key }}"
          sudo tar --zstd -xf work-after-05.tar.zst -C work
          rm -f work-after-05.tar.zst
          ls work; sudo du -sh work/sysroot

      - if: steps.work05.outputs.cache-matched-key == ''
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: work-after-02
          path: /mnt/kryptik

      - name: Unpack the stage 02 sysroot
        if: steps.work05.outputs.cache-matched-key == ''
        run: |
          cd /mnt/kryptik
          tar --zstd -xf work-after-02.tar.zst -C work
          rm -f work-after-02.tar.zst
          ls work; du -sh work/sysroot

      # Built outside the chroot, for musl, by the compiler build/config/rust.lock pins by hash.
      - name: The pinned Rust compiler
        run: |
          d=/mnt/kryptik/rust-dist; mkdir -p "$d"
          grep -v '^#' build/config/rust.lock > "$d/SHA256SUMS"
          while read -r _ f; do curl -fsSL --retry 3 -o "$d/$f" "https://static.rust-lang.org/dist/$f"; done < "$d/SHA256SUMS"
          (cd "$d" && sha256sum --check --strict SHA256SUMS)
          for t in "$d"/*.tar.xz; do tar -xf "$t" -C "$d"; done
          for i in "$d"/*/install.sh; do "$i" --prefix=/mnt/kryptik/rust --disable-ldconfig > /dev/null; done
          echo /mnt/kryptik/rust/bin >> "$GITHUB_PATH"

      - name: kryptikd, kryptik-wlproxy and kryptik-stick, static
        run: |
          want="$(sed -n 's/.*  rustc-\([0-9.]*\)-x86_64-unknown-linux-gnu\.tar\.xz$/\1/p' build/config/rust.lock)"
          rustc --version | grep -qF "rustc ${want} " || { echo "refusing to build what ships with $(rustc --version); the pin is ${want}"; exit 1; }
          # The checkout's, cargo's and the target directory's paths would land in panic
          # locations, which then differ between two builds of one tree.
          home="${CARGO_HOME:-$HOME/.cargo}"
          export RUSTFLAGS="--remap-path-prefix=${GITHUB_WORKSPACE}=/kryptik --remap-path-prefix=${home}=/cargo --remap-path-prefix=${CARGO_TARGET_DIR}=/target"
          (cd compartments/kryptikd && cargo build --locked --release --target x86_64-unknown-linux-musl)
          (cd compositor && cargo build --locked --release --target x86_64-unknown-linux-musl -p wlproxy --bin kryptik-wlproxy)
          (cd tools/stick && cargo build --locked --release --target x86_64-unknown-linux-musl)
          cp /mnt/kryptik/cargo/x86_64-unknown-linux-musl/release/kryptikd /mnt/kryptik/kryptikd-musl
          cp /mnt/kryptik/cargo/x86_64-unknown-linux-musl/release/kryptik-wlproxy /mnt/kryptik/kryptik-wlproxy-musl
          cp /mnt/kryptik/cargo/x86_64-unknown-linux-musl/release/kryptik-stick /mnt/kryptik/kryptik-stick-musl
          for b in /mnt/kryptik/kryptikd-musl /mnt/kryptik/kryptik-wlproxy-musl /mnt/kryptik/kryptik-stick-musl; do
            ! grep -aqF -e "$GITHUB_WORKSPACE" -e "$home" -e "$CARGO_TARGET_DIR" "$b" || { echo "$b names a path of this build"; exit 1; }
          done
          file /mnt/kryptik/kryptikd-musl /mnt/kryptik/kryptik-wlproxy-musl /mnt/kryptik/kryptik-stick-musl
          # The licence texts build/licences ships for what they link.
          tools/check-rust-licences.sh /mnt/kryptik/rust-dist

      - name: Stage 04, the base system (in the chroot, as root)
        id: stage04
        timeout-minutes: 180
        run: make system KRYPTIK_KRYPTIKD_BIN=/mnt/kryptik/kryptikd-musl KRYPTIK_WLPROXY_BIN=/mnt/kryptik/kryptik-wlproxy-musl KRYPTIK_STICK_BIN=/mnt/kryptik/kryptik-stick-musl

      - name: Stage 05, the hardened kernel
        timeout-minutes: 90
        # -o: the step before ran stage 04, which kernel would walk again.
        run: make -o system kernel KRYPTIK_KRYPTIKD_BIN=/mnt/kryptik/kryptikd-musl KRYPTIK_WLPROXY_BIN=/mnt/kryptik/kryptik-wlproxy-musl

      # Nothing is read for the cache while anything still runs or is mounted in the sysroot.
      - name: Stop what the stages left running
        id: quiesce
        if: ${{ !cancelled() && steps.stage04.outcome != '' && steps.stage04.outcome != 'skipped' }}
        run: |
          sudo pkill -KILL -f 'build/stages/0[345]-' || true
          # Pass after pass: a process not yet killed can start another.
          inside() {
            for p in /proc/[0-9]*; do
              case "$(sudo readlink "$p/root" 2>/dev/null)" in "$KRYPTIK_WORK/sysroot"|"$KRYPTIK_WORK/sysroot"/*) echo "${p#/proc/}" ;; esac
            done
          }
          for _ in 1 2 3 4 5 6 7 8 9 10; do
            mapfile -t left < <(inside)
            [ "${#left[@]}" -eq 0 ] && break
            sudo kill -KILL "${left[@]}" 2>/dev/null || true
            sleep 1
          done
          mapfile -t left < <(inside)
          if [ "${#left[@]}" -gt 0 ]; then echo "still running in the sysroot: ${left[*]}"; exit 1; fi
          make chroot-umount || true
          if findmnt -rn -o TARGET | grep -F "$KRYPTIK_WORK/sysroot/"; then echo "still mounted in the sysroot"; exit 1; fi

      # Before stage 06, so no key or image reaches the cache; the kernel tree goes with its stamps.
      - name: Progress of stages 04-05
        id: progress05
        if: ${{ !cancelled() && steps.quiesce.outcome == 'success' }}
        run: |
          d="$( { cd "$KRYPTIK_WORK/.stamps" && find . -path ./legacy -prune -o -type f ! -name '*.tmp.*' -print0 | LC_ALL=C sort -z | xargs -0r grep -H '^fingerprint: ' | sha256sum | cut -c1-16; } 2>/dev/null || true)"
          if [ -n "$d" ]; then echo "prefix=work05-${{ needs.toolchain.outputs.ttfp }}-${{ hashFiles('build/config/versions.env', 'sources.lock', 'build/stages/03-chroot-prep.sh') }}-$d-" >> "$GITHUB_OUTPUT"; fi

      # Never the module signing key: a pull request's run can restore this cache.
      - name: Pack the stage 04-05 tree for the cache
        id: pack05
        if: ${{ !cancelled() && github.ref_type != 'tag' && env.KRYPTIK_COLD != 'true' && steps.progress05.outputs.prefix != '' && !startsWith(steps.work05.outputs.cache-matched-key, steps.progress05.outputs.prefix) }}
        run: |
          cd /mnt/kryptik
          members=(.stamps sysroot)
          kernel_tree="build/linux-$(sed -n 's/^V_LINUX=//p' "$GITHUB_WORKSPACE/build/config/versions.env")"
          [ -d "work/$kernel_tree" ] && members+=("$kernel_tree")
          sudo tar -I 'zstd -T0' -cf work-after-05.tar.zst -C work --anchored \
            --exclude='sysroot/proc/*' --exclude='sysroot/sys/*' --exclude='sysroot/dev/*' \
            --exclude='sysroot/run/*' --exclude='sysroot/kryptik*/*' \
            --exclude="$kernel_tree/certs/signing_key.pem" "${members[@]}"
          if sudo tar -I zstd -tf work-after-05.tar.zst | grep -E 'certs/signing_key\.pem$|keys/release/'; then echo "a private key is in the cache tarball"; exit 1; fi
          sudo chown "$(id -u):$(id -g)" work-after-05.tar.zst
          ls -la work-after-05.tar.zst; df -h /mnt | tail -1

      # A cache that cannot be saved costs the next run time, not this one its verdict.
      - name: Save the stage 04-05 tree
        if: ${{ !cancelled() && steps.pack05.outcome == 'success' }}
        continue-on-error: true
        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: /mnt/kryptik/work-after-05.tar.zst
          key: ${{ steps.progress05.outputs.prefix }}${{ github.run_id }}-${{ github.run_attempt }}

      - name: Drop the cache tarball
        if: always()
        run: rm -f /mnt/kryptik/work-after-05.tar.zst

      # The next jobs' tree, as the cache keeps it, with the logs. Never the module signing key, as anyone
      # who can read the repository can read an artifact: the stage 06 job's stage 05 makes a new one.
      - name: Pack the stage 04-05 tree for the next jobs
        run: |
          cd /mnt/kryptik
          members=(.stamps sysroot logs)
          kernel_tree="build/linux-$(sed -n 's/^V_LINUX=//p' "$GITHUB_WORKSPACE/build/config/versions.env")"
          [ -d "work/$kernel_tree" ] && members+=("$kernel_tree")
          sudo tar -I 'zstd -T0' -cf work-after-05.tar.zst -C work --anchored \
            --exclude='sysroot/proc/*' --exclude='sysroot/sys/*' --exclude='sysroot/dev/*' \
            --exclude='sysroot/run/*' --exclude='sysroot/kryptik*/*' \
            --exclude="$kernel_tree/certs/signing_key.pem" "${members[@]}"
          if sudo tar -I zstd -tf work-after-05.tar.zst | grep -E 'certs/signing_key\.pem$|keys/release/'; then echo "a private key is in the tarball"; exit 1; fi
          sudo chown "$(id -u):$(id -g)" work-after-05.tar.zst
          ls -la work-after-05.tar.zst

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: work-after-05
          path: /mnt/kryptik/work-after-05.tar.zst
          # A tag's stage 06 job may be re-run after its owner's approval, so it keeps a week.
          retention-days: ${{ github.ref_type == 'tag' && 7 || 1 }}
          compression-level: 0

      - name: Stage logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: logs-base
          path: /mnt/kryptik/work/logs
          retention-days: 7
          if-no-files-found: ignore

  # The software images in a job of their own: Firefox alone takes hours, which beside a stage 04 built
  # from nothing (a tag's, or a new pin's) and stage 06 would not fit one job's six.
  images:
    name: Software images (the toolkit, the browser)
    needs: [toolchain, base]
    runs-on: ubuntu-24.04
    timeout-minutes: 360
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
      - uses: ./.github/actions/prepare
        with:
          sources: fetch
          image-sources: fetch

      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: work-after-05
          path: /mnt/kryptik

      - name: Unpack the stage 04-05 tree
        run: |
          cd /mnt/kryptik
          sudo tar --zstd -xf work-after-05.tar.zst -C work
          rm -f work-after-05.tar.zst
          ls work; sudo du -sh work/sysroot

      # The images built over this base system, up to the last step they build against (IMAGES_BASE in
      # 04-images.sh), from an earlier run: make images keeps each whose fingerprint still matches its
      # inputs and builds the rest. Not for a tag or a cold run, which build from nothing.
      - name: Which base system the images are built over
        id: images_key
        if: github.ref_type != 'tag' && env.KRYPTIK_COLD != 'true'
        run: |
          step="$(sed -n 's/^IMAGES_BASE="\([a-z0-9-]*\)"$/\1/p' build/stages/04-images.sh)"
          base="$(sudo sed -n 's/^fingerprint: //p' "$KRYPTIK_WORK/.stamps/bs-${step:-none}" | cut -c1-16)"
          if [ -n "$base" ]; then echo "prefix=images-v1-${{ needs.toolchain.outputs.ttfp }}-${base}-" >> "$GITHUB_OUTPUT"; fi

      - name: Software images from an earlier run (cached)
        id: images_cache
        if: steps.images_key.outputs.prefix != ''
        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: /mnt/kryptik/images-cache.tar.zst
          key: ${{ steps.images_key.outputs.prefix }}
          restore-keys: |
            ${{ steps.images_key.outputs.prefix }}

      - name: Unpack the cached images
        if: steps.images_cache.outputs.cache-matched-key != ''
        run: |
          cd /mnt/kryptik
          sudo tar --zstd -xf images-cache.tar.zst -C work
          rm -f images-cache.tar.zst
          sudo sed -n 's/^\(name\|fingerprint\): //p' work/image-build/*/record | paste - - || true

      # Stage 06 packs what this builds. The browser takes some three hours or more when its tree is not
      # cached, the toolkit some minutes.
      - name: Software images (over the base system, in the chroot)
        id: images
        timeout-minutes: 330
        run: make -o system images

      # Each image's tree and record, never an overlay's upper or work directory; saved when they changed.
      - name: Pack the images for their cache
        id: images_pack
        if: ${{ !cancelled() && steps.images.outcome == 'success' && steps.images_key.outputs.prefix != '' }}
        run: |
          d="$(cd "$KRYPTIK_WORK" && sudo cat image-build/*/record 2>/dev/null | sha256sum | cut -c1-16)"
          key="${{ steps.images_key.outputs.prefix }}${d}-"
          case "${{ steps.images_cache.outputs.cache-matched-key }}" in "$key"*) echo "the images are the cached ones"; exit 0 ;; esac
          cd /mnt/kryptik
          sudo tar -I 'zstd -T0' -cf images-cache.tar.zst -C work --exclude='image-build/*/upper' \
            --exclude='image-build/*/overlay-work' --exclude='image-build/*/sysroot' image-build
          sudo chown "$(id -u):$(id -g)" images-cache.tar.zst
          ls -la images-cache.tar.zst
          echo "key=${key}${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT"

      # A cache that cannot be saved costs the next run time, not this one its verdict.
      - name: Save the images
        if: ${{ !cancelled() && steps.images_pack.outputs.key != '' }}
        continue-on-error: true
        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: /mnt/kryptik/images-cache.tar.zst
          key: ${{ steps.images_pack.outputs.key }}

      - name: Drop the images' tarball
        if: always()
        run: rm -f /mnt/kryptik/images-cache.tar.zst

      # Each image's tree and record for stage 06, cached or not, and the stamps and logs the build left.
      # A layer that does not ship keeps only its record: stage 06 reads no more of it.
      - name: Pack the images for the next job
        run: |
          cd /mnt/kryptik
          layers=()
          for rec in work/image-build/*/record; do
            if [ "$(sudo sed -n 's/^ships: //p' "$rec")" = no ]; then layers+=(--exclude="image-build/$(basename "$(dirname "$rec")")/tree"); fi
          done
          sudo tar -I 'zstd -T0' -cf images-tree.tar.zst -C work --exclude='image-build/*/upper' \
            --exclude='image-build/*/overlay-work' --exclude='image-build/*/sysroot' "${layers[@]}" image-build .stamps logs
          sudo chown "$(id -u):$(id -g)" images-tree.tar.zst
          ls -la images-tree.tar.zst

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: images-tree
          path: /mnt/kryptik/images-tree.tar.zst
          retention-days: ${{ github.ref_type == 'tag' && 7 || 1 }}
          compression-level: 0

      - name: Stage logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: logs-images
          path: /mnt/kryptik/work/logs
          retention-days: 7
          if-no-files-found: ignore

  system:
    name: Stage 06 (the media)
    needs: [base, images]
    runs-on: ubuntu-24.04
    timeout-minutes: 360
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
      - uses: ./.github/actions/prepare
        with:
          sources: restore

      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: work-after-05
          path: /mnt/kryptik

      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: images-tree
          path: /mnt/kryptik

      - name: Unpack the stage 04-05 tree and the images
        run: |
          cd /mnt/kryptik
          sudo tar --zstd -xf work-after-05.tar.zst -C work
          sudo tar --zstd -xf images-tree.tar.zst -C work
          rm -f work-after-05.tar.zst images-tree.tar.zst
          ls work; sudo du -sh work/sysroot work/image-build

      # The tree came without its module signing key, so stage 05 builds and signs again with a new one,
      # as after a cache restore; stage 06 relinks the kernel from that tree.
      - name: Stage 05, keyed afresh
        timeout-minutes: 90
        run: make -o system kernel

      # The throwaway production pair, then A and B; a production release is only bound here, where upstream build scripts ran as root.
      - name: Stage 06, the media (production 1.0.0 and 1.0.1, release A and the update B)
        id: media
        timeout-minutes: 90
        run: |
          make production-pair
          ls -la "$KRYPTIK_WORK/images-production"
          # A tag's version decides its role; elsewhere the dispatch input does.
          ROLE=development
          if [ "$GITHUB_REF_TYPE" = tag ]; then
            VER_A=0.0.0; VER_B="${GITHUB_REF_NAME#v}"
            case "$GITHUB_REF_NAME" in v0.*) ;; *) ROLE=production ;; esac
          elif [ "${{ inputs.role }}" = production ]; then
            ROLE=production; VER_A=0.0.0; VER_B=0.0.1
          else
            VER_A="0.1.$(date -u +%Y%m%d).${GITHUB_SHA::8}"; VER_B="${VER_A}.1"
          fi
          echo "a ${ROLE} release: ${VER_B}, updating from ${VER_A}"
          echo "role=${ROLE}" >> "$GITHUB_OUTPUT"
          if [ "$ROLE" = development ]; then
            for v in "$VER_A" "$VER_B"; do make media KRYPTIK_VERSION="$v"; done
            ls -la "$KRYPTIK_WORK/images" | grep -E 'usb.img|\.iso|payload'
            exit 0
          fi
          # A tag's public half is the tree's; a rehearsal's comes from a throwaway medium the sign job gets whole.
          pub=/mnt/kryptik/keys-public; mkdir -p "$pub"
          if [ "$GITHUB_REF_TYPE" = tag ]; then
            cp build/config/release/release-signers build/config/release/kryptik-sb.crt "$pub/"
          else
            tools/throwaway-key-medium.sh /mnt/kryptik/rehearsal-medium
            cp /mnt/kryptik/rehearsal-medium/release-signers /mnt/kryptik/rehearsal-medium/kryptik-sb.crt "$pub/"
            tar -C /mnt/kryptik/rehearsal-medium -cz . | base64 -w0 > /mnt/kryptik/rehearsal-medium.b64
            rm -rf /mnt/kryptik/rehearsal-medium
          fi
          channel="https://${GITHUB_REPOSITORY_OWNER,,}.github.io/${GITHUB_REPOSITORY#*/}/stable/"
          for v in "$VER_A" "$VER_B"; do
            sudo rm -rf "$KRYPTIK_WORK/images"
            sudo find "$KRYPTIK_WORK/.stamps" -maxdepth 1 -name 'img-*' -delete
            make media KRYPTIK_MEDIA_PHASE=bind KRYPTIK_ROLE=production KRYPTIK_KEYS="$pub" \
              KRYPTIK_VERSION="$v" KRYPTIK_CHANNEL="$channel"
            sudo mkdir -p "$KRYPTIK_WORK/bound/$v/stamps"
            sudo mv "$KRYPTIK_WORK/images" "$KRYPTIK_WORK/bound/$v/images"
            sudo cp -a "$KRYPTIK_WORK/.stamps"/img-* "$KRYPTIK_WORK/bound/$v/stamps/"
          done

      - name: Pack the work tree for the next jobs
        run: |
          tools/image/pack-work.sh production-media
          if [ "${{ steps.media.outputs.role }}" = production ]; then
            tools/image/pack-work.sh work-bound
          else
            tools/image/pack-work.sh work-after-media
          fi

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: steps.media.outputs.role == 'development'
        with:
          name: work-after-media
          path: /mnt/kryptik/work-after-media.tar.zst
          retention-days: 3
          compression-level: 0

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: steps.media.outputs.role == 'production'
        with:
          name: work-bound
          path: /mnt/kryptik/work-bound.tar.zst
          # A release waits for its owner's approval; a rehearsal does not.
          retention-days: ${{ github.ref_type == 'tag' && 7 || 1 }}
          compression-level: 0

      # Throwaway keys, made for this rehearsal alone.
      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: steps.media.outputs.role == 'production' && github.ref_type != 'tag'
        with:
          name: rehearsal-medium
          path: /mnt/kryptik/rehearsal-medium.b64
          retention-days: 1

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: production-media
          path: /mnt/kryptik/production-media.tar.zst
          retention-days: ${{ github.ref_type == 'tag' && 7 || 3 }}
          compression-level: 0

      - name: Stage logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: logs-system
          path: /mnt/kryptik/work/logs
          retention-days: 7
          if-no-files-found: ignore

  # A production release signed on a fresh runner, unprivileged, from what the build job bound: nothing it hands over runs or links here.
  sign:
    name: Stage 06, signed (a production release)
    needs: system
    # From the ref, never from the build job's outputs, which its root processes could write.
    if: ${{ (github.ref_type == 'tag' && !startsWith(github.ref_name, 'v0.')) || (github.ref_type != 'tag' && inputs.role == 'production') }}
    # A tag's keys: the release environment takes only release tags, and its owner approves each run.
    environment: ${{ github.ref_type == 'tag' && !startsWith(github.ref_name, 'v0.') && 'release' || '' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
      - uses: ./.github/actions/prepare

      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: work-bound
          path: /mnt/kryptik

      - if: github.ref_type != 'tag'
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: rehearsal-medium
          path: /mnt/kryptik

      # The bound releases and the stamps alone, as the runner user, and nothing but plain files and directories: a link could reach the keys.
      - name: Unpack what was bound
        run: |
          cd /mnt/kryptik
          mkdir bound-in
          tar --zstd -xf work-bound.tar.zst -C bound-in ./bound ./.stamps
          odd="$(find bound-in ! -type f ! -type d)"
          [ -z "$odd" ] || { echo "the bound tree holds more than plain files and directories:"; echo "$odd" | head; exit 1; }
          ls bound-in/bound

      - name: Sign and assemble the media
        id: sign
        env:
          MEDIUM: ${{ secrets.KRYPTIK_KEY_MEDIUM }}
        run: |
          # The versions and the channel from the ref, as the build job made them.
          if [ "$GITHUB_REF_TYPE" = tag ]; then VER_B="${GITHUB_REF_NAME#v}"; else VER_B=0.0.1; fi
          [[ "$VER_B" =~ ^(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*)){2}$ ]] || { echo "${VER_B} is not a release version"; exit 1; }
          channel="https://${GITHUB_REPOSITORY_OWNER,,}.github.io/${GITHUB_REPOSITORY#*/}/stable/"
          m="$(mktemp -d)"; lines="$(mktemp)"
          trap 'rm -rf "$m" "$lines"' EXIT
          if [ "$GITHUB_REF_TYPE" = tag ]; then
            [ -n "$MEDIUM" ] || { echo "the release environment has no KRYPTIK_KEY_MEDIUM (docs/release-keys.md)"; exit 1; }
            printf '%s' "$MEDIUM" | base64 -d | tar -xz -C "$m"
            cmp build/config/release/release-signers "$m/release-signers" && cmp build/config/release/kryptik-sb.crt "$m/kryptik-sb.crt" \
              || { echo "the release environment's keys are not the ones build/config/release names"; exit 1; }
          else
            base64 -d /mnt/kryptik/rehearsal-medium.b64 | tar -xz -C "$m"
            rm -f /mnt/kryptik/rehearsal-medium.b64
          fi
          unset MEDIUM
          # The secret line of each signing key, which nothing signed or logged may hold.
          { for k in kryptik-release kryptik-latest; do [ ! -f "$m/$k" ] || sed -n 5p "$m/$k"; done
            tail -n 2 "$m/kryptik-sb.key" | head -n 1; } | grep -v '^$' > "$lines"
          sw=/mnt/kryptik/sign-work; so=/mnt/kryptik/sign-out
          mkdir -p "$sw/.stamps" "$sw/logs" "$so"
          cp /mnt/kryptik/bound-in/.stamps/kernel-install /mnt/kryptik/bound-in/.stamps/bs-licences "$sw/.stamps/"
          for v in 0.0.0 "$VER_B"; do
            rm -rf "$sw/images"; find "$sw/.stamps" -maxdepth 1 -name 'img-*' -delete
            mv "/mnt/kryptik/bound-in/bound/$v/images" "$sw/images"
            cp "/mnt/kryptik/bound-in/bound/$v/stamps"/img-* "$sw/.stamps/"
            make media SUDO= KRYPTIK_WORK="$sw" KRYPTIK_OUT="$so" KRYPTIK_MEDIA_PHASE=sign KRYPTIK_ROLE=production \
              KRYPTIK_KEYS="$m" KRYPTIK_VERSION="$v" KRYPTIK_CHANNEL="$channel"
            mv "$sw/images" "/mnt/kryptik/signed-$v"
          done
          # A rehearsal's suites sign control disks with its throwaway key; a release's get theirs from release-tests.
          if [ -f "$m/kryptik-testctl" ]; then mkdir /mnt/kryptik/rehearsal-testctl; cp "$m"/kryptik-testctl "$m"/kryptik-testctl.pub /mnt/kryptik/rehearsal-testctl/; fi
          rm -rf "$m"
          rc=0; grep -rlF -f "$lines" /mnt/kryptik/signed-* "$sw" "$so" || rc=$?
          [ "$rc" -eq 1 ] || { echo "a private key is in what was signed, or it could not be searched (grep exited ${rc})"; exit 1; }
          echo "versions=0.0.0 ${VER_B}" >> "$GITHUB_OUTPUT"

      # The rest of the tree only now, with the keys gone: the suites' tree with the signed media in it.
      - name: Pack the work tree for the acceptance
        run: |
          cd /mnt/kryptik
          sudo tar --zstd -xf work-bound.tar.zst -C work
          rm -f work-bound.tar.zst
          sudo rm -rf work/bound work/images
          sudo mkdir work/images
          for v in ${{ steps.sign.outputs.versions }}; do sudo cp -a "signed-$v/." work/images/; done
          if [ -d rehearsal-testctl ]; then sudo mkdir -p work/keys/medium; sudo cp rehearsal-testctl/* work/keys/medium/; fi
          rm -rf signed-* bound-in rehearsal-testctl
          ls -la work/images | grep -E 'usb.img|\.iso|payload'
          "$GITHUB_WORKSPACE/tools/image/pack-work.sh" work-after-media

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: work-after-media
          path: /mnt/kryptik/work-after-media.tar.zst
          retention-days: 3
          compression-level: 0

      # A release's logs only once the search found no key in them.
      - name: Stage logs
        if: ${{ success() || github.ref_type != 'tag' }}
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: logs-sign
          path: /mnt/kryptik/sign-work/logs
          retention-days: 7
          if-no-files-found: ignore

  # Independent VMs, one part per machine; the verdict job judges the whole run.
  acceptance:
    name: Acceptance (${{ matrix.suites }})
    needs: [system, sign]
    if: ${{ !cancelled() && needs.system.result == 'success' && (needs.sign.result == 'success' || needs.sign.result == 'skipped') }}
    # A release's control-disk key, which no artifact may carry; release-tests takes only release tags. The part
    # that runs the build's own programs as root stays out: its runner would hold the key in a step's environment.
    environment: ${{ github.ref_type == 'tag' && !startsWith(github.ref_name, 'v0.') && !contains(matrix.suites, 'build') && 'release-tests' || '' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 150
    strategy:
      fail-fast: false
      matrix:
        # About 18, 16, 12 and 11 minutes, and the pair's update suite beside build, the one that runs sysroot programs on the host.
        suites: [update, install, 'integrity,zones', 'inputs,boot,desktop', 'build,production']
    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
      - uses: ./.github/actions/prepare
        with:
          vm: 'true'
          sources: restore
          # The inputs suite checks every source against both locks; the other parts read only the root's.
          image-sources: ${{ contains(matrix.suites, 'inputs') && 'restore' || '' }}

      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: work-after-media
          path: /mnt/kryptik

      # The whole production pair, for its own part.
      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        if: contains(matrix.suites, 'production')
        with:
          name: production-media
          path: /mnt/kryptik

      # As root, so the sysroot keeps its owners and modes.
      - name: Unpack the work tree
        run: |
          cd /mnt/kryptik
          sudo tar --zstd -xf work-after-media.tar.zst -C work
          rm -f work-after-media.tar.zst
          if [ -f production-media.tar.zst ]; then sudo tar --zstd -xf production-media.tar.zst -C work; rm -f production-media.tar.zst; fi
          ls work/images; df -h /mnt | tail -1

      # The host suites' proxy-calls traces the static proxy's live tests, as CI's Compartment layer
      # does; root runs them with this user's cargo and rustup, which tools/acceptance.sh finds.
      - name: The proxy-calls suite's target and strace
        if: contains(matrix.suites, 'build')
        run: |
          command -v strace >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq strace; }
          rustup target add x86_64-unknown-linux-musl

      # The script, not make, which turns every failure into 2; exit 2 is a part's normal answer.
      - name: Acceptance of these suites
        env:
          SUITES: ${{ matrix.suites }}
          TESTCTL: ${{ secrets.KRYPTIK_TESTCTL_KEY }}
        run: |
          rc=0
          # A release's media honour control disks signed by their medium's key: a rehearsal's copy, or a release's secret.
          k="$KRYPTIK_WORK/keys/medium/kryptik-testctl"; [ -f "$k" ] || k=""
          if [ "$GITHUB_REF_TYPE" = tag ] && [ -n "$TESTCTL" ] && [[ ",${SUITES}," != *,build,* ]]; then
            k="$(mktemp -d)/kryptik-testctl"
            (umask 077; printf '%s\n' "$TESTCTL" > "$k")
          fi
          unset TESTCTL
          sudo env KRYPTIK_WORK="$KRYPTIK_WORK" KRYPTIK_SOURCES="$KRYPTIK_SOURCES" NO_COLOR=1 KRYPTIK_TESTCTL_KEY="$k" \
            TMPDIR="$TMPDIR" CARGO_TARGET_DIR="$CARGO_TARGET_DIR" tools/acceptance.sh --only "$SUITES" || rc=$?
          [ "$rc" -eq 0 ] || [ "$rc" -eq 2 ]

      - name: Hand the results to the runner user
        if: always()
        run: sudo chown -R "$(id -u):$(id -g)" /mnt/kryptik/work/acceptance /mnt/kryptik/work/logs 2>/dev/null || true

      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        if: always()
        with:
          name: acceptance-part-${{ strategy.job-index }}
          path: |
            /mnt/kryptik/work/acceptance
            /mnt/kryptik/work/logs
          retention-days: 14
          if-no-files-found: ignore

  verdict:
    name: make acceptance (the verdict over every suite)
    needs: [system, sign, acceptance]
    if: ${{ !cancelled() && needs.system.result == 'success' && (needs.sign.result == 'success' || needs.sign.result == 'skipped') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    steps:
      # The whole history and its tags, for the notes' changes since the last release.
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
        with:
          fetch-depth: 0
      # The same packages as the parts; this job boots nothing.
      - uses: ./.github/actions/prepare
        with:
          vm: 'true'

      # The media, for the export and the hashes the report names.
      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: work-after-media
          path: /mnt/kryptik

      - name: Unpack the work tree
        run: |
          cd /mnt/kryptik
          sudo tar --zstd -xf work-after-media.tar.zst -C work
          rm -f work-after-media.tar.zst

      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          pattern: acceptance-part-*
          path: /mnt/kryptik/parts

      - name: make acceptance, merged
        id: acceptance
        continue-on-error: true
        run: |
          mkdir -p /mnt/kryptik/export
          sudo env KRYPTIK_WORK="$KRYPTIK_WORK" KRYPTIK_SOURCES="$KRYPTIK_SOURCES" NO_COLOR=1 TMPDIR="$TMPDIR" \
            tools/acceptance.sh --merge /mnt/kryptik/parts --export /mnt/kryptik/export

      - name: Hand the results to the runner user
        if: always()
        run: sudo chown -R "$(id -u):$(id -g)" /mnt/kryptik/export /mnt/kryptik/work/acceptance 2>/dev/null || true

      - name: Acceptance report and per-item logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: acceptance-report
          path: |
            /mnt/kryptik/work/acceptance
            /mnt/kryptik/parts
          retention-days: 14
          if-no-files-found: ignore

      # Only a run in which every suite passed publishes images.
      - name: Tested images
        if: steps.acceptance.outcome == 'success'
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: kryptik-release
          path: /mnt/kryptik/export
          retention-days: 14
          if-no-files-found: error
          compression-level: 1

      - name: The verdict is the job's
        if: steps.acceptance.outcome != 'success'
        run: |
          echo "make acceptance did not pass; see the acceptance-report artifact"
          exit 1

  # A tag's tested export, source and acceptance logs, drafted on the Releases page (docs/releases.md); the only job that writes.
  release:
    name: Draft the release
    needs: [verdict]
    if: ${{ !cancelled() && github.ref_type == 'tag' && needs.verdict.result == 'success' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 120
    permissions:
      contents: write
    steps:
      # The tag itself, for the check that it is at the tested revision.
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
        with:
          fetch-depth: 0
      - uses: ./.github/actions/prepare
        with:
          sources: fetch
          image-sources: fetch
      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: kryptik-release
          path: /mnt/kryptik/export
      - name: The corresponding source
        run: make source-bundle
      - name: Draft the release
        env:
          GH_TOKEN: ${{ github.token }}
        run: tools/release-publish.sh /mnt/kryptik/export --source-bundle "$(ls "$KRYPTIK_OUT"/source-*.tar)"