diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..31b58e7e --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,24 @@ +changelog: + exclude: + labels: + - upstream-sync + - duplicate + - invalid + - wontfix + - question + categories: + - title: Breaking Changes + labels: + - breaking-change + - title: Features + labels: + - enhancement + - title: Bug Fixes + labels: + - bug + - title: Documentation + labels: + - documentation + - title: Other Changes + labels: + - "*" diff --git a/.github/scripts/release/constants.sh b/.github/scripts/release/constants.sh new file mode 100755 index 00000000..324a4a45 --- /dev/null +++ b/.github/scripts/release/constants.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +SERVERS=(disruption-manager dominator filegen-server fleet-manager hypervisor + image-unpacker imageserver imaginator installer mdbd subd) + +# Plain binary, no tarball. Trim this list to publish fewer clients. +CLIENTS=(ami-publisher builder-tool domtool filegen-client fs2objectcache + fsbench fsreadslow hyper-control imagetool list-cert-expirations logtool + mdb-relayd objecttool scan show-cert srpc-test subtool unpacker-tool + vm-control) + +DIST_DIR="dist" +SUPPORTED_DESTINATIONS=(github jfrog) diff --git a/.github/scripts/release/destinations.sh b/.github/scripts/release/destinations.sh new file mode 100755 index 00000000..a51c6557 --- /dev/null +++ b/.github/scripts/release/destinations.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/constants.sh" + +is_supported_destination() { + local d="$1" s + for s in "${SUPPORTED_DESTINATIONS[@]}"; do [[ "$s" == "$d" ]] && return 0; done + return 1 +} + +# destinations feeds release.yml's fromJson() matrix, so it must always be valid JSON, even empty. +resolve_destinations() { + : "${RAW_TARGETS:?set PUBLISH_TARGETS (repo var) or the publish_targets dispatch input}" + : "${VERSION:?}" + # Newline-string, not an array -- empty array expansion is unbound-variable on bash 3.2. + local raw dest matched="" parts dest_json + IFS=',' read -ra parts <<<"$RAW_TARGETS" + for raw in "${parts[@]}"; do + read -r dest <<<"$raw" + [[ -z "$dest" ]] && continue + if ! is_supported_destination "$dest"; then + gha_error "unsupported publish destination '$dest' (supported: ${SUPPORTED_DESTINATIONS[*]})" + exit 1 + fi + matched="${matched}${dest}"$'\n' + done + # Standalone assignment so set -e catches a failure here, not swallowed as gha_output's argument. + dest_json="$(printf '%s' "$matched" | sed '/^$/d' | sort -u | jq -R . | jq -sc .)" + gha_output version "$VERSION" + gha_output destinations "$dest_json" +} + +# Missing/empty here means a build silently produced nothing -- fail loudly, not crash downstream. +require_dist_populated() { + local group + for group in tarballs binaries; do + if [[ -z "$(ls -A "$DIST_DIR/$group" 2>/dev/null)" ]]; then + gha_error "dist/$group is missing or empty -- nothing to publish" + exit 1 + fi + done +} + +# Must run before compute_checksums -- a downloaded file otherwise carries no version of its own. +stamp_version() { + : "${VERSION:?}" + require_dist_populated + local f + for f in "$DIST_DIR"/tarballs/*.tar.gz; do + mv "$f" "$(dirname "$f")/$(basename "$f" .tar.gz)-${VERSION}.tar.gz" + done + for f in "$DIST_DIR"/binaries/*; do + mv "$f" "${f}-${VERSION}" + done +} + +# One combined file, not one per group -- GitHub release assets are a flat +# namespace, so a tarballs/SHA256SUMS and a binaries/SHA256SUMS would collide. +compute_checksums() { + require_dist_populated + (cd "$DIST_DIR" && sha256sum tarballs/* binaries/* | sed -E 's# (tarballs|binaries)/# #' >SHA256SUMS.tmp && mv SHA256SUMS.tmp SHA256SUMS) + cat "$DIST_DIR/SHA256SUMS" +} diff --git a/.github/scripts/release/package.sh b/.github/scripts/release/package.sh new file mode 100755 index 00000000..cf8da66a --- /dev/null +++ b/.github/scripts/release/package.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/constants.sh" + +# Catches a server added to one of SERVERS / Makefile .tarball targets but not the other. +verify_servers_list() { + local makefile_servers declared missing extra + makefile_servers="$(grep -oE '^[A-Za-z0-9_-]+\.tarball:' Makefile | sed 's/\.tarball:$//' | sort -u)" + declared="$(printf '%s\n' "${SERVERS[@]}" | sort -u)" + missing="$(comm -23 <(printf '%s\n' "$makefile_servers") <(printf '%s\n' "$declared"))" + extra="$(comm -13 <(printf '%s\n' "$makefile_servers") <(printf '%s\n' "$declared"))" + if [[ -n "$missing" || -n "$extra" ]]; then + gha_error "SERVERS in constants.sh is out of sync with Makefile .tarball targets (in Makefile but not SERVERS: ${missing:-none}; in SERVERS but no Makefile target: ${extra:-none})" + exit 1 + fi +} + +# Platform-suffixed so two build legs can merge into one dist/ without collisions. +# strict hard-fails on a missing binary instead of warning (Linux only). +collect_client_binaries() { + local src="$1" suffix="$2" strict="${3:-}" name + mkdir -p "$DIST_DIR/binaries" + for name in "${CLIENTS[@]}"; do + if [[ -f "$src/$name" ]]; then + cp -p "$src/$name" "$DIST_DIR/binaries/${name}-${suffix}" + elif [[ -n "$strict" ]]; then + gha_error "expected client binary '$name' not found in $src" + exit 1 + else + echo "::warning::expected client binary '$name' not found in $src, skipping" + fi + done +} + +build_linux() { + : "${PLATFORM:?}" + verify_servers_list + local out="/tmp/${LOGNAME:-runner}" gobin + GOPATH="$(go env GOPATH)" + export GOPATH + gobin="$GOPATH/bin" + mkdir -p "$out" "$DIST_DIR/tarballs" "$DIST_DIR/binaries" ssl + make all + # shellcheck disable=SC2046 # word splitting is the point: one make target per server + make $(printf '%s.tarball ' "${SERVERS[@]}") + cp "$out"/*.tar.gz "$DIST_DIR/tarballs/" + collect_client_binaries "$gobin" "$PLATFORM" strict + strip "$DIST_DIR"/binaries/* 2>/dev/null || true + cp lib/version/BUILD_INFO "$DIST_DIR/BUILD_INFO" + find "$DIST_DIR" -type f | sort +} + +# Servers won't compile on darwin, so only non-server cmd/ packages are built here. +build_darwin() { + : "${PLATFORM:?}" + make generate # //go:embed needs BUILD_INFO before go build touches that package + local out="/tmp/${LOGNAME:-runner}-darwin" targets=() name + mkdir -p "$out" + for name in "${CLIENTS[@]}"; do + targets+=("./cmd/$name") + done + # Trailing slash on -o: go build with multiple packages discards binaries without it. + CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -buildvcs=true -o "$out/" "${targets[@]}" + collect_client_binaries "$out" "$PLATFORM" + strip "$DIST_DIR"/binaries/* 2>/dev/null || true + cp lib/version/BUILD_INFO "$DIST_DIR/BUILD_INFO" + find "$DIST_DIR" -type f | sort +} diff --git a/.github/scripts/release/publish.sh b/.github/scripts/release/publish.sh new file mode 100755 index 00000000..8f08ab57 --- /dev/null +++ b/.github/scripts/release/publish.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/constants.sh" + +publish_github() { + : "${VERSION:?}" + local files=("$DIST_DIR"/tarballs/* "$DIST_DIR"/binaries/* "$DIST_DIR/BUILD_INFO" "$DIST_DIR/SHA256SUMS") + + if ! gh release view "$VERSION" >/dev/null 2>&1; then + gh release create "$VERSION" --title "$VERSION" --generate-notes --verify-tag --latest "${files[@]}" + return + fi + + # Release exists already -- resume by uploading only missing assets, refuse if complete. + local existing_assets missing=() f base + existing_assets="$(gh release view "$VERSION" --json assets --jq '.assets[].name')" + for f in "${files[@]}"; do + base="$(basename "$f")" + grep -qxF "$base" <<<"$existing_assets" || missing+=("$f") + done + + if [[ ${#missing[@]} -eq 0 ]]; then + echo "::notice::release $VERSION already has every expected asset -- nothing to publish" + return + fi + gh release upload "$VERSION" "${missing[@]}" +} + +publish_jfrog() { + : "${VERSION:?}" + : "${JFROG_REPO:?set the JFROG_REPO repository variable}" + local match_count + match_count="$(jf rt search "${JFROG_REPO}/dominator/${VERSION}/BUILD_INFO" | jq 'length')" + if [[ "$match_count" != "0" ]]; then + echo "::notice::artifacts already exist under ${JFROG_REPO}/dominator/${VERSION}/ -- nothing to publish" + return + fi + + local v + # --fail-no-op: jf rt upload otherwise exits 0 (success) even when the glob matches nothing. + for v in "$VERSION" latest; do + jf rt upload "$DIST_DIR/tarballs/*" "${JFROG_REPO}/dominator/${v}/tarballs/" --flat=true --fail-no-op + jf rt upload "$DIST_DIR/binaries/*" "${JFROG_REPO}/dominator/${v}/binaries/" --flat=true --fail-no-op + jf rt upload "$DIST_DIR/SHA256SUMS" "${JFROG_REPO}/dominator/${v}/" --flat=true --fail-no-op + done + # BUILD_INFO under $VERSION must be uploaded last -- it's what the existence check above trusts. + jf rt upload "$DIST_DIR/BUILD_INFO" "${JFROG_REPO}/dominator/latest/" --flat=true --fail-no-op + jf rt upload "$DIST_DIR/BUILD_INFO" "${JFROG_REPO}/dominator/${VERSION}/" --flat=true --fail-no-op +} diff --git a/.github/scripts/release/run.sh b/.github/scripts/release/run.sh new file mode 100755 index 00000000..d4eaf856 --- /dev/null +++ b/.github/scripts/release/run.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euxo pipefail # -x: full command trace in the log +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$dir/utils.sh" +source "$dir/constants.sh" +source "$dir/package.sh" +source "$dir/destinations.sh" +source "$dir/publish.sh" + +case "${1:-}" in + build-linux) build_linux ;; + build-darwin) build_darwin ;; + resolve-destinations) resolve_destinations ;; + stamp-version) stamp_version ;; + checksums) compute_checksums ;; + publish-github) publish_github ;; + publish-jfrog) publish_jfrog ;; + -h | --help | *) + echo "usage: $0 " >&2 + exit 1 + ;; +esac diff --git a/.github/scripts/release/utils.sh b/.github/scripts/release/utils.sh new file mode 100755 index 00000000..689ad888 --- /dev/null +++ b/.github/scripts/release/utils.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +gha_output() { + local key="$1" value="$2" + if [[ -z "${GITHUB_OUTPUT:-}" ]]; then + echo " [output] $key=$value" + return + fi + if [[ "$value" == *$'\n'* ]]; then + { + echo "$key<<__DELIM__" + echo "$value" + echo "__DELIM__" + } >>"$GITHUB_OUTPUT" + else + echo "$key=$value" >>"$GITHUB_OUTPUT" + fi +} + +gha_error() { + echo "::error::$1" +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..315ac0b2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,71 @@ +# Shared build job. Called by ci.yml (every PR/master push) and release.yml +# (every tag push), so both pipelines run identical build logic -- never two +# YAML copies that can silently drift apart. +# +# Linux is the gate: it's where tests run, and it builds both servers and +# clients. Darwin runs after Linux passes (never in parallel) and builds +# clients only (servers aren't cross-platform) -- its failure blocks CI and +# release alike, since a broken darwin compile is a real regression, not +# runner noise. +name: Build + +on: + workflow_call: + +jobs: + build-linux: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 # full history for git describe + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + + - name: Generate build info + run: make generate # //go:embed needs BUILD_INFO before go test/go build touch that package + + - name: Run tests + run: make test + + - name: Build & package + env: + PLATFORM: linux-amd64 + run: .github/scripts/release/run.sh build-linux + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: github.ref_type == 'tag' + with: + name: dominator-build-linux-amd64 + path: dist/ + if-no-files-found: error + + build-darwin: + needs: build-linux + runs-on: macos-15 # native arm64 -- avoids cross-compile/strip issues + timeout-minutes: 30 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 # full history for git describe + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + + - name: Build & package + env: + PLATFORM: darwin-arm64 + run: .github/scripts/release/run.sh build-darwin + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: github.ref_type == 'tag' + with: + name: dominator-build-darwin-arm64 + path: dist/ + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..17ae54fe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +# CI for Dominator. Runs the shared build+test (see build.yml) on every PR and +# master push -- nothing about releases here. +name: CI + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + uses: ./.github/workflows/build.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..6b11dcb2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,128 @@ +# Release pipeline for Dominator. Fires on version tags. `build` (see build.yml) +# is the exact same jobs ci.yml uses for every PR, so a tag never ships something +# PRs never validated. prepare/publish are release.yml-only. +# Logic lives in .github/scripts/release/; this file is orchestration only. +# +# Required variables (Settings -> Variables -> Actions): +# PUBLISH_TARGETS -- comma-separated destinations, e.g. "github,jfrog" +# JFROG_REPO -- required if publishing to jfrog +# Required secrets: JF_URL, JF_ACCESS_TOKEN (required if publishing to jfrog) +# +# A release must build reproducibly from exactly what's in the tag, so this +# pipeline never fabricates a go.work -- a committed one is used as-is (Go picks +# it up automatically), and testing against an unmerged dependency branch is a +# go.mod pseudo-version in the PR (`go get module@branch`), not CI automation -- +# see https://go.dev/ref/mod (workspaces: "CI systems should generally not be +# allowed to use the go.work file"). +name: Release + +on: + push: + tags: ['v[0-9]*.[0-9]*.[0-9]*'] + workflow_dispatch: + inputs: + publish_targets: + description: 'Comma-separated destinations (e.g. github,jfrog); blank = use PUBLISH_TARGETS repo var' + type: string + default: '' + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false # never cancel a release build mid-flight + +permissions: + contents: read + +jobs: + build: + uses: ./.github/workflows/build.yml + + prepare: + needs: build + # ref_type == 'tag' guards workflow_dispatch against an accidental branch run. + if: (vars.PUBLISH_TARGETS != '' || inputs.publish_targets != '') && github.ref_type == 'tag' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + id-token: write # both required by attest-build-provenance's Sigstore signing + attestations: write + outputs: + destinations: ${{ steps.dest.outputs.destinations }} + version: ${{ steps.dest.outputs.version }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: dominator-build-* # not dominator-* -- that also matches this job's own dominator-dist output + path: dist + merge-multiple: true # binaries are platform-suffixed; BUILD_INFO is identical on both legs + + - name: Stamp version into filenames + env: + VERSION: ${{ github.ref_name }} + run: .github/scripts/release/run.sh stamp-version + + - name: Checksums + run: .github/scripts/release/run.sh checksums + + - name: Resolve & validate destinations + id: dest + env: + RAW_TARGETS: ${{ inputs.publish_targets || vars.PUBLISH_TARGETS }} + VERSION: ${{ github.ref_name }} + run: .github/scripts/release/run.sh resolve-destinations + + # Signed once here, before the fan-out, so every destination traces to the same build. + - name: Attest build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: 'dist/**/*' # one broad glob -- combined globs can silently drop matches (#133) + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dominator-dist + path: dist/ + if-no-files-found: error + + publish: + needs: prepare + if: needs.prepare.outputs.destinations != '[]' + strategy: + fail-fast: false + matrix: + destination: ${{ fromJson(needs.prepare.outputs.destinations) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: release # waits for manual approval; add reviewers to this environment + permissions: + contents: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dominator-dist + path: dist + + - name: Publish to GitHub Release + if: matrix.destination == 'github' + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.prepare.outputs.version }} + run: .github/scripts/release/run.sh publish-github + + - name: Setup JFrog CLI + if: matrix.destination == 'jfrog' + uses: jfrog/setup-jfrog-cli@5a4fbe1e30dbb570cc62d2513421c4b9bc3c9959 # v4 + env: + JF_URL: ${{ secrets.JF_URL }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Publish to JFrog Artifactory + if: matrix.destination == 'jfrog' + env: + JFROG_REPO: ${{ vars.JFROG_REPO }} + VERSION: ${{ needs.prepare.outputs.version }} + run: .github/scripts/release/run.sh publish-jfrog