diff --git a/.github/workflows/java-sdk-release-verify.yml b/.github/workflows/java-sdk-release-verify.yml new file mode 100644 index 0000000000000..9384f8746b198 --- /dev/null +++ b/.github/workflows/java-sdk-release-verify.yml @@ -0,0 +1,177 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Runs the Java SDK reviewer checklist ("Verifying a release" in +# java-sdk/README.md) automatically, minus the GPG signature (only the release +# manager's key can satisfy that). +# +# * on a pull request touching java-sdk/ -> verify HEAD can produce a source +# release that is self-contained and builds from scratch. +# * on a `java-sdk/**` tag push -> the same check against the tagged commit +# (release readiness gate). +# * on manual dispatch -> verify the ACTUAL staged artifacts: the source +# package in dist/dev and the closed Nexus staging repo. Run this after +# staging, before sending the [VOTE] email. +# +# The environment setup intentionally mirror what a human verifying the release +# would do. An (unpinned) latest Gradle is downloaded to create the Gradle +# wrapper from, and a more modern JDK is used to run things. + +--- +name: Java SDK release verification + +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - "java-sdk/**" + - ".github/workflows/java-sdk-release-verify.yml" + push: + tags: + - "java-sdk/**" + workflow_dispatch: + inputs: + tag: + description: "Java SDK release tag (starts with 'java-sdk/') to verify" + required: true + type: string + nexus_repo_id: + description: "Nexus staging repo id (the NNNN in orgapacheairflow-NNNN)" + required: true + type: string + +permissions: + contents: read + +concurrency: + group: java-sdk-release-verify-${{ github.event_name }}-${{ github.ref }}-${{ inputs.tag }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + JAVA_VERSION: "21" + +jobs: + source-release: + name: Source release builds cleanly + if: github.event_name != 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event_name == 'push' && github.ref || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + - name: Use an isolated Gradle home + run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV" + - name: Resolve the ref to build the source release from + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "push" ]; then + echo "RELEASE_REF=${GITHUB_REF_NAME}" >> "$GITHUB_ENV" + else # A PR HEAD. + echo "RELEASE_REF=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + fi + - name: Build the source tarball from the ref (project's wrapper) + working-directory: java-sdk + run: ./gradlew --no-daemon sourceTarball checksumSourceTarball -PgitRef="${RELEASE_REF}" + - name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball + run: | + set -euo pipefail + meta=$(curl -fsSL https://services.gradle.org/versions/current) + ver=$(echo "$meta" | jq -r .version) + url=$(echo "$meta" | jq -r .downloadUrl) + sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)") + curl -fsSL -o /tmp/gradle.zip "$url" + echo "${sum} /tmp/gradle.zip" | sha256sum -c - + unzip -q -d "$HOME/gradle" /tmp/gradle.zip + echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH" + - name: Verify the source tarball + run: | + set -euo pipefail + tarball=$(ls java-sdk/build/distributions/apache-airflow-java-sdk-*-src.tar.gz) + java-sdk/scripts/ci/verify-source-release.sh \ + --tarball "${tarball}" \ + --sha512 "${tarball}.sha512" \ + --tag "${RELEASE_REF}" + + artifact: + name: Staged artifacts pass the reviewer checklist + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout tag + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.tag }} + fetch-depth: 0 + persist-credentials: false + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + - name: Use an isolated Gradle home + run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV" + - name: Download the staged source package from dist/dev + env: + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + rcdir="${TAG#java-sdk/}" # -rc + version="${rcdir%-rc*}" # + base="https://dist.apache.org/repos/dist/dev/airflow/java-sdk/${rcdir}" + file="apache-airflow-java-sdk-${version}-src.tar.gz" + mkdir -p staged + curl -fsSL -o "staged/${file}" "${base}/${file}" + curl -fsSL -o "staged/${file}.sha512" "${base}/${file}.sha512" + - name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball + run: | + set -euo pipefail + meta=$(curl -fsSL https://services.gradle.org/versions/current) + ver=$(echo "$meta" | jq -r .version) + url=$(echo "$meta" | jq -r .downloadUrl) + sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)") + curl -fsSL -o /tmp/gradle.zip "$url" + echo "${sum} /tmp/gradle.zip" | sha256sum -c - + unzip -q -d "$HOME/gradle" /tmp/gradle.zip + echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH" + - name: Verify the staged source package + env: + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}" + file="apache-airflow-java-sdk-${version}-src.tar.gz" + java-sdk/scripts/ci/verify-source-release.sh \ + --tarball "staged/${file}" \ + --sha512 "staged/${file}.sha512" \ + --tag "${TAG}" + - name: Smoke-test the staged convenience binaries + env: + TAG: ${{ inputs.tag }} + NEXUS_REPO_ID: ${{ inputs.nexus_repo_id }} + run: | + set -euo pipefail + rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}" + java-sdk/scripts/ci/smoke-test-staged-binaries.sh \ + --nexus-repo-id "${NEXUS_REPO_ID}" \ + --version "${version}" diff --git a/java-sdk/.gitattributes b/java-sdk/.gitattributes index 6cd69336377c2..04a9c526a5ec5 100644 --- a/java-sdk/.gitattributes +++ b/java-sdk/.gitattributes @@ -18,3 +18,7 @@ /gradlew export-ignore /gradlew.bat export-ignore /gradle/wrapper/gradle-wrapper.jar export-ignore + +# ktlint reads .editorconfig for its formatting rules at build time, and the +# build task requires the lint to pass, so the source release must ship it. +/.editorconfig -export-ignore diff --git a/java-sdk/README.md b/java-sdk/README.md index 01c68139a92be..18c295607254d 100644 --- a/java-sdk/README.md +++ b/java-sdk/README.md @@ -389,16 +389,17 @@ manager, before sending the vote — should run against the source package in 3. **Diff against the git tag.** Extract the tarball and compare it with a clean checkout of the tag it claims to be built from. They should be - identical except `gradlew`, `gradlew.bat`, and `gradle-wrapper.jar`. The - extracted top-level directory should be `apache-airflow-java-sdk-` - without the `-src` suffix that only appears in the tarball's own filename: + identical except for the files kept out of the source release via + `.gitattributes` `export-ignore`. The extracted top-level directory should be + `apache-airflow-java-sdk-` without the `-src` suffix that only + appears in the tarball's own filename: ```bash tar xzf apache-airflow-java-sdk--src.tar.gz git clone --branch java-sdk/-rc \ https://github.com/apache/airflow.git tag-checkout diff -rq apache-airflow-java-sdk-/ tag-checkout/java-sdk/ \ - | grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar)$' + | grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar|scripts)$' ``` Any remaining diff output is unexpected and should block the vote. diff --git a/java-sdk/scripts/ci/smoke-test-staged-binaries.sh b/java-sdk/scripts/ci/smoke-test-staged-binaries.sh new file mode 100755 index 0000000000000..b6198f0662830 --- /dev/null +++ b/java-sdk/scripts/ci/smoke-test-staged-binaries.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Smoke-test the STAGED convenience binaries: resolve the airflow-sdk-bom and +# every artifact it manages from the ASF Nexus staging repository. This is only +# meaningful for a staged release candidate, so it lives apart from +# verify-source-release.sh (which checks the source package and also runs on PRs +# and tag pushes, where nothing is staged). +# +# Usage: +# smoke-test-staged-binaries.sh --nexus-repo-id --version +# +# Requirements: +# Bash with common shell tools, Gradle, a JDK, and network access to +# repository.apache.org and Maven Central. + +set -euo pipefail + +nexus_repo_id="" version="" +while [ "$#" -gt 0 ]; do + case "$1" in + --nexus-repo-id) nexus_repo_id="$2"; shift 2 ;; + --version) version="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done +if [ -z "$nexus_repo_id" ] || [ -z "$version" ]; then + echo "ERROR: --nexus-repo-id and --version are required" >&2 + exit 2 +fi + +nexus_url="https://repository.apache.org/content/repositories/orgapacheairflow-$nexus_repo_id" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +echo "==> Staged-binary smoke test (Nexus repo orgapacheairflow-$nexus_repo_id)" + +# Derive the artifact list from the published BOM rather than hard-coding it. The +# BOM is the authoritative set (kept honest against the published modules by +# :bom:verifyBomCoverage), so this never drifts as modules are added/removed. +# Fetching the BOM pom also confirms the BOM itself is staged. +bom_pom="$work/airflow-sdk-bom.pom" +curl -fsSL -o "$bom_pom" \ + "$nexus_url/org/apache/airflow/airflow-sdk-bom/$version/airflow-sdk-bom-$version.pom" +artifacts="$(sed -n '//,/<\/dependencyManagement>/p' "$bom_pom" \ + | grep -oE '[^<]+' | sed -E 's###g' | sort -u)" +if [ -z "$artifacts" ]; then + echo "ERROR: no managed artifacts found in the BOM ($bom_pom)" >&2 + exit 1 +fi +echo " BOM manages:" +while IFS= read -r a; do echo " $a"; done <<< "$artifacts" + +smoke="$work/smoke" +mkdir -p "$smoke" +echo 'rootProject.name = "airflow-sdk-nexus-smoke"' > "$smoke/settings.gradle.kts" +{ + echo 'plugins { id("java-library") }' + echo 'repositories {' + echo " maven { url = uri(\"$nexus_url/\") }" + echo ' mavenCentral()' + echo '}' + echo 'dependencies {' + echo " implementation(platform(\"org.apache.airflow:airflow-sdk-bom:$version\"))" + for a in $artifacts; do + echo " implementation(\"org.apache.airflow:$a\")" + done + echo '}' +} > "$smoke/build.gradle.kts" + +( + cd "$smoke" + # Forces resolution of every BOM-managed artifact from the staging repo. + gradle --no-daemon dependencies --configuration runtimeClasspath +) +echo " OK" +echo "Staged-binary smoke test passed." diff --git a/java-sdk/scripts/ci/verify-source-release.sh b/java-sdk/scripts/ci/verify-source-release.sh new file mode 100755 index 0000000000000..496c8e6a4aec3 --- /dev/null +++ b/java-sdk/scripts/ci/verify-source-release.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Run the reviewer checklist from java-sdk/README.md against an Apache Airflow +# Java SDK source-release tarball (except the GPG signature). +# +# Usage: +# verify-source-release.sh --tarball --tag java-sdk/-rc \ +# [--sha512 ] +# +# Requirements: +# Bash with common shell tools, Git, Gradle, JDK 11. +# +# Staged convenience binaries are checked separately by +# smoke-test-staged-binaries.sh (only meaningful for a staged release). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +tarball="" tag="" sha512="" +while [ "$#" -gt 0 ]; do + case "$1" in + --tarball) tarball="$2"; shift 2 ;; + --tag) tag="$2"; shift 2 ;; + --sha512) sha512="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done +if [ -z "$tarball" ] || [ -z "$tag" ]; then + echo "ERROR: --tarball and --tag are required" >&2 + exit 2 +fi +tarball="$(cd "$(dirname "$tarball")" && pwd)/$(basename "$tarball")" +[ -z "$sha512" ] || sha512="$(cd "$(dirname "$sha512")" && pwd)/$(basename "$sha512")" + +work="$(mktemp -d)" +tagco="$work/tagco" +cleanup() { + git -C "$REPO_ROOT" worktree remove --force "$tagco" >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT + +echo "==> 1. Checksum" +if [ -n "$sha512" ]; then + ( cd "$(dirname "$tarball")" && sha512sum -c "$sha512" ) +else + echo " no --sha512 given; skipping" +fi + +echo "==> 2. Extract to a clean directory outside the repo" +tar -xzf "$tarball" -C "$work" +extracted="$(find "$work" -maxdepth 1 -type d -name 'apache-airflow-java-sdk-*' | head -n1)" +[ -n "$extracted" ] || { echo "ERROR: no apache-airflow-java-sdk-* directory inside the tarball" >&2; exit 1; } +echo " $extracted" + +echo "==> 3. No compiled/binary files (ASF source releases must be source-only)" +bins="$(find "$extracted" -type f -exec sh -c 'file -b "$1" | grep -qviE "text|json|xml|empty" && echo "$1"' _ {} \; || true)" +if [ -n "$bins" ]; then + echo "ERROR: non-source files found in the tarball:" >&2 + echo "$bins" >&2 + exit 1 +fi +echo " OK" + +echo "==> 4. Diff against the tagged source (only export-ignored files may differ)" +git -C "$REPO_ROOT" worktree add --detach --quiet "$tagco" "$tag" +diffout="$(diff -rq "$extracted" "$tagco/java-sdk" \ + | grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar|scripts)$' || true)" +if [ -n "$diffout" ]; then + echo "ERROR: tarball differs from the tag beyond the export-ignored files:" >&2 + echo "$diffout" >&2 + exit 1 +fi +echo " OK" + +echo "==> 5. Build from scratch (regenerate the wrapper, then ./gradlew build)" +props="$extracted/gradle/wrapper/gradle-wrapper.properties" +gv="$(sed -n 's#^distributionUrl=.*/gradle-\(.*\)-bin\.zip#\1#p' "$props")" +gsum="$(sed -n 's#^distributionSha256Sum=##p' "$props")" +if [ -z "$gv" ] || [ -z "$gsum" ]; then + echo "ERROR: could not read Gradle version/sha256 from $props" >&2 + exit 1 +fi +( + cd "$extracted" + gradle --no-daemon wrapper --gradle-version "$gv" --gradle-distribution-sha256-sum "$gsum" + ./gradlew --no-daemon build # `build` runs `check`, which runs :bom:verifyBomCoverage +) +echo " OK" + +echo "==> 6. LICENSE and NOTICE present in every published jar" +repo="$work/maven-repo" +( cd "$extracted" && ./gradlew --no-daemon publish -PmavenUrl="file://$repo" -PskipSigning=true ) +missing=0 +while IFS= read -r jar; do + for entry in META-INF/LICENSE META-INF/NOTICE; do + if ! unzip -l "$jar" | grep -qE "${entry}$"; then + echo "ERROR: ${entry} missing from ${jar##*/}" >&2 + missing=1 + fi + done +done < <(find "$repo" -name '*.jar') +[ "$missing" -eq 0 ] || exit 1 +echo " OK" + +echo "All release-verification checks passed."