Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 177 additions & 0 deletions .github/workflows/java-sdk-release-verify.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Runs the Java SDK reviewer checklist ("Verifying a release" in
# java-sdk/README.md) automatically, minus the GPG signature (only the release
# manager's key can satisfy that).
#
# * on a pull request touching java-sdk/ -> verify HEAD can produce a source
# release that is self-contained and builds from scratch.
# * on a `java-sdk/**` tag push -> the same check against the tagged commit
# (release readiness gate).
# * on manual dispatch -> verify the ACTUAL staged artifacts: the source
# package in dist/dev and the closed Nexus staging repo. Run this after
# staging, before sending the [VOTE] email.
#
# The environment setup intentionally mirror what a human verifying the release
# would do. An (unpinned) latest Gradle is downloaded to create the Gradle
# wrapper from, and a more modern JDK is used to run things.

---
name: Java SDK release verification

on: # yamllint disable-line rule:truthy
pull_request:
paths:
- "java-sdk/**"
- ".github/workflows/java-sdk-release-verify.yml"
push:
tags:
- "java-sdk/**"
workflow_dispatch:
inputs:
tag:
description: "Java SDK release tag (starts with 'java-sdk/') to verify"
required: true
type: string
nexus_repo_id:
description: "Nexus staging repo id (the NNNN in orgapacheairflow-NNNN)"
required: true
type: string

permissions:
contents: read

concurrency:
group: java-sdk-release-verify-${{ github.event_name }}-${{ github.ref }}-${{ inputs.tag }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
JAVA_VERSION: "21"

jobs:
source-release:
name: Source release builds cleanly
if: github.event_name != 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event_name == 'push' && github.ref || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Set up JDK ${{ env.JAVA_VERSION }}
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
with:
distribution: temurin
java-version: ${{ env.JAVA_VERSION }}
- name: Use an isolated Gradle home
run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV"
- name: Resolve the ref to build the source release from
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "push" ]; then
echo "RELEASE_REF=${GITHUB_REF_NAME}" >> "$GITHUB_ENV"
else # A PR HEAD.
echo "RELEASE_REF=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
fi
- name: Build the source tarball from the ref (project's wrapper)
working-directory: java-sdk
run: ./gradlew --no-daemon sourceTarball checksumSourceTarball -PgitRef="${RELEASE_REF}"
- name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball
run: |
set -euo pipefail
meta=$(curl -fsSL https://services.gradle.org/versions/current)
ver=$(echo "$meta" | jq -r .version)
url=$(echo "$meta" | jq -r .downloadUrl)
sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)")
curl -fsSL -o /tmp/gradle.zip "$url"
echo "${sum} /tmp/gradle.zip" | sha256sum -c -
unzip -q -d "$HOME/gradle" /tmp/gradle.zip
echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH"
Comment thread
uranusjr marked this conversation as resolved.
- name: Verify the source tarball
run: |
set -euo pipefail
tarball=$(ls java-sdk/build/distributions/apache-airflow-java-sdk-*-src.tar.gz)
java-sdk/scripts/ci/verify-source-release.sh \
--tarball "${tarball}" \
--sha512 "${tarball}.sha512" \
--tag "${RELEASE_REF}"

artifact:
name: Staged artifacts pass the reviewer checklist
if: github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
steps:
- name: Checkout tag
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ inputs.tag }}
fetch-depth: 0
persist-credentials: false
- name: Set up JDK ${{ env.JAVA_VERSION }}
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
with:
distribution: temurin
java-version: ${{ env.JAVA_VERSION }}
- name: Use an isolated Gradle home
run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV"
- name: Download the staged source package from dist/dev
env:
TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
rcdir="${TAG#java-sdk/}" # <version>-rc<N>
version="${rcdir%-rc*}" # <version>
base="https://dist.apache.org/repos/dist/dev/airflow/java-sdk/${rcdir}"
file="apache-airflow-java-sdk-${version}-src.tar.gz"
mkdir -p staged
curl -fsSL -o "staged/${file}" "${base}/${file}"
curl -fsSL -o "staged/${file}.sha512" "${base}/${file}.sha512"
- name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball
Comment thread
uranusjr marked this conversation as resolved.
run: |
set -euo pipefail
meta=$(curl -fsSL https://services.gradle.org/versions/current)
ver=$(echo "$meta" | jq -r .version)
url=$(echo "$meta" | jq -r .downloadUrl)
sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)")
curl -fsSL -o /tmp/gradle.zip "$url"
echo "${sum} /tmp/gradle.zip" | sha256sum -c -
unzip -q -d "$HOME/gradle" /tmp/gradle.zip
echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH"
- name: Verify the staged source package
env:
TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}"
file="apache-airflow-java-sdk-${version}-src.tar.gz"
java-sdk/scripts/ci/verify-source-release.sh \
--tarball "staged/${file}" \
--sha512 "staged/${file}.sha512" \
--tag "${TAG}"
- name: Smoke-test the staged convenience binaries
env:
TAG: ${{ inputs.tag }}
NEXUS_REPO_ID: ${{ inputs.nexus_repo_id }}
run: |
set -euo pipefail
rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}"
java-sdk/scripts/ci/smoke-test-staged-binaries.sh \
--nexus-repo-id "${NEXUS_REPO_ID}" \
--version "${version}"
4 changes: 4 additions & 0 deletions java-sdk/.gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,7 @@
/gradlew export-ignore
/gradlew.bat export-ignore
/gradle/wrapper/gradle-wrapper.jar export-ignore

# ktlint reads .editorconfig for its formatting rules at build time, and the
# build task requires the lint to pass, so the source release must ship it.
/.editorconfig -export-ignore
9 changes: 5 additions & 4 deletions java-sdk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -389,16 +389,17 @@ manager, before sending the vote — should run against the source package in

3. **Diff against the git tag.** Extract the tarball and compare it with a
clean checkout of the tag it claims to be built from. They should be
identical except `gradlew`, `gradlew.bat`, and `gradle-wrapper.jar`. The
extracted top-level directory should be `apache-airflow-java-sdk-<version>`
without the `-src` suffix that only appears in the tarball's own filename:
identical except for the files kept out of the source release via
`.gitattributes` `export-ignore`. The extracted top-level directory should be
`apache-airflow-java-sdk-<version>` without the `-src` suffix that only
appears in the tarball's own filename:

```bash
tar xzf apache-airflow-java-sdk-<VERSION>-src.tar.gz
git clone --branch java-sdk/<VERSION>-rc<N> \
https://github.com/apache/airflow.git tag-checkout
diff -rq apache-airflow-java-sdk-<VERSION>/ tag-checkout/java-sdk/ \
| grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar)$'
| grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar|scripts)$'
```

Any remaining diff output is unexpected and should block the vote.
Expand Down
92 changes: 92 additions & 0 deletions java-sdk/scripts/ci/smoke-test-staged-binaries.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Smoke-test the STAGED convenience binaries: resolve the airflow-sdk-bom and
# every artifact it manages from the ASF Nexus staging repository. This is only
# meaningful for a staged release candidate, so it lives apart from
# verify-source-release.sh (which checks the source package and also runs on PRs
# and tag pushes, where nothing is staged).
#
# Usage:
# smoke-test-staged-binaries.sh --nexus-repo-id <NNNN> --version <VERSION>
#
# Requirements:
# Bash with common shell tools, Gradle, a JDK, and network access to
# repository.apache.org and Maven Central.

set -euo pipefail

nexus_repo_id="" version=""
while [ "$#" -gt 0 ]; do
case "$1" in
--nexus-repo-id) nexus_repo_id="$2"; shift 2 ;;
--version) version="$2"; shift 2 ;;
*) echo "Unknown argument: $1" >&2; exit 2 ;;
esac
done
if [ -z "$nexus_repo_id" ] || [ -z "$version" ]; then
echo "ERROR: --nexus-repo-id and --version are required" >&2
exit 2
fi

nexus_url="https://repository.apache.org/content/repositories/orgapacheairflow-$nexus_repo_id"
work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT

echo "==> Staged-binary smoke test (Nexus repo orgapacheairflow-$nexus_repo_id)"

# Derive the artifact list from the published BOM rather than hard-coding it. The
# BOM is the authoritative set (kept honest against the published modules by
# :bom:verifyBomCoverage), so this never drifts as modules are added/removed.
# Fetching the BOM pom also confirms the BOM itself is staged.
bom_pom="$work/airflow-sdk-bom.pom"
curl -fsSL -o "$bom_pom" \
"$nexus_url/org/apache/airflow/airflow-sdk-bom/$version/airflow-sdk-bom-$version.pom"
artifacts="$(sed -n '/<dependencyManagement>/,/<\/dependencyManagement>/p' "$bom_pom" \
| grep -oE '<artifactId>[^<]+</artifactId>' | sed -E 's#</?artifactId>##g' | sort -u)"
if [ -z "$artifacts" ]; then
echo "ERROR: no managed artifacts found in the BOM ($bom_pom)" >&2
exit 1
fi
echo " BOM manages:"
while IFS= read -r a; do echo " $a"; done <<< "$artifacts"

smoke="$work/smoke"
mkdir -p "$smoke"
echo 'rootProject.name = "airflow-sdk-nexus-smoke"' > "$smoke/settings.gradle.kts"
{
echo 'plugins { id("java-library") }'
echo 'repositories {'
echo " maven { url = uri(\"$nexus_url/\") }"
echo ' mavenCentral()'
echo '}'
echo 'dependencies {'
echo " implementation(platform(\"org.apache.airflow:airflow-sdk-bom:$version\"))"
for a in $artifacts; do
echo " implementation(\"org.apache.airflow:$a\")"
done
echo '}'
} > "$smoke/build.gradle.kts"

(
cd "$smoke"
# Forces resolution of every BOM-managed artifact from the staging repo.
gradle --no-daemon dependencies --configuration runtimeClasspath
)
echo " OK"
echo "Staged-binary smoke test passed."
Loading
Loading