Skip to content

Commit 3d494c3

Browse files
uranusjruranusjr
committed
Verify Java SDK artifact integrity continuously
This allows us to be more confident about things *before* actually trying to do an RC.
1 parent f0953fb commit 3d494c3

3 files changed

Lines changed: 321 additions & 4 deletions

File tree

‎.github/workflows/java-sdk-release-verify.yml‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# Runs the Java SDK reviewer checklist ("Verifying a release" in
19+
# java-sdk/README.md) automatically, minus the GPG signature (only the release
20+
# manager's key can satisfy that).
21+
#
22+
# * on a pull request touching java-sdk/ -> verify HEAD can produce a source
23+
# release that is self-contained and builds from scratch.
24+
# * on a `java-sdk/**` tag push -> the same check against the tagged commit
25+
# (release readiness gate).
26+
# * on manual dispatch -> verify the ACTUAL staged artifacts: the source
27+
# package in dist/dev and the closed Nexus staging repo. Run this after
28+
# staging, before sending the [VOTE] email.
29+
---
30+
name: Java SDK release verification
31+
32+
on: # yamllint disable-line rule:truthy
33+
pull_request:
34+
paths:
35+
- "java-sdk/**"
36+
- ".github/workflows/java-sdk-release-verify.yml"
37+
push:
38+
tags:
39+
- "java-sdk/**"
40+
workflow_dispatch:
41+
inputs:
42+
tag:
43+
description: "Java SDK release tag (starts with 'java-sdk/') to verify"
44+
required: true
45+
type: string
46+
nexus_repo_id:
47+
description: "Nexus staging repo id (the NNNN in orgapacheairflow-NNNN)"
48+
required: true
49+
type: string
50+
51+
permissions:
52+
contents: read
53+
54+
concurrency:
55+
group: java-sdk-release-verify-${{ github.event_name }}-${{ github.ref }}-${{ inputs.tag }}
56+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
57+
58+
env:
59+
JAVA_VERSION: "11"
60+
61+
jobs:
62+
source-release:
63+
name: Source release builds cleanly
64+
if: github.event_name != 'workflow_dispatch'
65+
runs-on: ubuntu-latest
66+
steps:
67+
- name: Checkout
68+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
69+
with:
70+
ref: ${{ github.event_name == 'push' && github.ref || github.sha }}
71+
fetch-depth: 0
72+
persist-credentials: false
73+
- name: Set up JDK ${{ env.JAVA_VERSION }}
74+
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
75+
with:
76+
distribution: temurin
77+
java-version: ${{ env.JAVA_VERSION }}
78+
- name: Use an isolated Gradle home
79+
run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV"
80+
- name: Resolve the ref to build the source release from
81+
run: |
82+
set -euo pipefail
83+
if [ "${{ github.event_name }}" = "push" ]; then
84+
echo "RELEASE_REF=${GITHUB_REF_NAME}" >> "$GITHUB_ENV"
85+
else # A PR HEAD.
86+
echo "RELEASE_REF=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
87+
fi
88+
- name: Build the source tarball from the ref (project's wrapper)
89+
working-directory: java-sdk
90+
run: ./gradlew --no-daemon sourceTarball checksumSourceTarball -PgitRef="${RELEASE_REF}"
91+
- name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball
92+
run: |
93+
set -euo pipefail
94+
meta=$(curl -fsSL https://services.gradle.org/versions/current)
95+
ver=$(echo "$meta" | jq -r .version)
96+
url=$(echo "$meta" | jq -r .downloadUrl)
97+
sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)")
98+
curl -fsSL -o /tmp/gradle.zip "$url"
99+
echo "${sum} /tmp/gradle.zip" | sha256sum -c -
100+
unzip -q -d "$HOME/gradle" /tmp/gradle.zip
101+
echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH"
102+
- name: Verify the source tarball
103+
run: |
104+
set -euo pipefail
105+
tarball=$(ls java-sdk/build/distributions/apache-airflow-java-sdk-*-src.tar.gz)
106+
java-sdk/scripts/ci/verify-source-release.sh \
107+
--tarball "${tarball}" \
108+
--sha512 "${tarball}.sha512" \
109+
--tag "${RELEASE_REF}"
110+
111+
artifact:
112+
name: Staged artifacts pass the reviewer checklist
113+
if: github.event_name == 'workflow_dispatch'
114+
runs-on: ubuntu-latest
115+
steps:
116+
- name: Checkout tag
117+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
118+
with:
119+
ref: ${{ inputs.tag }}
120+
fetch-depth: 0
121+
persist-credentials: false
122+
- name: Set up JDK ${{ env.JAVA_VERSION }}
123+
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
124+
with:
125+
distribution: temurin
126+
java-version: ${{ env.JAVA_VERSION }}
127+
- name: Use an isolated Gradle home
128+
run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-home" >> "$GITHUB_ENV"
129+
- name: Download the staged source package from dist/dev
130+
env:
131+
TAG: ${{ inputs.tag }}
132+
run: |
133+
set -euo pipefail
134+
rcdir="${TAG#java-sdk/}" # <version>-rc<N>
135+
version="${rcdir%-rc*}" # <version>
136+
base="https://dist.apache.org/repos/dist/dev/airflow/java-sdk/${rcdir}"
137+
file="apache-airflow-java-sdk-${version}-src.tar.gz"
138+
mkdir -p staged
139+
curl -fsSL -o "staged/${file}" "${base}/${file}"
140+
curl -fsSL -o "staged/${file}.sha512" "${base}/${file}.sha512"
141+
- name: Install the latest Gradle to bootstrap a new wrapper for the source extracted from the tarball
142+
run: |
143+
set -euo pipefail
144+
meta=$(curl -fsSL https://services.gradle.org/versions/current)
145+
ver=$(echo "$meta" | jq -r .version)
146+
url=$(echo "$meta" | jq -r .downloadUrl)
147+
sum=$(curl -fsSL "$(echo "$meta" | jq -r .checksumUrl)")
148+
curl -fsSL -o /tmp/gradle.zip "$url"
149+
echo "${sum} /tmp/gradle.zip" | sha256sum -c -
150+
unzip -q -d "$HOME/gradle" /tmp/gradle.zip
151+
echo "$HOME/gradle/gradle-${ver}/bin" >> "$GITHUB_PATH"
152+
- name: Verify the staged artifacts
153+
env:
154+
TAG: ${{ inputs.tag }}
155+
NEXUS_REPO_ID: ${{ inputs.nexus_repo_id }}
156+
run: |
157+
set -euo pipefail
158+
rcdir="${TAG#java-sdk/}"; version="${rcdir%-rc*}"
159+
file="apache-airflow-java-sdk-${version}-src.tar.gz"
160+
java-sdk/scripts/ci/verify-source-release.sh \
161+
--tarball "staged/${file}" \
162+
--sha512 "staged/${file}.sha512" \
163+
--tag "${TAG}" \
164+
--nexus-repo-id "${NEXUS_REPO_ID}"

‎java-sdk/README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -389,16 +389,17 @@ manager, before sending the vote — should run against the source package in
389389

390390
3. **Diff against the git tag.** Extract the tarball and compare it with a
391391
clean checkout of the tag it claims to be built from. They should be
392-
identical except `gradlew`, `gradlew.bat`, and `gradle-wrapper.jar`. The
393-
extracted top-level directory should be `apache-airflow-java-sdk-<version>`
394-
without the `-src` suffix that only appears in the tarball's own filename:
392+
identical except for the files kept out of the source release via
393+
`.gitattributes` `export-ignore`. The extracted top-level directory should be
394+
`apache-airflow-java-sdk-<version>` without the `-src` suffix that only
395+
appears in the tarball's own filename:
395396

396397
```bash
397398
tar xzf apache-airflow-java-sdk-<VERSION>-src.tar.gz
398399
git clone --branch java-sdk/<VERSION>-rc<N> \
399400
https://github.com/apache/airflow.git tag-checkout
400401
diff -rq apache-airflow-java-sdk-<VERSION>/ tag-checkout/java-sdk/ \
401-
| grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar)$'
402+
| grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar|\.editorconfig|scripts)$'
402403
```
403404

404405
Any remaining diff output is unexpected and should block the vote.

‎java-sdk/scripts/ci/verify-source-release.sh‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
# Run the reviewer checklist from java-sdk/README.md against an Apache Airflow
20+
# Java SDK source-release tarball (except the GPG signature).
21+
#
22+
# Usage:
23+
# verify-source-release.sh --tarball <path> --tag java-sdk/<version>-rc<N> \
24+
# [--sha512 <path>] [--nexus-repo-id <NNNN>]
25+
#
26+
# Requirements:
27+
# Bash with common shell tools, Git, Gradle, JDK 11,
28+
# and (for --nexus-repo-id) network access.
29+
30+
set -euo pipefail
31+
32+
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
33+
34+
tarball="" tag="" sha512="" nexus_repo_id=""
35+
while [ "$#" -gt 0 ]; do
36+
case "$1" in
37+
--tarball) tarball="$2"; shift 2 ;;
38+
--tag) tag="$2"; shift 2 ;;
39+
--sha512) sha512="$2"; shift 2 ;;
40+
--nexus-repo-id) nexus_repo_id="$2"; shift 2 ;;
41+
*) echo "Unknown argument: $1" >&2; exit 2 ;;
42+
esac
43+
done
44+
if [ -z "$tarball" ] || [ -z "$tag" ]; then
45+
echo "ERROR: --tarball and --tag are required" >&2
46+
exit 2
47+
fi
48+
tarball="$(cd "$(dirname "$tarball")" && pwd)/$(basename "$tarball")"
49+
[ -z "$sha512" ] || sha512="$(cd "$(dirname "$sha512")" && pwd)/$(basename "$sha512")"
50+
51+
work="$(mktemp -d)"
52+
tagco="$work/tagco"
53+
cleanup() {
54+
git -C "$REPO_ROOT" worktree remove --force "$tagco" >/dev/null 2>&1 || true
55+
rm -rf "$work"
56+
}
57+
trap cleanup EXIT
58+
59+
echo "==> 1. Checksum"
60+
if [ -n "$sha512" ]; then
61+
( cd "$(dirname "$tarball")" && sha512sum -c "$sha512" )
62+
else
63+
echo " no --sha512 given; skipping"
64+
fi
65+
66+
echo "==> 2. Extract to a clean directory outside the repo"
67+
tar -xzf "$tarball" -C "$work"
68+
extracted="$(find "$work" -maxdepth 1 -type d -name 'apache-airflow-java-sdk-*' | head -n1)"
69+
[ -n "$extracted" ] || { echo "ERROR: no apache-airflow-java-sdk-* directory inside the tarball" >&2; exit 1; }
70+
echo " $extracted"
71+
72+
echo "==> 3. No compiled/binary files (ASF source releases must be source-only)"
73+
bins="$(find "$extracted" -type f -exec sh -c 'file -b "$1" | grep -qviE "text|json|xml|empty" && echo "$1"' _ {} \; || true)"
74+
if [ -n "$bins" ]; then
75+
echo "ERROR: non-source files found in the tarball:" >&2
76+
echo "$bins" >&2
77+
exit 1
78+
fi
79+
echo " OK"
80+
81+
echo "==> 4. Diff against the tagged source (only export-ignored files may differ)"
82+
git -C "$REPO_ROOT" worktree add --detach --quiet "$tagco" "$tag"
83+
diffout="$(diff -rq "$extracted" "$tagco/java-sdk" \
84+
| grep -vE ': (gradlew|gradlew\.bat|gradle-wrapper\.jar|\.editorconfig|scripts)$' || true)"
85+
if [ -n "$diffout" ]; then
86+
echo "ERROR: tarball differs from the tag beyond the export-ignored files:" >&2
87+
echo "$diffout" >&2
88+
exit 1
89+
fi
90+
echo " OK"
91+
92+
echo "==> 5. Build from scratch (regenerate the wrapper, then ./gradlew build)"
93+
props="$extracted/gradle/wrapper/gradle-wrapper.properties"
94+
gv="$(sed -n 's#^distributionUrl=.*/gradle-\(.*\)-bin\.zip#\1#p' "$props")"
95+
gsum="$(sed -n 's#^distributionSha256Sum=##p' "$props")"
96+
if [ -z "$gv" ] || [ -z "$gsum" ]; then
97+
echo "ERROR: could not read Gradle version/sha256 from $props" >&2
98+
exit 1
99+
fi
100+
(
101+
cd "$extracted"
102+
gradle --no-daemon wrapper --gradle-version "$gv" --gradle-distribution-sha256-sum "$gsum"
103+
./gradlew --no-daemon build # `build` runs `check`, which runs :bom:verifyBomCoverage
104+
)
105+
echo " OK"
106+
107+
echo "==> 6. LICENSE and NOTICE present in every built jar"
108+
missing=0
109+
while IFS= read -r jar; do
110+
for entry in META-INF/LICENSE META-INF/NOTICE; do
111+
if ! unzip -l "$jar" | grep -qE "${entry}$"; then
112+
echo "ERROR: ${entry} missing from ${jar##*/}" >&2
113+
missing=1
114+
fi
115+
done
116+
done < <(find "$extracted" -path '*/build/libs/*.jar')
117+
[ "$missing" -eq 0 ] || exit 1
118+
echo " OK"
119+
120+
if [ -n "$nexus_repo_id" ]; then
121+
echo "==> 7. Staged-binary smoke test (Nexus repo orgapacheairflow-$nexus_repo_id)"
122+
version="$(basename "$extracted" | sed 's/^apache-airflow-java-sdk-//')"
123+
smoke="$work/smoke"
124+
mkdir -p "$smoke"
125+
cat > "$smoke/settings.gradle.kts" <<'SETTINGS'
126+
rootProject.name = "airflow-sdk-nexus-smoke"
127+
SETTINGS
128+
cat > "$smoke/build.gradle.kts" <<BUILD
129+
plugins { id("java-library") }
130+
repositories {
131+
maven { url = uri("https://repository.apache.org/content/repositories/orgapacheairflow-${nexus_repo_id}/") }
132+
mavenCentral()
133+
}
134+
dependencies {
135+
implementation(platform("org.apache.airflow:airflow-sdk-bom:${version}"))
136+
implementation("org.apache.airflow:airflow-sdk")
137+
implementation("org.apache.airflow:airflow-sdk-jpl")
138+
implementation("org.apache.airflow:airflow-sdk-jul")
139+
implementation("org.apache.airflow:airflow-sdk-log4j2")
140+
implementation("org.apache.airflow:airflow-sdk-slf4j")
141+
implementation("org.apache.airflow:airflow-sdk-processor")
142+
}
143+
BUILD
144+
(
145+
cd "$smoke"
146+
# Forces resolution of every BOM-managed artifact from the staging repo.
147+
gradle --no-daemon dependencies --configuration runtimeClasspath
148+
)
149+
echo " OK"
150+
fi
151+
152+
echo "All release-verification checks passed."

0 commit comments

Comments
 (0)