Skip to content

Commit 0ea83c1

Browse files
potiukpotiuk
committed
Support the trixie hardened images' Debian-packaged Python and test legacy images like regular ones
The Debian 13 hardened images install Python as Debian packages under /usr instead of /opt/python, so the image has to find the base Python's prefix and the system-Python guard must not mistake those packages for Debian's own Python. The legacy canary now runs the regular PROD image tests, not just a smoke check, so the legacy images are covered as well as the default ones. Generated-by: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014np1fVAxHmGaBFt3qebJ8x
1 parent b4e7582 commit 0ea83c1

6 files changed

Lines changed: 270 additions & 105 deletions

File tree

‎.github/workflows/scheduled-legacy-prod-image-canary.yml‎

Lines changed: 196 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@
1919
name: "Scheduled legacy PROD image canary"
2020
# The default PROD images are hardened and based on Debian trixie, and that is all that PR and canary
2121
# builds test. The "legacy" flavor - Python compiled from sources on Debian bookworm - is still
22-
# published for the 3.4.x line, so this checks once a week that it can still be built and passes the
23-
# PROD image checks, instead of finding out only when a release is being prepared.
22+
# published for the 3.4.x line, so once a week this builds legacy bookworm PROD images and runs the
23+
# regular PROD image tests against them, instead of finding out only when a release is being prepared.
2424
#
25-
# The image is built the way release images are: with the Airflow version released to PyPI and its
26-
# constraints, on top of the Dockerfile from the checked-out branch.
25+
# It reuses the PROD image workflows of the main CI build. The constraints those workflows normally get
26+
# from the CI image are taken from the constraints branch instead, so no CI image has to be built.
2727
on: # yamllint disable-line rule:truthy
2828
schedule:
2929
- cron: '43 2 * * 0'
@@ -34,53 +34,149 @@ concurrency:
3434
group: scheduled-legacy-prod-image-canary-${{ github.ref }}
3535
cancel-in-progress: true
3636
jobs:
37-
python-versions:
38-
name: "Python versions to test"
39-
runs-on: ["ubuntu-slim"]
37+
build-info:
38+
name: "Build info"
39+
runs-on: ["ubuntu-22.04"]
4040
if: github.repository == 'apache/airflow' || github.event_name == 'workflow_dispatch'
41-
timeout-minutes: 10
41+
timeout-minutes: 15
4242
outputs:
43-
python-versions: ${{ steps.versions.outputs.python-versions }}
44-
airflow-version: ${{ steps.versions.outputs.airflow-version }}
43+
canary-run: ${{ steps.source-run-info.outputs.canary-run }}
44+
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
45+
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
46+
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
47+
default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
48+
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
49+
disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
50+
include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
51+
java-sdk-version: ${{ steps.selective-checks.outputs.java-sdk-version }}
52+
kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
53+
providers-compatibility-tests-matrix: >
54+
${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }}
55+
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
56+
python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
57+
run-airflow-ctl-integration-tests: >-
58+
${{ steps.selective-checks.outputs.run-airflow-ctl-integration-tests }}
59+
run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }}
60+
run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }}
61+
run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }}
62+
run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }}
63+
run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }}
64+
run-remote-logging-elasticsearch-e2e-tests: >-
65+
${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
66+
run-remote-logging-opensearch-e2e-tests: >-
67+
${{ steps.selective-checks.outputs.run-remote-logging-opensearch-e2e-tests }}
68+
run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
69+
run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
70+
run-ts-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-ts-sdk-e2e-tests }}
71+
run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }}
72+
use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }}
4573
steps:
74+
- name: "Cleanup repo"
75+
shell: bash
76+
run: sudo rm -rf ${GITHUB_WORKSPACE}/*
4677
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
4778
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
4879
with:
80+
fetch-depth: 2
4981
persist-credentials: false
50-
- name: "Install uv"
51-
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
52-
with:
53-
version-file: uv.lock
54-
- name: "Get Python versions and the latest released Airflow version"
55-
id: versions
82+
- name: "Install Breeze"
83+
uses: ./.github/actions/breeze
84+
- name: "Save github context to file"
85+
# Same as in ci-amd.yml: a scheduled or dispatched run has no attacker-controlled context, and
86+
# the single-quoted heredoc disables every shell expansion of the JSON anyway.
87+
shell: bash
88+
run: | # zizmor: ignore[template-injection]
89+
cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__'
90+
${{ toJson(github) }}
91+
__GITHUB_CONTEXT_END__
92+
- name: "Get information about the Workflow"
93+
id: source-run-info
94+
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
95+
env:
96+
SKIP_BREEZE_SELF_UPGRADE_CHECK: "true"
97+
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
98+
- name: "Selective checks"
99+
id: selective-checks
100+
env:
101+
COMMIT_REF: "${{ github.sha }}"
102+
VERBOSE: "false"
103+
GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
104+
PLATFORM: "linux/amd64"
105+
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
106+
107+
constraints:
108+
name: "Constraints ${{ matrix.python-version }}"
109+
needs: [build-info]
110+
runs-on: ["ubuntu-slim"]
111+
timeout-minutes: 10
112+
strategy:
113+
matrix:
114+
python-version: ${{ fromJSON(needs.build-info.outputs.python-versions) }}
115+
env:
116+
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
117+
CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
118+
steps:
119+
- name: "Download constraints from ${{ env.CONSTRAINTS_BRANCH }}"
56120
run: |
57-
python_versions=$(uv run --project dev/breeze python -c "
58-
import json
59-
from airflow_breeze.global_constants import CURRENT_PYTHON_MAJOR_MINOR_VERSIONS
60-
print(json.dumps(CURRENT_PYTHON_MAJOR_MINOR_VERSIONS))
61-
")
62-
airflow_version=$(curl -fsSL --retry 3 https://pypi.org/pypi/apache-airflow/json \
63-
| jq -r .info.version)
64-
echo "python-versions=${python_versions}" >> "${GITHUB_OUTPUT}"
65-
echo "airflow-version=${airflow_version}" >> "${GITHUB_OUTPUT}"
66-
echo "Python versions: ${python_versions}, Airflow version: ${airflow_version}"
121+
base_url="https://raw.githubusercontent.com/apache/airflow/${CONSTRAINTS_BRANCH}"
122+
target_dir="files/constraints-${PYTHON_MAJOR_MINOR_VERSION}"
123+
mkdir -p "${target_dir}"
124+
for prefix in constraints constraints-no-providers constraints-source-providers; do
125+
file="${prefix}-${PYTHON_MAJOR_MINOR_VERSION}.txt"
126+
curl -fsSL --retry 5 "${base_url}/${file}" -o "${target_dir}/${file}"
127+
done
128+
ls -l "files/constraints-${PYTHON_MAJOR_MINOR_VERSION}"
129+
# Same artifact name and layout as generate-constraints.yml uploads, which is what the PROD image
130+
# build downloads.
131+
- name: "Upload constraint artifacts"
132+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
133+
with:
134+
name: constraints-${{ matrix.python-version }}
135+
path: ./files/constraints-${{ matrix.python-version }}/constraints-*.txt
136+
retention-days: 7
137+
if-no-files-found: error
67138

68-
legacy-prod-image:
69-
name: "Legacy bookworm PROD image ${{ matrix.python-version }}"
70-
needs: [python-versions]
139+
build-prod-images:
140+
name: "Build legacy bookworm PROD images"
141+
needs: [build-info, constraints]
142+
uses: ./.github/workflows/prod-image-build.yml
143+
permissions:
144+
contents: read
145+
packages: read
146+
with:
147+
runners: '["ubuntu-22.04"]'
148+
platform: "linux/amd64"
149+
build-type: "Legacy"
150+
debian-version: "bookworm"
151+
image-flavor: "legacy"
152+
push-image: "false"
153+
upload-image-artifact: "true"
154+
upload-package-artifact: "true"
155+
python-versions: ${{ needs.build-info.outputs.python-versions }}
156+
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
157+
branch: ${{ needs.build-info.outputs.default-branch }}
158+
use-uv: ${{ needs.build-info.outputs.use-uv }}
159+
upgrade-to-newer-dependencies: "false"
160+
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
161+
# The cache in the registry is the one of the hardened images - it would not match anyway.
162+
docker-cache: "disabled"
163+
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
164+
prod-image-build: "true"
165+
166+
verify-legacy-flavor:
167+
name: "Verify legacy bookworm flavor ${{ matrix.python-version }}"
168+
needs: [build-info, build-prod-images]
71169
runs-on: ["ubuntu-22.04"]
72-
timeout-minutes: 90
170+
timeout-minutes: 20
73171
strategy:
74172
fail-fast: false
75173
matrix:
76-
python-version: ${{ fromJSON(needs.python-versions.outputs.python-versions) }}
174+
python-version: ${{ fromJSON(needs.build-info.outputs.python-versions) }}
77175
env:
78176
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
79-
AIRFLOW_VERSION: ${{ needs.python-versions.outputs.airflow-version }}
80177
GITHUB_REPOSITORY: ${{ github.repository }}
81178
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
82179
GITHUB_USERNAME: ${{ github.actor }}
83-
VERBOSE: "true"
84180
steps:
85181
- name: "Cleanup repo"
86182
shell: bash
@@ -89,32 +185,78 @@ jobs:
89185
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
90186
with:
91187
persist-credentials: false
92-
- name: "Make /mnt writeable"
93-
run: ./scripts/ci/make_mnt_writeable.sh
94-
- name: "Move docker to /mnt"
95-
run: ./scripts/ci/move_docker_to_mnt.sh
96-
- name: "Install Breeze"
97-
uses: ./.github/actions/breeze
98-
- name: "Build legacy bookworm PROD image ${{ matrix.python-version }}"
99-
run: >
100-
breeze prod-image build
101-
--python "${PYTHON_MAJOR_MINOR_VERSION}"
102-
--image-flavor legacy
103-
--debian-version bookworm
104-
--installation-method apache-airflow
105-
--install-airflow-version "${AIRFLOW_VERSION}"
106-
--airflow-constraints-reference "constraints-${AIRFLOW_VERSION}"
107-
--docker-cache disabled
188+
- name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
189+
uses: ./.github/actions/prepare_breeze_and_image
190+
with:
191+
platform: "linux/amd64"
192+
image-type: "prod"
193+
python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
194+
use-uv: ${{ needs.build-info.outputs.use-uv }}
195+
make-mnt-writeable-and-cleanup: true
196+
# Guards against the canary silently testing the default flavor if the flavor or Debian version
197+
# stop being passed through to the build.
108198
- name: "Verify the image is the legacy bookworm flavor"
199+
env:
200+
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
109201
run: |
110-
image="ghcr.io/${GITHUB_REPOSITORY,,}/main/prod/python${PYTHON_MAJOR_MINOR_VERSION}"
202+
image="ghcr.io/${GITHUB_REPOSITORY,,}/${DEFAULT_BRANCH}/prod/python${PYTHON_MAJOR_MINOR_VERSION}"
111203
flavor=$(docker inspect "${image}" \
112204
--format '{{ index .Config.Labels "org.apache.airflow.image.flavor" }}')
113205
codename=$(docker run --rm --entrypoint /bin/bash "${image}" \
114206
-c '. /etc/os-release && echo "${VERSION_CODENAME}"')
115207
echo "Image flavor: ${flavor}, Debian: ${codename}"
116208
[[ "${flavor}" == "legacy" && "${codename}" == "bookworm" ]]
117-
- name: "Verify PROD image ${{ matrix.python-version }}"
118-
run: breeze prod-image verify --python "${PYTHON_MAJOR_MINOR_VERSION}"
119-
- name: "Test docker-compose quick start ${{ matrix.python-version }}"
120-
run: breeze testing docker-compose-tests --python "${PYTHON_MAJOR_MINOR_VERSION}"
209+
210+
additional-prod-image-tests:
211+
name: "PROD image tests"
212+
needs: [build-info, build-prod-images, constraints]
213+
uses: ./.github/workflows/additional-prod-image-tests.yml
214+
with:
215+
runners: '["ubuntu-22.04"]'
216+
platform: "linux/amd64"
217+
default-branch: ${{ needs.build-info.outputs.default-branch }}
218+
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
219+
upgrade-to-newer-dependencies: "false"
220+
docker-cache: "disabled"
221+
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
222+
default-python-version: "${{ needs.build-info.outputs.default-python-version }}"
223+
run-task-sdk-integration-tests: ${{ needs.build-info.outputs.run-task-sdk-integration-tests }}
224+
canary-run: ${{ needs.build-info.outputs.canary-run }}
225+
run-remote-logging-elasticsearch-e2e-tests: >-
226+
${{ needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }}
227+
run-remote-logging-opensearch-e2e-tests: >-
228+
${{ needs.build-info.outputs.run-remote-logging-opensearch-e2e-tests }}
229+
run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
230+
run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }}
231+
run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }}
232+
java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
233+
run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }}
234+
run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }}
235+
run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }}
236+
providers-compatibility-tests-matrix: >
237+
${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
238+
run-ts-sdk-e2e-tests: ${{ needs.build-info.outputs.run-ts-sdk-e2e-tests }}
239+
use-uv: ${{ needs.build-info.outputs.use-uv }}
240+
run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
241+
run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }}
242+
243+
tests-kubernetes:
244+
name: "Kubernetes tests"
245+
needs: [build-info, build-prod-images]
246+
uses: ./.github/workflows/k8s-tests.yml
247+
permissions:
248+
contents: read
249+
packages: read
250+
with:
251+
runners: '["ubuntu-22.04"]'
252+
platform: "linux/amd64"
253+
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
254+
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
255+
use-uv: ${{ needs.build-info.outputs.use-uv }}
256+
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
257+
kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }}
258+
java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
259+
# yamllint disable rule:line-length
260+
lang-sdk-kubernetes-combo: >-
261+
${{ needs.build-info.outputs.default-python-version }}-${{ needs.build-info.outputs.default-kubernetes-version }}
262+
# yamllint enable rule:line-length

‎Dockerfile‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,9 @@ AIRFLOW_IMAGE_FLAVOR=${AIRFLOW_IMAGE_FLAVOR:-hardened}
128128
AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION:-3.13.16}
129129
COSIGN_VERSION=${COSIGN_VERSION:-3.0.5}
130130
if [[ "${AIRFLOW_IMAGE_FLAVOR}" == "hardened" ]]; then
131-
# The hardened Python base images ship Python under /opt/python.
132-
PYTHON_HOME=${PYTHON_HOME:-/opt/python}
131+
# The Debian 12 hardened images ship Python under /opt/python, the Debian 13 ones install it as
132+
# Debian packages under /usr - so ask the base image's Python where it lives.
133+
PYTHON_HOME=${PYTHON_HOME:-$(python3 -c 'import sys; print(sys.base_prefix)')}
133134
elif [[ "${AIRFLOW_IMAGE_FLAVOR}" == "legacy" ]]; then
134135
PYTHON_HOME=${PYTHON_HOME:-/usr/python}
135136
else
@@ -351,8 +352,8 @@ function install_additional_dev_dependencies() {
351352

352353
function link_python() {
353354
# Airflow images have always exposed Python under /usr/python - documentation, volume mounts and
354-
# user customizations refer to that path - while the hardened base images ship it in /opt/python,
355-
# so keep the historical location working as a symlink.
355+
# user customizations refer to that path - while the hardened base images ship it in /opt/python
356+
# (Debian 12) or /usr (Debian 13), so keep the historical location working as a symlink.
356357
if [[ ! -e /usr/python ]]; then
357358
ln -sv "${PYTHON_HOME}" /usr/python
358359
fi
@@ -376,15 +377,19 @@ function link_python() {
376377
fi
377378
done
378379
done
379-
for dst in /usr/python/lib/*
380-
do
381-
src="/usr/local/lib/$(basename "${dst}")"
382-
if [[ -e "${src}" ]]; then
383-
rm -rf "${src}"
384-
fi
385-
echo "Linking ${dst} to ${src}"
386-
ln -sv "${dst}" "${src}"
387-
done
380+
# A Python installed under /usr already has its libraries where the dynamic linker looks, and
381+
# linking all of /usr/lib into /usr/local/lib would only shadow the system libraries.
382+
if [[ "$(readlink -f "${PYTHON_HOME}")" != "/usr" ]]; then
383+
for dst in /usr/python/lib/*
384+
do
385+
src="/usr/local/lib/$(basename "${dst}")"
386+
if [[ -e "${src}" ]]; then
387+
rm -rf "${src}"
388+
fi
389+
echo "Linking ${dst} to ${src}"
390+
ln -sv "${dst}" "${src}"
391+
done
392+
fi
388393
ldconfig
389394
}
390395

@@ -417,18 +422,20 @@ function compile_python_stdlib() {
417422
}
418423

419424
function check_no_system_python() {
420-
# Python comes from the hardened base image (in /opt/python) and must stay the only Python in the
425+
# Python - from the hardened base image or compiled from sources - must stay the only Python in the
421426
# image. A system Python pulled in as a dependency of an apt package shares its shared libraries
422427
# with ours and leads to errors such as:
423428
# /usr/python/lib/python3.11/lib-dynload/_ssl.cpython-311-aarch64-linux-gnu.so: undefined symbol: _PyModule_Add
424-
if dpkg -l | grep '^ii' | grep '^ii libpython' >/dev/null; then
429+
# Debian names its Python libraries "libpython3.13". The trixie hardened images register their own
430+
# Python with dpkg as "libpython-3.13", which is the Python we want, so it must not match.
431+
if dpkg -l | grep -E '^ii libpython3\.[0-9]+' >/dev/null; then
425432
echo
426433
echo "ERROR! System python is installed by one of the previous steps"
427434
echo
428435
echo "Please make sure that no python packages are installed by default. Displaying the reason why libpython is installed:"
429436
echo
430437
apt-get install -yqq aptitude >/dev/null
431-
aptitude why "$(dpkg -l | grep '^ii libpython' | head -1 | awk '{print $2}')"
438+
aptitude why "$(dpkg -l | grep -E '^ii libpython3\.[0-9]+' | head -1 | awk '{print $2}')"
432439
echo
433440
exit 1
434441
else

0 commit comments

Comments
 (0)