Repository navigation

Collect the tests from where they are #315

Workflow file for this run

	name: Deploy release

	on:
	push:
	tags:
	- 'v[0-9]+.[0-9]+.[0-9]+'
	branches:
	- main

	pull_request:
	branches:
	- main
	paths:
	- '.github/workflows/deploy.yml'
	- 'pyproject.toml'

	workflow_dispatch:

	concurrency:
	group: deploy-release-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	generate_stubs:
	name: Generate type stubs
	runs-on: ubuntu-latest
	env:
	CCACHE_BASEDIR: /
	CCACHE_NOHASHDIR: 1
	CCACHE_SLOPPINESS: time_macros,include_file_mtime,include_file_ctime,locale
	steps:
	- name: Checkout repository
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	submodules: true

	# Temporary: drop this and re-enable renovate once rexlib tags releases.
	- name: Move the submodule to rexlib's main
	run: git submodule update --remote external/rexlib

	- name: Set up Python
	uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
	with:
	python-version: '3.13'

	- name: Set up compiler cache
	uses: gigabit-clowns/.github/.github/composites/restore-ccache@main
	with:
	key: stubs-ubuntu-latest
	max-size: 1G

	# Nothing points at a rexlib, so this builds the one in
	# external/rexlib, the same way a wheel does.
	- name: Install with pip
	run: \|
	pip install . -v \
	-Ccmake.define.FETCHCONTENT_BASE_DIR=/tmp/fetched \
	-Ccmake.define.CMAKE_C_COMPILER_LAUNCHER=ccache \
	-Ccmake.define.CMAKE_CXX_COMPILER_LAUNCHER=ccache

	- name: Save the compiler cache
	if: github.ref == 'refs/heads/main'
	uses: gigabit-clowns/.github/.github/composites/save-ccache@main
	with:
	key: stubs-ubuntu-latest

	- name: Collect the generated stubs
	run: \|
	mkdir -p stubs
	cp "$(python -c 'import rexlib, pathlib; print(pathlib.Path(rexlib.__file__).parent)')"/_binding/*.pyi stubs/
	ls -l stubs

	- name: Upload stubs
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: type-stubs
	path: stubs/*.pyi

	build_wheels:
	needs: generate_stubs
	name: Build wheels for ${{ matrix.os }} (${{ matrix.architectures }})
	strategy:
	matrix:
	include:
	- os: ubuntu-latest
	architectures: "x86_64"
	- os: ubuntu-24.04-arm
	architectures: "aarch64"
	# 10.13 is the oldest a C++17 build supports; arm64 did not
	# exist before 11.0.
	- os: macos-latest
	architectures: "x86_64"
	deployment-target: "10.13"
	- os: macos-latest
	architectures: "arm64"
	deployment-target: "11.0"
	- os: windows-latest
	architectures: "AMD64"
	- os: windows-11-arm
	architectures: "ARM64"

	runs-on: ${{ matrix.os }}
	steps:
	- name: Checkout repository
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	submodules: true

	# Temporary: drop this and re-enable renovate once rexlib tags releases.
	- name: Move the submodule to rexlib's main
	run: git submodule update --remote external/rexlib

	- name: Download the type stubs
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
	with:
	name: type-stubs
	path: stubs

	- name: Set up compiler cache
	if: runner.os != 'Linux'
	uses: gigabit-clowns/.github/.github/composites/restore-ccache@main
	with:
	key: wheels-${{ matrix.os }}-${{ matrix.architectures }}
	max-size: 2G

	# The container's CCACHE_DIR, seen from the runner.
	- name: Cache the container's compiler cache
	if: runner.os == 'Linux'
	uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
	with:
	path: /tmp/ccache
	key: wheels-${{ matrix.os }}-${{ matrix.architectures }}-${{ github.run_id }}
	restore-keys: wheels-${{ matrix.os }}-${{ matrix.architectures }}-

	- name: Note when the container's cache was restored
	if: runner.os == 'Linux'
	shell: bash
	run: echo "CCACHE_RESTORED_AT=$(date -u +%s)" >> "$GITHUB_ENV"

	# Ninja does not locate MSVC on its own the way the Visual Studio
	# generator does; without this the configure step finds no cl.exe.
	- name: Set up MSVC environment
	if: runner.os == 'Windows'
	uses: gigabit-clowns/.github/.github/composites/setup-msvc@main

	- name: Build wheels
	uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0
	env:
	CIBW_ARCHS: "${{ matrix.architectures }}"
	# No rexlib is pointed at, so each wheel builds the one in
	# external/rexlib. That is the point: it is compiled by
	# cibuildwheel, with whatever architecture, deployment target and
	# SDK cibuildwheel chose for that wheel, rather than by a step
	# outside it that has to be told each of those separately.
	CIBW_BEFORE_ALL_LINUX: >
	command -v ccache \|\|
	dnf install -y ccache \|\|
	yum install -y ccache
	# A platform-specific value replaces the general one rather than
	# adding to it, so the shared part is repeated.
	#
	# ccache hashes the compiler's working directory and absolute
	# paths by default; every wheel builds in a fresh temporary
	# directory, so without these nothing would ever hit.
	CIBW_ENVIRONMENT: >-
	SKBUILD_CMAKE_DEFINE="FETCHCONTENT_BASE_DIR=/tmp/fetched;REXLIB_PYTHON_STUBS_DIR=stubs;CMAKE_C_COMPILER_LAUNCHER=ccache;CMAKE_CXX_COMPILER_LAUNCHER=ccache"
	CCACHE_DIR=/host/tmp/ccache
	CCACHE_BASEDIR=/
	CCACHE_NOHASHDIR=1
	CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,locale
	CCACHE_MAXSIZE=2G
	CIBW_ENVIRONMENT_MACOS: >-
	SKBUILD_CMAKE_DEFINE="FETCHCONTENT_BASE_DIR=/tmp/fetched;REXLIB_PYTHON_STUBS_DIR=stubs;CMAKE_C_COMPILER_LAUNCHER=ccache;CMAKE_CXX_COMPILER_LAUNCHER=ccache"
	MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment-target }}
	CCACHE_BASEDIR=/
	CCACHE_NOHASHDIR=1
	CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,locale
	# Only Ninja and the Makefile generators honour COMPILER_LAUNCHER;
	# the Visual Studio one ignores it, so ccache never ran here.
	CIBW_ENVIRONMENT_WINDOWS: >-
	SKBUILD_CMAKE_DEFINE="FETCHCONTENT_BASE_DIR=C:/fetched;REXLIB_PYTHON_STUBS_DIR=stubs;CMAKE_C_COMPILER_LAUNCHER=ccache;CMAKE_CXX_COMPILER_LAUNCHER=ccache"
	CMAKE_GENERATOR=Ninja
	CCACHE_BASEDIR=C:/
	CCACHE_NOHASHDIR=1
	CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,locale
	# No musllinux: NVIDIA ships no CUDA for musl, so rexlib-cuda is
	# manylinux only and an Alpine install could never reach the GPU
	# backend. torch and cupy skip it for the same reason. Alpine can
	# still build from the source distribution.
	CIBW_SKIP: >-
	cp3??t-win*
	-musllinux_
	# librexlib now ships inside the wheel, put there by this
	# project's install rules. auditwheel would otherwise re-vendor
	# it under a hash-mangled SONAME, which breaks find_package for
	# plugin authors and the sibling plugin directory lookup.
	CIBW_REPAIR_WHEEL_COMMAND_LINUX: >
	auditwheel repair -w {dest_dir} {wheel}
	--exclude "librexlib.so*"
	CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
	delocate-wheel --require-archs {delocate_archs}
	-w {dest_dir} -v {wheel}
	--ignore-missing-dependencies
	# rexlib.dll is placed beside the extension by this project's
	# own install rules; letting delvewheel vendor it would mangle
	# the name the plugin directory is resolved from.
	CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >
	delvewheel repair -w {dest_dir} -v {wheel}
	--exclude rexlib.dll

	- name: Save the compiler cache
	if: runner.os != 'Linux' && github.ref == 'refs/heads/main'
	uses: gigabit-clowns/.github/.github/composites/save-ccache@main
	with:
	key: wheels-${{ matrix.os }}-${{ matrix.architectures }}

	# ccache ran in the container, so the eviction needs one out here.
	- name: Drop what this job did not use
	if: runner.os == 'Linux' && github.ref == 'refs/heads/main'
	shell: bash
	env:
	CCACHE_DIR: /tmp/ccache
	run: \|
	command -v ccache > /dev/null \|\| sudo apt-get install -y -qq ccache
	ccache --evict-older-than "$(( $(date -u +%s) - CCACHE_RESTORED_AT ))s"

	- name: Save the container's compiler cache
	if: runner.os == 'Linux' && github.ref == 'refs/heads/main'
	uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
	with:
	path: /tmp/ccache
	key: wheels-${{ matrix.os }}-${{ matrix.architectures }}-${{ github.run_id }}

	# ccache reuses rexlib's objects between interpreters. Sound because
	# rexlib names no Python, and checked here rather than assumed.
	- name: Check every wheel carries the same rexlib
	shell: bash
	run: \|
	# macOS runners ship python3 alone.
	command -v python3 > /dev/null && py=python3 \|\| py=python
	"$py" scripts/check_wheels_agree.py wheelhouse

	- name: Upload wheels
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
	path: ./wheelhouse/*.whl

	build_sdist:
	name: Build source distribution
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	submodules: true

	# Temporary: drop this and re-enable renovate once rexlib tags releases.
	- name: Move the submodule to rexlib's main
	run: git submodule update --remote external/rexlib

	- name: Build sdist
	run: pipx run build --sdist

	- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: cibw-sdist
	path: dist/*.tar.gz

	test_sdist:
	name: Verify the source distribution
	needs: build_sdist
	runs-on: ubuntu-latest
	env:
	CCACHE_BASEDIR: /
	CCACHE_NOHASHDIR: 1
	CCACHE_SLOPPINESS: time_macros,include_file_mtime,include_file_ctime,locale
	steps:
	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
	with:
	name: cibw-sdist
	path: dist

	- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
	with:
	python-version: 3.13

	- name: Set up compiler cache
	uses: gigabit-clowns/.github/.github/composites/restore-ccache@main
	with:
	key: sdist-ubuntu-latest
	max-size: 1G

	- name: Install
	env:
	CMAKE_BUILD_PARALLEL_LEVEL: 4
	SKBUILD_CMAKE_DEFINE: FETCHCONTENT_BASE_DIR=/tmp/fetched;CMAKE_C_COMPILER_LAUNCHER=ccache;CMAKE_CXX_COMPILER_LAUNCHER=ccache
	run: pip install --verbose dist/*.tar.gz

	- name: Save the compiler cache
	if: github.ref == 'refs/heads/main'
	uses: gigabit-clowns/.github/.github/composites/save-ccache@main
	with:
	key: sdist-ubuntu-latest

	- name: Test import
	run: \|
	cd "$RUNNER_TEMP"
	python -c "import rexlib; print(rexlib.__version__, rexlib.rexlib_version)"
	python -c "import rexlib; rexlib.PluginManager().discover_plugins()"

	upload_pypi:
	name: Upload to PyPI
	needs: [build_wheels, build_sdist, test_sdist]
	if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
	runs-on: ubuntu-latest
	steps:
	- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
	with:
	pattern: cibw-*
	path: dist
	merge-multiple: true

	- name: List artifacts
	run: ls -l ./dist

	- name: Upload to PyPI # TODO enable this in the future
	if: false
	uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1

	create_developer_release:
	name: Create development release
	needs: [build_wheels, build_sdist, test_sdist]
	runs-on: ubuntu-latest
	permissions:
	contents: write
	if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
	env:
	TAG_NAME: "development"
	steps:
	- name: Checkout repository
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

	- name: Download artifacs
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
	with:
	pattern: "cibw-*"
	path: dist
	merge-multiple: true

	- name: Create or update development tag
	run: \|
	git config --global user.name 'GitHub Actions'
	git config --global user.email 'action@github.com'
	git add dist
	git commit -m "Add pre-built binaries"
	git tag -fa $TAG_NAME -m 'Update development tag with main branch'
	git push origin $TAG_NAME --force

	- name: Delete all old assets from development release
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	gh release view $TAG_NAME --json assets -q ".assets[].name" \| while read asset; do
	gh release delete-asset $TAG_NAME "$asset" -y
	done

	- name: Create or update development release
	uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	with:
	tag_name: ${{ env.TAG_NAME }}
	name: "Development build"
	body: >-
	This is an automated release containing binaries built with
	with the head of the main branch.
	draft: false
	prerelease: true
	files: dist/**

Navigation Menu

Repository navigation

Collect the tests from where they are #315

Workflow file

Collect the tests from where they are #315

Uh oh!

Workflow file for this run