diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 595183b..bd6fa9a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -39,10 +39,20 @@ jobs:
- name: Validate GitHub Actions syntax
run: docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:1.7.7
- rlm-windows-frozen-smoke:
+ rlm-standalone-smoke:
needs: test
- runs-on: windows-latest
- timeout-minutes: 30
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - target: darwin-arm64
+ runner: macos-14
+ - target: linux-x64
+ runner: ubuntu-latest
+ - target: win-x64
+ runner: windows-latest
steps:
- uses: actions/checkout@v7
- uses: actions/setup-python@v6
@@ -52,11 +62,17 @@ jobs:
with:
version: "0.11.29"
enable-cache: false
- - name: Build and smoke the frozen Windows RLM CLIs
+ - name: Build, extract, and smoke the frozen RLM archive
shell: bash
run: >-
python scripts/toolchain.py build --manifest manifests/rlm-tools-bsl.json
--repo-root .
- --target win-x64
- --work-dir .build/ci-rlm-tools-bsl-win-x64
- --out-dir dist/ci-rlm-tools-bsl-win-x64
+ --target "${{ matrix.target }}"
+ --work-dir ".build/ci-rlm-tools-bsl-${{ matrix.target }}"
+ --out-dir "dist/ci-rlm-tools-bsl-${{ matrix.target }}"
+ - name: Emit target provenance and checksum
+ shell: bash
+ run: |
+ set -euo pipefail
+ python -m json.tool "dist/ci-rlm-tools-bsl-${{ matrix.target }}/provenance-rlm-tools-bsl-${{ matrix.target }}.json"
+ cat "dist/ci-rlm-tools-bsl-${{ matrix.target }}/checksums-rlm-tools-bsl-${{ matrix.target }}.txt"
diff --git a/.github/workflows/release-tool.yml b/.github/workflows/release-tool.yml
index b2d1bba..99e2975 100644
--- a/.github/workflows/release-tool.yml
+++ b/.github/workflows/release-tool.yml
@@ -93,7 +93,7 @@ jobs:
build:
needs: metadata
runs-on: ${{ matrix.runner }}
- timeout-minutes: 30
+ timeout-minutes: 60
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.metadata.outputs.matrix) }}
@@ -103,7 +103,7 @@ jobs:
with:
python-version: ${{ needs.metadata.outputs.python_version }}
- uses: astral-sh/setup-uv@v7
- if: needs.metadata.outputs.builder_kind == 'python-pyinstaller'
+ if: needs.metadata.outputs.builder_kind == 'python-pyinstaller' || needs.metadata.outputs.builder_kind == 'python-nuitka-standalone'
with:
version: ${{ needs.metadata.outputs.uv_version }}
enable-cache: false
@@ -192,12 +192,18 @@ jobs:
exit 1
fi
- name: Attest native executables
+ if: needs.metadata.outputs.builder_kind != 'python-nuitka-standalone'
uses: actions/attest-build-provenance@v2
with:
subject-path: |
dist/*-darwin-arm64
dist/*-linux-x64
dist/*-win-x64.exe
+ - name: Attest standalone runtime archives
+ if: needs.metadata.outputs.builder_kind == 'python-nuitka-standalone'
+ uses: actions/attest-build-provenance@v2
+ with:
+ subject-path: dist/*.tar.gz
- uses: softprops/action-gh-release@v3
with:
tag_name: ${{ needs.metadata.outputs.release_tag }}
diff --git a/manifests/rlm-tools-bsl.json b/manifests/rlm-tools-bsl.json
index d70bbb1..2d57a3a 100644
--- a/manifests/rlm-tools-bsl.json
+++ b/manifests/rlm-tools-bsl.json
@@ -2,7 +2,7 @@
"schemaVersion": 3,
"name": "rlm-tools-bsl",
"version": "1.33.0",
- "buildRevision": 2,
+ "buildRevision": 3,
"source": {
"kind": "release",
"repository": "https://github.com/Dach-Coin/rlm-tools-bsl",
@@ -20,20 +20,13 @@
},
"patches": [],
"builder": {
- "kind": "python-pyinstaller",
+ "kind": "python-nuitka-standalone",
"pythonVersion": "3.12.10",
"uvVersion": "0.11.29",
- "pyinstallerVersion": "6.21.0",
+ "nuitkaVersion": "4.1.3",
"lockFile": "uv.lock",
- "collectAll": "rlm_tools_bsl",
+ "includePackage": "rlm_tools_bsl",
"binaries": [
- {
- "package": "rlm_tools_bsl",
- "sourceName": "rlm-tools-bsl",
- "module": "rlm_tools_bsl.server",
- "assetBase": "rlm-bsl-mcp",
- "smokeArgs": ["--help"]
- },
{
"package": "rlm_tools_bsl",
"sourceName": "rlm-bsl-index",
@@ -57,6 +50,13 @@
"expectedOutput": ["usage: rlm-bsl-index index info"]
}
]
+ },
+ {
+ "package": "rlm_tools_bsl",
+ "sourceName": "rlm-tools-bsl",
+ "module": "rlm_tools_bsl.server",
+ "assetBase": "rlm-bsl-mcp",
+ "smokeArgs": ["--help"]
}
]
},
diff --git a/scripts/toolchain.py b/scripts/toolchain.py
index bdf5f2f..dc73aed 100644
--- a/scripts/toolchain.py
+++ b/scripts/toolchain.py
@@ -18,15 +18,23 @@
WINDOWS_STDIO_POLICY,
build_python_pyinstaller,
)
+from toolchain.builders.python_nuitka_standalone import ( # noqa: E402
+ build_python_nuitka_standalone,
+)
from toolchain.manifest import ( # noqa: E402
CargoBuilderSpec,
PythonBuilderSpec,
+ PythonNuitkaStandaloneSpec,
ToolManifest,
expected_release_files,
load_manifest,
release_tag,
)
from toolchain.provenance import write_target_metadata # noqa: E402
+from toolchain.runtime_archive import ( # noqa: E402
+ materialize_runtime_archive,
+ validate_runtime_archive,
+)
from toolchain.source import ( # noqa: E402
checkout_source,
copy_license_assets,
@@ -43,10 +51,33 @@ def builder_versions(manifest: ToolManifest) -> dict[str, str]:
"uv": manifest.builder.uv_version,
"pyinstaller": manifest.builder.pyinstaller_version,
}
+ if isinstance(manifest.builder, PythonNuitkaStandaloneSpec):
+ return {
+ "python": manifest.builder.python_version,
+ "uv": manifest.builder.uv_version,
+ "nuitka": manifest.builder.nuitka_version,
+ }
raise SystemExit(f"unsupported builder: {manifest.builder}")
-def builder_identity(manifest: ToolManifest, target_key: str) -> dict:
+def builder_identity(
+ manifest: ToolManifest,
+ target_key: str,
+ observed: dict | None = None,
+) -> dict:
+ if isinstance(manifest.builder, PythonNuitkaStandaloneSpec):
+ if observed is None:
+ raise SystemExit("Nuitka builder identity must come from the build report")
+ expected = {"kind": manifest.builder.kind, **builder_versions(manifest)}
+ for key, value in expected.items():
+ if observed.get(key) != value:
+ raise SystemExit(
+ f"observed Nuitka builder {key} is {observed.get(key)}, expected {value}"
+ )
+ compiler = observed.get("compiler")
+ if not isinstance(compiler, dict) or not compiler:
+ raise SystemExit("observed Nuitka builder is missing compiler identity")
+ return observed
identity: dict = {"kind": manifest.builder.kind, **builder_versions(manifest)}
if isinstance(manifest.builder, PythonBuilderSpec) and target_key == "win-x64":
identity["stdio"] = dict(WINDOWS_STDIO_POLICY)
@@ -112,20 +143,65 @@ def validate_source(
)
-def _smoke(manifest: ToolManifest, target_key: str, assets: list[Path]) -> None:
- by_name = {path.name: path for path in assets}
+def _smoke(
+ manifest: ToolManifest,
+ target_key: str,
+ assets: list[Path],
+ *,
+ smoke_root: Path | None = None,
+ expected_builder_identity: dict | None = None,
+) -> None:
target = manifest.targets[target_key]
+ if isinstance(manifest.builder, PythonNuitkaStandaloneSpec):
+ if len(assets) != 1:
+ raise SystemExit(
+ f"Nuitka build must produce one archive, got {[path.name for path in assets]}"
+ )
+ if smoke_root is None:
+ raise SystemExit("Nuitka archive smoke requires an isolated destination")
+ expected_entrypoints = {
+ binary.asset_base: f"{binary.asset_base}{target.exe}"
+ for binary in manifest.builder.binaries
+ }
+ validated = validate_runtime_archive(
+ assets[0],
+ expected_release_tag=release_tag(manifest),
+ expected_source_ref=manifest.source.ref,
+ expected_source_commit=manifest.source.commit,
+ expected_target_key=target_key,
+ expected_target_triple=target.target_triple,
+ expected_entrypoints=expected_entrypoints,
+ )
+ if validated.manifest.builder != expected_builder_identity:
+ raise SystemExit(
+ "archive builder identity mismatch: "
+ f"{validated.manifest.builder} != {expected_builder_identity}"
+ )
+ materialized = materialize_runtime_archive(validated, smoke_root)
+ by_base = {
+ name: materialized[relative]
+ for name, relative in validated.manifest.entrypoints.items()
+ }
+ else:
+ by_name = {path.name: path for path in assets}
+ by_base = {
+ binary.asset_base: by_name[
+ f"{binary.asset_base}-{target_key}{target.exe}"
+ ]
+ for binary in manifest.builder.binaries
+ }
for binary in manifest.builder.binaries:
- name = f"{binary.asset_base}-{target_key}{target.exe}"
+ executable = by_base[binary.asset_base]
+ name = executable.name
if not binary.smoke_checks:
- subprocess.run([str(by_name[name]), *binary.smoke_args], check=True)
+ subprocess.run([str(executable), *binary.smoke_args], check=True)
continue
checks = [(binary.smoke_args, ())]
checks.extend(
(check.args, check.expected_output) for check in binary.smoke_checks
)
for args, expected_output in checks:
- command = [str(by_name[name]), *args]
+ command = [str(executable), *args]
result = subprocess.run(
command,
check=False,
@@ -168,18 +244,36 @@ def build(
prepared = _prepare(manifest, repo_root, work_dir)
if isinstance(manifest.builder, CargoBuilderSpec):
assets = build_cargo(manifest, target_key, prepared, out_dir, work_dir)
+ observed_identity = None
elif isinstance(manifest.builder, PythonBuilderSpec):
assets = build_python_pyinstaller(manifest, target_key, prepared, out_dir, work_dir)
+ observed_identity = None
+ elif isinstance(manifest.builder, PythonNuitkaStandaloneSpec):
+ result = build_python_nuitka_standalone(
+ manifest, target_key, prepared, out_dir, work_dir
+ )
+ assets = list(result.assets)
+ observed_identity = result.builder_identity
else:
raise SystemExit(f"unsupported builder: {manifest.builder}")
- _smoke(manifest, target_key, assets)
+ _smoke(
+ manifest,
+ target_key,
+ assets,
+ smoke_root=(work_dir / "smoke-runtime")
+ if isinstance(manifest.builder, PythonNuitkaStandaloneSpec)
+ else None,
+ expected_builder_identity=observed_identity,
+ )
write_target_metadata(
manifest,
target_key,
prepared,
assets,
out_dir,
- builder_identity=builder_identity(manifest, target_key),
+ builder_identity=builder_identity(
+ manifest, target_key, observed=observed_identity
+ ),
)
diff --git a/tests/test_manifest.py b/tests/test_manifest.py
index b27954b..3a5b373 100644
--- a/tests/test_manifest.py
+++ b/tests/test_manifest.py
@@ -9,6 +9,7 @@
from toolchain.manifest import (
CargoBuilderSpec,
PythonBuilderSpec,
+ PythonNuitkaStandaloneSpec,
expected_asset_names,
expected_release_files,
load_manifest,
@@ -110,6 +111,20 @@ def python_manifest() -> dict:
return data
+def python_nuitka_manifest() -> dict:
+ data = python_manifest()
+ data["builder"] = {
+ "kind": "python-nuitka-standalone",
+ "pythonVersion": "3.12.10",
+ "uvVersion": "0.11.29",
+ "nuitkaVersion": "4.1.3",
+ "lockFile": "uv.lock",
+ "includePackage": "rlm_tools_bsl",
+ "binaries": data["builder"]["binaries"],
+ }
+ return data
+
+
class ManifestTests(unittest.TestCase):
def write_manifest(self, data: dict) -> Path:
root = Path(self.enterContext(tempfile.TemporaryDirectory()))
@@ -151,6 +166,55 @@ def test_loads_python_builder(self) -> None:
self.assertEqual(manifest.builder.python_version, "3.12.10")
self.assertEqual(manifest.builder.binaries[0].module, "rlm_tools_bsl.server")
+ def test_loads_nuitka_standalone_builder_and_generates_one_archive_per_target(self) -> None:
+ manifest = load_manifest(self.write_manifest(python_nuitka_manifest()))
+
+ self.assertIsInstance(manifest.builder, PythonNuitkaStandaloneSpec)
+ self.assertEqual(manifest.builder.python_version, "3.12.10")
+ self.assertEqual(manifest.builder.uv_version, "0.11.29")
+ self.assertEqual(manifest.builder.nuitka_version, "4.1.3")
+ self.assertEqual(manifest.builder.include_package, "rlm_tools_bsl")
+ self.assertEqual(
+ expected_asset_names(manifest),
+ {
+ "rlm-tools-bsl-darwin-arm64.tar.gz",
+ "rlm-tools-bsl-linux-x64.tar.gz",
+ "rlm-tools-bsl-win-x64.tar.gz",
+ },
+ )
+
+ def test_nuitka_builder_rejects_pyinstaller_fields_and_invalid_identity(self) -> None:
+ cases: list[tuple[dict, str]] = []
+
+ missing_version = python_nuitka_manifest()
+ del missing_version["builder"]["nuitkaVersion"]
+ cases.append((missing_version, "missing fields: nuitkaVersion"))
+
+ empty_package = python_nuitka_manifest()
+ empty_package["builder"]["includePackage"] = ""
+ cases.append((empty_package, "includePackage must be a non-empty string"))
+
+ pyinstaller_field = python_nuitka_manifest()
+ pyinstaller_field["builder"]["collectAll"] = "rlm_tools_bsl"
+ cases.append((pyinstaller_field, "unknown fields: collectAll"))
+
+ duplicate_source = python_nuitka_manifest()
+ duplicate_source["builder"]["binaries"].append(
+ dict(duplicate_source["builder"]["binaries"][0], assetBase="other")
+ )
+ cases.append((duplicate_source, "sourceName values must be unique"))
+
+ duplicate_asset = python_nuitka_manifest()
+ duplicate_asset["builder"]["binaries"].append(
+ dict(duplicate_asset["builder"]["binaries"][0], sourceName="other")
+ )
+ cases.append((duplicate_asset, "assetBase values must be unique"))
+
+ for data, message in cases:
+ with self.subTest(message=message):
+ with self.assertRaisesRegex(SystemExit, message):
+ load_manifest(self.write_manifest(data))
+
def test_loads_captured_smoke_checks_with_literal_output_contracts(self) -> None:
data = python_manifest()
binary = data["builder"]["binaries"][0]
diff --git a/tests/test_provenance.py b/tests/test_provenance.py
index 3cb9464..0615ac7 100644
--- a/tests/test_provenance.py
+++ b/tests/test_provenance.py
@@ -6,7 +6,7 @@
import unittest
from pathlib import Path
-from tests.test_manifest import cargo_manifest
+from tests.test_manifest import cargo_manifest, python_nuitka_manifest
from toolchain.manifest import load_manifest
from toolchain.provenance import write_target_metadata
from toolchain.source import AppliedPatch, PreparedSource
@@ -56,6 +56,55 @@ def test_records_source_patches_builder_target_and_assets_with_lf(self) -> None:
self.assertNotIn(b"\r\n", checksums_path.read_bytes())
self.assertIn(asset.name.encode(), checksums_path.read_bytes())
+ def test_nuitka_provenance_hashes_only_the_target_archive(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ manifest_data = python_nuitka_manifest()
+ manifest_path = root / "manifest.json"
+ manifest_path.write_text(json.dumps(manifest_data), encoding="utf-8")
+ manifest = load_manifest(manifest_path)
+ source = PreparedSource(root / "source", "a" * 40, "b" * 40, ())
+ out_dir = root / "out"
+ out_dir.mkdir()
+ archive = out_dir / "rlm-tools-bsl-linux-x64.tar.gz"
+ archive.write_bytes(b"verified archive")
+ identity = {
+ "kind": "python-nuitka-standalone",
+ "python": "3.12.10",
+ "uv": "0.11.29",
+ "nuitka": "4.1.3",
+ "compiler": {
+ "cCompiler": "GCC",
+ "ccName": "gcc",
+ "compiler": "gcc",
+ },
+ }
+
+ provenance_path, checksums_path = write_target_metadata(
+ manifest,
+ "linux-x64",
+ source,
+ [archive],
+ out_dir,
+ builder_identity=identity,
+ )
+
+ provenance = json.loads(provenance_path.read_text(encoding="utf-8"))
+ self.assertEqual(provenance["builder"], identity)
+ self.assertEqual(
+ provenance["assets"],
+ [
+ {
+ "name": archive.name,
+ "sha256": hashlib.sha256(b"verified archive").hexdigest(),
+ "size": len(b"verified archive"),
+ }
+ ],
+ )
+ self.assertEqual(
+ checksums_path.read_text(encoding="utf-8"),
+ f"{hashlib.sha256(b'verified archive').hexdigest()} {archive.name}\n",
+ )
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_python_nuitka_standalone.py b/tests/test_python_nuitka_standalone.py
new file mode 100644
index 0000000..484ef07
--- /dev/null
+++ b/tests/test_python_nuitka_standalone.py
@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from tests.test_manifest import python_nuitka_manifest
+from toolchain.builders.python_nuitka_standalone import (
+ build_python_nuitka_standalone,
+ parse_nuitka_report,
+ parse_nuitka_version,
+)
+from toolchain.manifest import load_manifest, release_tag
+from toolchain.runtime_archive import validate_runtime_archive
+from toolchain.source import PreparedSource
+
+
+class PythonNuitkaStandaloneTests(unittest.TestCase):
+ def make_manifest(self, root: Path, *, target: str = "darwin-arm64"):
+ data = python_nuitka_manifest()
+ data["builder"]["binaries"] = [
+ {
+ "package": "rlm_tools_bsl",
+ "sourceName": "rlm-bsl-index",
+ "module": "rlm_tools_bsl.cli",
+ "assetBase": "rlm-bsl-index",
+ "smokeArgs": ["--help"],
+ },
+ {
+ "package": "rlm_tools_bsl",
+ "sourceName": "rlm-tools-bsl",
+ "module": "rlm_tools_bsl.server",
+ "assetBase": "rlm-bsl-mcp",
+ "smokeArgs": ["--help"],
+ },
+ ]
+ path = root / "manifest.json"
+ path.write_text(json.dumps(data), encoding="utf-8")
+ return load_manifest(path), target
+
+ def test_builds_one_archive_from_two_main_programs_and_records_compiler(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ manifest, target_key = self.make_manifest(root)
+ source_dir = root / "source"
+ source_dir.mkdir()
+ source = PreparedSource(
+ source_dir,
+ manifest.source.commit,
+ "b" * 40,
+ (),
+ )
+ out_dir = root / "out"
+ work_dir = root / "work"
+ commands: list[list[str]] = []
+
+ def fake_runner(command: list[str], *, cwd=None, env=None) -> str:
+ commands.append(command)
+ if command == ["uv", "--version"]:
+ return "uv 0.11.29 (test)"
+ if command[-1:] == ["--version"] and "nuitka" in command:
+ return "4.1.3"
+ if command[-1:] == ["--version"]:
+ return "Python 3.12.10"
+ if "--mode=standalone" in command:
+ output_arg = next(item for item in command if item.startswith("--output-dir="))
+ report_arg = next(item for item in command if item.startswith("--report="))
+ output = Path(output_arg.split("=", 1)[1])
+ report = Path(report_arg.split("=", 1)[1])
+ dist = output / "rlm-bsl-index.dist"
+ nested = dist / "package"
+ nested.mkdir(parents=True)
+ executable = dist / "rlm-bsl-index"
+ executable.write_bytes(b"compiled multidist")
+ executable.chmod(0o755)
+ (dist / "libpython3.12.dylib").write_bytes(b"python")
+ (nested / "data.json").write_bytes(b"{}")
+ report.parent.mkdir(parents=True, exist_ok=True)
+ report.write_text(
+ ''
+ ''
+ "",
+ encoding="utf-8",
+ )
+ return ""
+
+ modules = {
+ "rlm-bsl-index": ("rlm_tools_bsl.cli", "main"),
+ "rlm-tools-bsl": ("rlm_tools_bsl.server", "main"),
+ }
+ result = build_python_nuitka_standalone(
+ manifest,
+ target_key,
+ source,
+ out_dir,
+ work_dir,
+ runner=fake_runner,
+ entrypoint_resolver=lambda _python, name: modules[name],
+ )
+
+ self.assertEqual(
+ [path.name for path in result.assets],
+ ["rlm-tools-bsl-darwin-arm64.tar.gz"],
+ )
+ compile_commands = [command for command in commands if "--mode=standalone" in command]
+ self.assertEqual(len(compile_commands), 1)
+ compile_command = compile_commands[0]
+ main_args = [item for item in compile_command if item.startswith("--main=")]
+ self.assertEqual(len(main_args), 2)
+ self.assertTrue(main_args[0].endswith("/rlm-bsl-index.py"))
+ self.assertTrue(main_args[1].endswith("/rlm-bsl-mcp.py"))
+ self.assertIn("--include-package=rlm_tools_bsl", compile_command)
+ self.assertIn("--include-package-data=rlm_tools_bsl", compile_command)
+ self.assertIn("--output-filename=rlm-bsl-index", compile_command)
+ self.assertIn("--assume-yes-for-downloads", compile_command)
+ self.assertFalse(any("onefile" in item for item in compile_command))
+ self.assertFalse(any("tempdir" in item for item in compile_command))
+ self.assertEqual(
+ result.builder_identity,
+ {
+ "kind": "python-nuitka-standalone",
+ "python": "3.12.10",
+ "uv": "0.11.29",
+ "nuitka": "4.1.3",
+ "compiler": {
+ "cCompiler": "Clang",
+ "ccName": "clang",
+ "compiler": "clang",
+ },
+ },
+ )
+
+ archive = validate_runtime_archive(
+ result.assets[0],
+ expected_release_tag=release_tag(manifest),
+ expected_source_ref=manifest.source.ref,
+ expected_source_commit=manifest.source.commit,
+ expected_target_key=target_key,
+ expected_target_triple=manifest.targets[target_key].target_triple,
+ expected_entrypoints={
+ "rlm-bsl-index": "rlm-bsl-index",
+ "rlm-bsl-mcp": "rlm-bsl-mcp",
+ },
+ )
+ by_path = {item.path: item for item in archive.files}
+ self.assertEqual(
+ by_path["rlm-bsl-index"].sha256,
+ by_path["rlm-bsl-mcp"].sha256,
+ )
+ self.assertIn("libpython3.12.dylib", by_path)
+ self.assertIn("package/data.json", by_path)
+ self.assertEqual(archive.manifest.builder, result.builder_identity)
+
+ def test_windows_build_uses_exe_names_and_utf8_entrypoints(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ manifest, target_key = self.make_manifest(root, target="win-x64")
+ source_dir = root / "source"
+ source_dir.mkdir()
+ source = PreparedSource(source_dir, manifest.source.commit, "b" * 40, ())
+
+ commands: list[list[str]] = []
+
+ def fake_runner(command: list[str], *, cwd=None, env=None) -> str:
+ commands.append(command)
+ if command == ["uv", "--version"]:
+ return "uv 0.11.29"
+ if command[-1:] == ["--version"] and "nuitka" in command:
+ return "4.1.3"
+ if command[-1:] == ["--version"]:
+ return "Python 3.12.10"
+ if "--mode=standalone" in command:
+ output = Path(next(item for item in command if item.startswith("--output-dir=")).split("=", 1)[1])
+ report = Path(next(item for item in command if item.startswith("--report=")).split("=", 1)[1])
+ dist = output / "rlm-bsl-index.dist"
+ dist.mkdir(parents=True)
+ (dist / "rlm-bsl-index.exe").write_bytes(b"compiled")
+ report.parent.mkdir(parents=True, exist_ok=True)
+ report.write_text(
+ ''
+ ''
+ "",
+ encoding="utf-8",
+ )
+ return ""
+
+ result = build_python_nuitka_standalone(
+ manifest,
+ target_key,
+ source,
+ root / "out",
+ root / "work",
+ runner=fake_runner,
+ entrypoint_resolver=lambda _python, name: {
+ "rlm-bsl-index": ("rlm_tools_bsl.cli", "main"),
+ "rlm-tools-bsl": ("rlm_tools_bsl.server", "main"),
+ }[name],
+ )
+
+ compile_command = next(
+ command for command in commands if "--mode=standalone" in command
+ )
+ self.assertIn("--output-filename=rlm-bsl-index.exe", compile_command)
+
+ archive = validate_runtime_archive(
+ result.assets[0],
+ expected_release_tag=release_tag(manifest),
+ expected_source_ref=manifest.source.ref,
+ expected_source_commit=manifest.source.commit,
+ expected_target_key=target_key,
+ expected_target_triple=manifest.targets[target_key].target_triple,
+ expected_entrypoints={
+ "rlm-bsl-index": "rlm-bsl-index.exe",
+ "rlm-bsl-mcp": "rlm-bsl-mcp.exe",
+ },
+ )
+ self.assertEqual(
+ {item.path for item in archive.files},
+ {"rlm-bsl-index.exe", "rlm-bsl-mcp.exe"},
+ )
+ stubs = root / "work" / "nuitka" / "entrypoints"
+ for name in ("rlm-bsl-index.py", "rlm-bsl-mcp.py"):
+ source_text = (stubs / name).read_text(encoding="utf-8")
+ self.assertLess(
+ source_text.index("configure_utf8_stdio()"),
+ source_text.index("importlib.import_module"),
+ )
+
+ def test_rejects_wrong_versions_modules_and_incomplete_compiler_report(self) -> None:
+ self.assertEqual(parse_nuitka_version("4.1.3\nCommercial: not installed"), "4.1.3")
+ with self.assertRaisesRegex(SystemExit, "cannot parse Nuitka version"):
+ parse_nuitka_version("Nuitka unknown")
+
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ report = root / "report.xml"
+ report.write_text(
+ ''
+ ""
+ "",
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(SystemExit, "compiler identity"):
+ parse_nuitka_report(report, expected_nuitka="4.1.3")
+
+ manifest, target_key = self.make_manifest(root)
+ source_dir = root / "source"
+ source_dir.mkdir()
+ source = PreparedSource(source_dir, manifest.source.commit, "b" * 40, ())
+
+ def wrong_version(command: list[str], *, cwd=None, env=None) -> str:
+ if command == ["uv", "--version"]:
+ return "uv 0.11.29"
+ if command[-1:] == ["--version"] and "nuitka" in command:
+ return "4.1.2"
+ if command[-1:] == ["--version"]:
+ return "Python 3.12.10"
+ return ""
+
+ with self.assertRaisesRegex(SystemExit, "Nuitka 4.1.2, expected 4.1.3"):
+ build_python_nuitka_standalone(
+ manifest,
+ target_key,
+ source,
+ root / "out",
+ root / "work-version",
+ runner=wrong_version,
+ entrypoint_resolver=lambda _python, _name: ("ignored", "main"),
+ )
+
+ with self.assertRaisesRegex(
+ SystemExit,
+ "rlm-bsl-index resolves to other.module, expected rlm_tools_bsl.cli",
+ ):
+ build_python_nuitka_standalone(
+ manifest,
+ target_key,
+ source,
+ root / "out-module",
+ root / "work-module",
+ runner=lambda command, cwd=None, env=None: (
+ "uv 0.11.29"
+ if command == ["uv", "--version"]
+ else "4.1.3"
+ if command[-1:] == ["--version"] and "nuitka" in command
+ else "Python 3.12.10"
+ if command[-1:] == ["--version"]
+ else ""
+ ),
+ entrypoint_resolver=lambda _python, _name: ("other.module", "main"),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_repository_contract.py b/tests/test_repository_contract.py
index d614238..8482883 100644
--- a/tests/test_repository_contract.py
+++ b/tests/test_repository_contract.py
@@ -5,7 +5,7 @@
from toolchain.manifest import (
CargoBuilderSpec,
- PythonBuilderSpec,
+ PythonNuitkaStandaloneSpec,
expected_asset_names,
load_manifest,
release_tag,
@@ -26,7 +26,7 @@ def test_checked_in_tools_have_independent_release_identities(self) -> None:
"release",
"v1.33.0",
"3e6920cd015a61af4ba7aa1a5f1fedd8bc935549",
- "rlm-tools-bsl-v1.33.0-build.2",
+ "rlm-tools-bsl-v1.33.0-build.3",
),
"bsl-analyzer": (
"release",
@@ -84,18 +84,19 @@ def test_cargo_tools_pin_rust_and_normalize_assets(self) -> None:
},
)
- def test_rlm_pins_python_builder_and_both_entrypoints(self) -> None:
+ def test_rlm_pins_nuitka_builder_and_one_archive_per_target(self) -> None:
manifest = self.load("rlm-tools-bsl")
- self.assertIsInstance(manifest.builder, PythonBuilderSpec)
+ self.assertIsInstance(manifest.builder, PythonNuitkaStandaloneSpec)
self.assertEqual(manifest.builder.python_version, "3.12.10")
self.assertEqual(manifest.builder.uv_version, "0.11.29")
- self.assertEqual(manifest.builder.pyinstaller_version, "6.21.0")
+ self.assertEqual(manifest.builder.nuitka_version, "4.1.3")
+ self.assertEqual(manifest.builder.include_package, "rlm_tools_bsl")
self.assertEqual(
[binary.module for binary in manifest.builder.binaries],
- ["rlm_tools_bsl.server", "rlm_tools_bsl.cli"],
+ ["rlm_tools_bsl.cli", "rlm_tools_bsl.server"],
)
- self.assertEqual(len(expected_asset_names(manifest)), 6)
+ self.assertEqual(len(expected_asset_names(manifest)), 3)
def test_rlm_only_implementation_is_removed(self) -> None:
self.assertFalse((REPO_ROOT / "scripts" / "build_rlm.py").exists())
@@ -122,6 +123,8 @@ def test_one_generic_release_workflow_builds_only_on_dispatch(self) -> None:
self.assertIn("refs/tags/", text)
self.assertIn("expected_release_files", text)
self.assertIn("actions/attest-build-provenance@v2", text)
+ self.assertIn("dist/*.tar.gz", text)
+ self.assertIn("python-nuitka-standalone", text)
self.assertIn("softprops/action-gh-release@v3", text)
self.assertIn("make_latest: false", text)
@@ -135,10 +138,14 @@ def test_pull_request_ci_validates_sources_without_release_operations(self) -> N
self.assertNotIn("softprops/action-gh-release", text)
self.assertNotIn("gh release", text)
- def test_pull_request_ci_builds_and_smokes_frozen_rlm_on_windows(self) -> None:
+ def test_pull_request_ci_builds_and_smokes_frozen_rlm_archives_on_three_targets(self) -> None:
text = (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8")
- self.assertIn("rlm-windows-frozen-smoke:", text)
- self.assertIn("runs-on: windows-latest", text)
+ self.assertIn("rlm-standalone-smoke:", text)
+ self.assertIn("runs-on: ${{ matrix.runner }}", text)
+ self.assertIn("fail-fast: false", text)
+ self.assertIn("macos-14", text)
+ self.assertIn("ubuntu-latest", text)
+ self.assertIn("windows-latest", text)
self.assertIn('python-version: "3.12.10"', text)
self.assertIn("uses: astral-sh/setup-uv@v7", text)
self.assertIn('version: "0.11.29"', text)
@@ -146,7 +153,17 @@ def test_pull_request_ci_builds_and_smokes_frozen_rlm_on_windows(self) -> None:
"python scripts/toolchain.py build --manifest manifests/rlm-tools-bsl.json",
text,
)
- self.assertIn("--target win-x64", text)
+ self.assertIn('--target "${{ matrix.target }}"', text)
+ self.assertIn("Emit target provenance and checksum", text)
+ self.assertIn(
+ 'python -m json.tool "dist/ci-rlm-tools-bsl-${{ matrix.target }}/provenance-rlm-tools-bsl-${{ matrix.target }}.json"',
+ text,
+ )
+ self.assertIn(
+ 'cat "dist/ci-rlm-tools-bsl-${{ matrix.target }}/checksums-rlm-tools-bsl-${{ matrix.target }}.txt"',
+ text,
+ )
+ self.assertNotIn("softprops/action-gh-release", text)
if __name__ == "__main__":
diff --git a/tests/test_rlm_manifest.py b/tests/test_rlm_manifest.py
index c8935af..6489cc1 100644
--- a/tests/test_rlm_manifest.py
+++ b/tests/test_rlm_manifest.py
@@ -2,7 +2,7 @@
import unittest
from toolchain.manifest import (
- PythonBuilderSpec,
+ PythonNuitkaStandaloneSpec,
expected_asset_names,
expected_release_files,
load_manifest,
@@ -17,17 +17,17 @@ class RlmManifestTests(unittest.TestCase):
def test_v1_33_release_keeps_upstream_entrypoints_and_renames_assets(self) -> None:
manifest = load_manifest(REPO_ROOT / "manifests" / "rlm-tools-bsl.json")
- self.assertIsInstance(manifest.builder, PythonBuilderSpec)
+ self.assertIsInstance(manifest.builder, PythonNuitkaStandaloneSpec)
self.assertEqual(manifest.name, "rlm-tools-bsl")
self.assertEqual(manifest.version, "1.33.0")
- self.assertEqual(manifest.build_revision, 2)
+ self.assertEqual(manifest.build_revision, 3)
self.assertEqual(manifest.source.kind, "release")
self.assertEqual(manifest.source.ref, "v1.33.0")
self.assertEqual(
manifest.source.commit,
"3e6920cd015a61af4ba7aa1a5f1fedd8bc935549",
)
- self.assertEqual(release_tag(manifest), "rlm-tools-bsl-v1.33.0-build.2")
+ self.assertEqual(release_tag(manifest), "rlm-tools-bsl-v1.33.0-build.3")
self.assertEqual(manifest.patches, ())
self.assertEqual(
@@ -36,11 +36,15 @@ def test_v1_33_release_keeps_upstream_entrypoints_and_renames_assets(self) -> No
for binary in manifest.builder.binaries
],
[
- ("rlm-tools-bsl", "rlm-bsl-mcp", "rlm_tools_bsl", "rlm_tools_bsl.server"),
("rlm-bsl-index", "rlm-bsl-index", "rlm_tools_bsl", "rlm_tools_bsl.cli"),
+ ("rlm-tools-bsl", "rlm-bsl-mcp", "rlm_tools_bsl", "rlm_tools_bsl.server"),
],
)
- index = manifest.builder.binaries[1]
+ self.assertEqual(manifest.builder.python_version, "3.12.10")
+ self.assertEqual(manifest.builder.uv_version, "0.11.29")
+ self.assertEqual(manifest.builder.nuitka_version, "4.1.3")
+ self.assertEqual(manifest.builder.include_package, "rlm_tools_bsl")
+ index = manifest.builder.binaries[0]
self.assertEqual(index.smoke_args, ("--help",))
self.assertEqual(
[(check.args, check.expected_output) for check in index.smoke_checks],
@@ -62,12 +66,9 @@ def test_v1_33_release_keeps_upstream_entrypoints_and_renames_assets(self) -> No
self.assertEqual(
expected_asset_names(manifest),
{
- "rlm-bsl-mcp-darwin-arm64",
- "rlm-bsl-mcp-linux-x64",
- "rlm-bsl-mcp-win-x64.exe",
- "rlm-bsl-index-darwin-arm64",
- "rlm-bsl-index-linux-x64",
- "rlm-bsl-index-win-x64.exe",
+ "rlm-tools-bsl-darwin-arm64.tar.gz",
+ "rlm-tools-bsl-linux-x64.tar.gz",
+ "rlm-tools-bsl-win-x64.tar.gz",
},
)
self.assertEqual(
diff --git a/tests/test_runtime_archive.py b/tests/test_runtime_archive.py
new file mode 100644
index 0000000..5090d79
--- /dev/null
+++ b/tests/test_runtime_archive.py
@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import gzip
+import hashlib
+import io
+import json
+import tarfile
+import tempfile
+import unittest
+from pathlib import Path
+
+from toolchain.runtime_archive import (
+ materialize_runtime_archive,
+ validate_runtime_archive,
+ write_runtime_archive,
+)
+
+
+RELEASE_TAG = "rlm-tools-bsl-v1.33.0-build.3"
+SOURCE = {
+ "ref": "v1.33.0",
+ "commit": "3e6920cd015a61af4ba7aa1a5f1fedd8bc935549",
+ "tree": "4b321de0454d4d0998762659891374a3a1326cd0",
+ "patches": [],
+}
+TARGET = {
+ "key": "linux-x64",
+ "triple": "x86_64-unknown-linux-gnu",
+}
+ENTRYPOINTS = {
+ "rlm-bsl-index": "rlm-bsl-index",
+ "rlm-bsl-mcp": "rlm-bsl-mcp",
+}
+BUILDER = {
+ "kind": "python-nuitka-standalone",
+ "python": "3.12.10",
+ "uv": "0.11.29",
+ "nuitka": "4.1.3",
+ "compiler": {"cCompiler": "Clang", "ccName": "clang", "compiler": "clang"},
+}
+
+
+def add_member(
+ archive: tarfile.TarFile,
+ name: str,
+ payload: bytes,
+ *,
+ mode: int = 0o644,
+ type_: bytes = tarfile.REGTYPE,
+ linkname: str = "",
+) -> None:
+ info = tarfile.TarInfo(name)
+ info.size = len(payload) if type_ == tarfile.REGTYPE else 0
+ info.mode = mode
+ info.type = type_
+ info.linkname = linkname
+ archive.addfile(info, io.BytesIO(payload) if type_ == tarfile.REGTYPE else None)
+
+
+def raw_archive(path: Path, members: list[tuple[str, bytes, int, bytes, str]]) -> None:
+ with path.open("wb") as raw:
+ with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed:
+ with tarfile.open(fileobj=compressed, mode="w") as archive:
+ for name, payload, mode, type_, linkname in members:
+ add_member(
+ archive,
+ name,
+ payload,
+ mode=mode,
+ type_=type_,
+ linkname=linkname,
+ )
+
+
+def expected_manifest(binary: bytes = b"multidist") -> dict:
+ files = [
+ {
+ "path": "libpython3.12.so.1.0",
+ "sha256": hashlib.sha256(b"shared").hexdigest(),
+ "size": len(b"shared"),
+ "executable": False,
+ },
+ {
+ "path": "rlm-bsl-index",
+ "sha256": hashlib.sha256(binary).hexdigest(),
+ "size": len(binary),
+ "executable": True,
+ },
+ {
+ "path": "rlm-bsl-mcp",
+ "sha256": hashlib.sha256(binary).hexdigest(),
+ "size": len(binary),
+ "executable": True,
+ },
+ ]
+ return {
+ "schemaVersion": 1,
+ "releaseTag": RELEASE_TAG,
+ "source": SOURCE,
+ "target": TARGET,
+ "entrypoints": ENTRYPOINTS,
+ "builder": BUILDER,
+ "files": files,
+ }
+
+
+class RuntimeArchiveTests(unittest.TestCase):
+ def make_payload(self, root: Path) -> Path:
+ payload = root / "payload"
+ payload.mkdir()
+ for name, data in (
+ ("rlm-bsl-index", b"multidist"),
+ ("rlm-bsl-mcp", b"multidist"),
+ ("libpython3.12.so.1.0", b"shared"),
+ ):
+ path = payload / name
+ path.write_bytes(data)
+ path.chmod(0o755 if name.startswith("rlm-bsl-") else 0o644)
+ return payload
+
+ def validate(self, path: Path):
+ return validate_runtime_archive(
+ path,
+ expected_release_tag=RELEASE_TAG,
+ expected_source_ref=SOURCE["ref"],
+ expected_source_commit=SOURCE["commit"],
+ expected_target_key=TARGET["key"],
+ expected_target_triple=TARGET["triple"],
+ expected_entrypoints=ENTRYPOINTS,
+ )
+
+ def test_archive_is_deterministic_complete_and_materializes_exact_payload(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ payload = self.make_payload(root)
+ first = root / "first.tar.gz"
+ second = root / "second.tar.gz"
+
+ for archive in (first, second):
+ write_runtime_archive(
+ archive_path=archive,
+ payload_root=payload,
+ release_tag=RELEASE_TAG,
+ source=SOURCE,
+ target=TARGET,
+ entrypoints=ENTRYPOINTS,
+ builder=BUILDER,
+ )
+
+ self.assertEqual(first.read_bytes(), second.read_bytes())
+ with tarfile.open(first, "r:gz") as archive:
+ members = archive.getmembers()
+ self.assertEqual(
+ [member.name for member in members],
+ [
+ "manifest.json",
+ "payload/libpython3.12.so.1.0",
+ "payload/rlm-bsl-index",
+ "payload/rlm-bsl-mcp",
+ ],
+ )
+ self.assertTrue(all(member.uid == 0 and member.gid == 0 for member in members))
+ self.assertTrue(all(member.uname == "" and member.gname == "" for member in members))
+ self.assertTrue(all(member.mtime == 0 for member in members))
+ manifest = json.loads(archive.extractfile("manifest.json").read())
+ self.assertEqual(manifest, expected_manifest())
+
+ validated = self.validate(first)
+ destination = root / "materialized"
+ paths = materialize_runtime_archive(validated, destination)
+ self.assertEqual(set(paths), set(ENTRYPOINTS.values()) | {"libpython3.12.so.1.0"})
+ self.assertEqual((destination / "rlm-bsl-index").read_bytes(), b"multidist")
+ self.assertEqual((destination / "rlm-bsl-mcp").read_bytes(), b"multidist")
+ self.assertEqual((destination / "libpython3.12.so.1.0").read_bytes(), b"shared")
+ self.assertTrue((destination / "rlm-bsl-index").stat().st_mode & 0o111)
+ self.assertFalse((destination / "libpython3.12.so.1.0").stat().st_mode & 0o111)
+
+ def test_rejects_unsafe_duplicate_and_non_regular_archive_members(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ manifest = json.dumps(expected_manifest(), separators=(",", ":")).encode()
+ base = [
+ ("manifest.json", manifest, 0o644, tarfile.REGTYPE, ""),
+ ("payload/libpython3.12.so.1.0", b"shared", 0o644, tarfile.REGTYPE, ""),
+ ("payload/rlm-bsl-index", b"multidist", 0o755, tarfile.REGTYPE, ""),
+ ("payload/rlm-bsl-mcp", b"multidist", 0o755, tarfile.REGTYPE, ""),
+ ]
+ cases = {
+ "absolute": base + [("/absolute", b"x", 0o644, tarfile.REGTYPE, "")],
+ "parent": base + [("payload/../escape", b"x", 0o644, tarfile.REGTYPE, "")],
+ "backslash": base + [("payload\\escape", b"x", 0o644, tarfile.REGTYPE, "")],
+ "duplicate": base + [("payload/rlm-bsl-index", b"multidist", 0o755, tarfile.REGTYPE, "")],
+ "symlink": base + [("payload/link", b"", 0o777, tarfile.SYMTYPE, "rlm-bsl-index")],
+ "hardlink": base + [("payload/link", b"", 0o777, tarfile.LNKTYPE, "rlm-bsl-index")],
+ "fifo": base + [("payload/fifo", b"", 0o644, tarfile.FIFOTYPE, "")],
+ }
+
+ for label, members in cases.items():
+ with self.subTest(label=label):
+ archive = root / f"{label}.tar.gz"
+ raw_archive(archive, members)
+ with self.assertRaisesRegex(SystemExit, "unsafe|duplicate|ordinary"):
+ self.validate(archive)
+
+ def test_rejects_manifest_identity_file_set_and_payload_drift(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ mutations: list[tuple[str, dict, list[tuple[str, bytes, int, bytes, str]], str]] = []
+ base_members = [
+ ("payload/libpython3.12.so.1.0", b"shared", 0o644, tarfile.REGTYPE, ""),
+ ("payload/rlm-bsl-index", b"multidist", 0o755, tarfile.REGTYPE, ""),
+ ("payload/rlm-bsl-mcp", b"multidist", 0o755, tarfile.REGTYPE, ""),
+ ]
+
+ wrong_release = expected_manifest()
+ wrong_release["releaseTag"] = "other"
+ mutations.append(("release", wrong_release, base_members, "releaseTag"))
+
+ wrong_source = expected_manifest()
+ wrong_source["source"] = dict(SOURCE, commit="a" * 40)
+ mutations.append(("source", wrong_source, base_members, "source.commit"))
+
+ wrong_target = expected_manifest()
+ wrong_target["target"] = dict(TARGET, key="win-x64")
+ mutations.append(("target", wrong_target, base_members, "target.key"))
+
+ wrong_entrypoint = expected_manifest()
+ wrong_entrypoint["entrypoints"] = {"rlm-bsl-index": "rlm-bsl-index"}
+ mutations.append(("entrypoint", wrong_entrypoint, base_members, "entrypoints"))
+
+ wrong_digest = expected_manifest()
+ wrong_digest["files"][1]["sha256"] = "0" * 64
+ mutations.append(("digest", wrong_digest, base_members, "sha256"))
+
+ wrong_size = expected_manifest()
+ wrong_size["files"][1]["size"] = 99
+ mutations.append(("size", wrong_size, base_members, "size"))
+
+ missing_file = expected_manifest()
+ mutations.append(("missing", missing_file, base_members[:-1], "file set"))
+
+ extra_file = expected_manifest()
+ mutations.append(
+ (
+ "extra",
+ extra_file,
+ base_members + [("payload/extra", b"x", 0o644, tarfile.REGTYPE, "")],
+ "file set",
+ )
+ )
+
+ unequal = expected_manifest(binary=b"index")
+ unequal["files"][2] = {
+ "path": "rlm-bsl-mcp",
+ "sha256": hashlib.sha256(b"mcp").hexdigest(),
+ "size": 3,
+ "executable": True,
+ }
+ mutations.append(
+ (
+ "unequal",
+ unequal,
+ [
+ base_members[0],
+ ("payload/rlm-bsl-index", b"index", 0o755, tarfile.REGTYPE, ""),
+ ("payload/rlm-bsl-mcp", b"mcp", 0o755, tarfile.REGTYPE, ""),
+ ],
+ "byte-identical",
+ )
+ )
+
+ for label, manifest, payload_members, message in mutations:
+ with self.subTest(label=label):
+ archive = root / f"{label}.tar.gz"
+ members = [
+ (
+ "manifest.json",
+ json.dumps(manifest, separators=(",", ":")).encode(),
+ 0o644,
+ tarfile.REGTYPE,
+ "",
+ ),
+ *payload_members,
+ ]
+ raw_archive(archive, members)
+ with self.assertRaisesRegex(SystemExit, message):
+ self.validate(archive)
+
+ def test_writer_rejects_symlink_and_different_entrypoint_bytes(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ payload = self.make_payload(root)
+ (payload / "rlm-bsl-mcp").write_bytes(b"different")
+ with self.assertRaisesRegex(SystemExit, "byte-identical"):
+ write_runtime_archive(
+ archive_path=root / "different.tar.gz",
+ payload_root=payload,
+ release_tag=RELEASE_TAG,
+ source=SOURCE,
+ target=TARGET,
+ entrypoints=ENTRYPOINTS,
+ builder=BUILDER,
+ )
+
+ (payload / "rlm-bsl-mcp").unlink()
+ (payload / "rlm-bsl-mcp").symlink_to(payload / "rlm-bsl-index")
+ with self.assertRaisesRegex(SystemExit, "ordinary"):
+ write_runtime_archive(
+ archive_path=root / "symlink.tar.gz",
+ payload_root=payload,
+ release_tag=RELEASE_TAG,
+ source=SOURCE,
+ target=TARGET,
+ entrypoints=ENTRYPOINTS,
+ builder=BUILDER,
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_toolchain_cli.py b/tests/test_toolchain_cli.py
index 8a7b168..60c9305 100644
--- a/tests/test_toolchain_cli.py
+++ b/tests/test_toolchain_cli.py
@@ -11,8 +11,10 @@
from contextlib import redirect_stdout
from unittest.mock import Mock, patch
-from tests.test_manifest import cargo_manifest, python_manifest
-from toolchain.manifest import load_manifest
+from tests.test_manifest import cargo_manifest, python_manifest, python_nuitka_manifest
+from toolchain.builders.python_nuitka_standalone import NuitkaBuildResult
+from toolchain.manifest import load_manifest, release_tag
+from toolchain.runtime_archive import write_runtime_archive
from toolchain.source import PreparedSource
@@ -276,6 +278,233 @@ def test_python_builder_identity_records_only_applied_windows_stdio_policy(self)
},
)
+ def test_nuitka_build_smokes_extracted_archive_before_writing_metadata(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ data = python_nuitka_manifest()
+ data["builder"]["binaries"] = [
+ {
+ "package": "rlm_tools_bsl",
+ "sourceName": "rlm-bsl-index",
+ "module": "rlm_tools_bsl.cli",
+ "assetBase": "rlm-bsl-index",
+ "smokeArgs": ["--help"],
+ "smokeChecks": [
+ {
+ "args": ["index", "build", "--help"],
+ "expectedOutput": ["Строить неполный индекс"],
+ },
+ {
+ "args": ["index", "update", "--help"],
+ "expectedOutput": ["usage: rlm-bsl-index index update"],
+ },
+ {
+ "args": ["index", "info", "--help"],
+ "expectedOutput": ["usage: rlm-bsl-index index info"],
+ },
+ ],
+ },
+ {
+ "package": "rlm_tools_bsl",
+ "sourceName": "rlm-tools-bsl",
+ "module": "rlm_tools_bsl.server",
+ "assetBase": "rlm-bsl-mcp",
+ "smokeArgs": ["--help"],
+ },
+ ]
+ manifest_path = root / "manifest.json"
+ manifest_path.write_text(json.dumps(data), encoding="utf-8")
+ manifest = load_manifest(manifest_path)
+ source = PreparedSource(root / "source", manifest.source.commit, "b" * 40, ())
+ source.path.mkdir()
+ payload = root / "payload"
+ payload.mkdir()
+ calls = root / "calls.txt"
+ executable = (
+ "#!/usr/bin/env python3\n"
+ "from pathlib import Path\n"
+ "import sys\n"
+ f"log = Path({str(calls)!r})\n"
+ "with log.open('a', encoding='utf-8') as stream:\n"
+ " stream.write(Path(sys.argv[0]).name + ' ' + ' '.join(sys.argv[1:]) + '\\n')\n"
+ "name = Path(sys.argv[0]).name\n"
+ "args = sys.argv[1:]\n"
+ "if name == 'rlm-bsl-mcp':\n"
+ " print('usage: rlm-bsl-mcp')\n"
+ "elif args == ['--help']:\n"
+ " print('usage: rlm-bsl-index')\n"
+ "elif args == ['index', 'build', '--help']:\n"
+ " print('Строить неполный индекс')\n"
+ "elif args == ['index', 'update', '--help']:\n"
+ " print('usage: rlm-bsl-index index update')\n"
+ "elif args == ['index', 'info', '--help']:\n"
+ " print('usage: rlm-bsl-index index info')\n"
+ "else:\n"
+ " raise SystemExit(9)\n"
+ ).encode()
+ for name in ("rlm-bsl-index", "rlm-bsl-mcp"):
+ path = payload / name
+ path.write_bytes(executable)
+ path.chmod(0o755)
+ archive = root / "out" / "rlm-tools-bsl-darwin-arm64.tar.gz"
+ builder_identity = {
+ "kind": "python-nuitka-standalone",
+ "python": "3.12.10",
+ "uv": "0.11.29",
+ "nuitka": "4.1.3",
+ "compiler": {
+ "cCompiler": "Clang",
+ "ccName": "clang",
+ "compiler": "clang",
+ },
+ }
+ write_runtime_archive(
+ archive_path=archive,
+ payload_root=payload,
+ release_tag=release_tag(manifest),
+ source={
+ "ref": manifest.source.ref,
+ "commit": source.commit,
+ "tree": source.tree,
+ "patches": [],
+ },
+ target={
+ "key": "darwin-arm64",
+ "triple": manifest.targets["darwin-arm64"].target_triple,
+ },
+ entrypoints={
+ "rlm-bsl-index": "rlm-bsl-index",
+ "rlm-bsl-mcp": "rlm-bsl-mcp",
+ },
+ builder=builder_identity,
+ )
+ module = load_script()
+ metadata = Mock()
+
+ with (
+ patch.object(module, "_prepare", return_value=source),
+ patch.object(
+ module,
+ "build_python_nuitka_standalone",
+ return_value=NuitkaBuildResult((archive,), builder_identity),
+ create=True,
+ ),
+ patch.object(module, "write_target_metadata", metadata),
+ ):
+ module.build(
+ manifest,
+ root,
+ "darwin-arm64",
+ root / "work",
+ root / "out",
+ )
+
+ self.assertEqual(
+ calls.read_text(encoding="utf-8").splitlines(),
+ [
+ "rlm-bsl-index --help",
+ "rlm-bsl-index index build --help",
+ "rlm-bsl-index index update --help",
+ "rlm-bsl-index index info --help",
+ "rlm-bsl-mcp --help",
+ ],
+ )
+ metadata.assert_called_once()
+ self.assertEqual(metadata.call_args.args[3], [archive])
+ self.assertEqual(
+ metadata.call_args.kwargs["builder_identity"], builder_identity
+ )
+
+ def test_nuitka_build_rejects_corrupt_archive_before_metadata(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ data = python_nuitka_manifest()
+ data["builder"]["binaries"] = [
+ dict(data["builder"]["binaries"][0], sourceName="rlm-bsl-index", module="rlm_tools_bsl.cli", assetBase="rlm-bsl-index"),
+ dict(data["builder"]["binaries"][0], sourceName="rlm-tools-bsl", module="rlm_tools_bsl.server", assetBase="rlm-bsl-mcp"),
+ ]
+ manifest_path = root / "manifest.json"
+ manifest_path.write_text(json.dumps(data), encoding="utf-8")
+ manifest = load_manifest(manifest_path)
+ source = PreparedSource(root / "source", manifest.source.commit, "b" * 40, ())
+ archive = root / "out" / "rlm-tools-bsl-darwin-arm64.tar.gz"
+ archive.parent.mkdir()
+ archive.write_bytes(b"not a tar archive")
+ identity = {
+ "kind": "python-nuitka-standalone",
+ "python": "3.12.10",
+ "uv": "0.11.29",
+ "nuitka": "4.1.3",
+ "compiler": {"cCompiler": "Clang", "ccName": "clang", "compiler": "clang"},
+ }
+ module = load_script()
+ metadata = Mock()
+
+ with (
+ patch.object(module, "_prepare", return_value=source),
+ patch.object(
+ module,
+ "build_python_nuitka_standalone",
+ return_value=NuitkaBuildResult((archive,), identity),
+ create=True,
+ ),
+ patch.object(module, "write_target_metadata", metadata),
+ ):
+ with self.assertRaisesRegex(SystemExit, "cannot read runtime archive"):
+ module.build(manifest, root, "darwin-arm64", root / "work", root / "out")
+
+ metadata.assert_not_called()
+
+ def test_nuitka_build_rejects_archive_builder_identity_drift_before_metadata(self) -> None:
+ root = Path(self.enterContext(tempfile.TemporaryDirectory()))
+ data = python_nuitka_manifest()
+ data["builder"]["binaries"] = [
+ dict(data["builder"]["binaries"][0], sourceName="rlm-bsl-index", module="rlm_tools_bsl.cli", assetBase="rlm-bsl-index"),
+ dict(data["builder"]["binaries"][0], sourceName="rlm-tools-bsl", module="rlm_tools_bsl.server", assetBase="rlm-bsl-mcp"),
+ ]
+ manifest_path = root / "manifest.json"
+ manifest_path.write_text(json.dumps(data), encoding="utf-8")
+ manifest = load_manifest(manifest_path)
+ source = PreparedSource(root / "source", manifest.source.commit, "b" * 40, ())
+ payload = root / "payload"
+ payload.mkdir()
+ for name in ("rlm-bsl-index", "rlm-bsl-mcp"):
+ executable = payload / name
+ executable.write_bytes(b"#!/bin/sh\nexit 0\n")
+ executable.chmod(0o755)
+ archive = root / "out" / "rlm-tools-bsl-darwin-arm64.tar.gz"
+ write_runtime_archive(
+ archive_path=archive,
+ payload_root=payload,
+ release_tag=release_tag(manifest),
+ source={"ref": manifest.source.ref, "commit": source.commit, "tree": source.tree, "patches": []},
+ target={"key": "darwin-arm64", "triple": manifest.targets["darwin-arm64"].target_triple},
+ entrypoints={"rlm-bsl-index": "rlm-bsl-index", "rlm-bsl-mcp": "rlm-bsl-mcp"},
+ builder={"kind": "tampered-builder"},
+ )
+ observed = {
+ "kind": "python-nuitka-standalone",
+ "python": "3.12.10",
+ "uv": "0.11.29",
+ "nuitka": "4.1.3",
+ "compiler": {"cCompiler": "Clang", "ccName": "clang", "compiler": "clang"},
+ }
+ module = load_script()
+ metadata = Mock()
+
+ with (
+ patch.object(module, "_prepare", return_value=source),
+ patch.object(
+ module,
+ "build_python_nuitka_standalone",
+ return_value=NuitkaBuildResult((archive,), observed),
+ create=True,
+ ),
+ patch.object(module, "write_target_metadata", metadata),
+ ):
+ with self.assertRaisesRegex(SystemExit, "archive builder identity mismatch"):
+ module.build(manifest, root, "darwin-arm64", root / "work", root / "out")
+
+ metadata.assert_not_called()
+
def test_main_resolves_relative_work_and_output_paths_before_build(self) -> None:
root = Path(self.enterContext(tempfile.TemporaryDirectory()))
manifest_path = self.write_manifest(root)
diff --git a/toolchain/builders/python_nuitka_standalone.py b/toolchain/builders/python_nuitka_standalone.py
new file mode 100644
index 0000000..fc16bec
--- /dev/null
+++ b/toolchain/builders/python_nuitka_standalone.py
@@ -0,0 +1,276 @@
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import sys
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from toolchain.builders.python_pyinstaller import (
+ parse_python_version,
+ parse_uv_version,
+ resolve_entrypoint,
+ run_command,
+ write_entrypoint_stub,
+)
+from toolchain.manifest import (
+ PythonNuitkaStandaloneSpec,
+ ToolManifest,
+ release_tag,
+)
+from toolchain.runtime_archive import write_runtime_archive
+from toolchain.source import PreparedSource
+
+
+Runner = Callable[..., str]
+EntrypointResolver = Callable[[Path, str], tuple[str, str]]
+NUITKA_VERSION = re.compile(r"^([0-9]+\.[0-9]+\.[0-9]+)(?:\s|$)")
+
+
+@dataclass(frozen=True)
+class NuitkaBuildResult:
+ assets: tuple[Path, ...]
+ builder_identity: dict[str, Any]
+
+
+def parse_nuitka_version(output: str) -> str:
+ match = NUITKA_VERSION.match(output.strip())
+ if match is None:
+ raise SystemExit(f"cannot parse Nuitka version: {output.strip()}")
+ return match.group(1)
+
+
+def parse_nuitka_report(
+ path: Path,
+ *,
+ expected_nuitka: str,
+) -> dict[str, str]:
+ try:
+ root = ET.parse(path).getroot()
+ except (OSError, ET.ParseError) as exc:
+ raise SystemExit(f"cannot read Nuitka report: {exc}") from exc
+ if root.tag != "nuitka-compilation-report":
+ raise SystemExit(f"unexpected Nuitka report root: {root.tag}")
+ report_version = root.get("nuitka_version")
+ if report_version != expected_nuitka:
+ raise SystemExit(
+ f"Nuitka report version is {report_version}, expected {expected_nuitka}"
+ )
+ environment = root.find("scons_environment")
+ attributes = (
+ environment.attrib if environment is not None else {}
+ )
+ required = ("c_compiler", "the_cc_name", "the_compiler")
+ if any(not attributes.get(name) for name in required):
+ raise SystemExit("Nuitka report is missing compiler identity")
+ return {
+ "cCompiler": attributes["c_compiler"],
+ "ccName": attributes["the_cc_name"],
+ "compiler": attributes["the_compiler"],
+ }
+
+
+def _verify_identity(
+ *,
+ python_version: str,
+ uv_version: str,
+ nuitka_version: str,
+ builder: PythonNuitkaStandaloneSpec,
+) -> None:
+ checks = (
+ ("Python", python_version, builder.python_version),
+ ("uv", uv_version, builder.uv_version),
+ ("Nuitka", nuitka_version, builder.nuitka_version),
+ )
+ for name, actual, expected in checks:
+ if actual != expected:
+ raise SystemExit(f"builder uses {name} {actual}, expected {expected}")
+
+
+def _copy_dist_payload(
+ *,
+ dist_dir: Path,
+ compiled_executable: Path,
+ payload_root: Path,
+ entrypoint_paths: set[str],
+) -> None:
+ if not dist_dir.is_dir() or dist_dir.is_symlink():
+ raise SystemExit(f"Nuitka standalone directory not found: {dist_dir}")
+ payload_root.mkdir(parents=True)
+ for source_path in sorted(dist_dir.rglob("*"), key=lambda item: item.as_posix()):
+ relative = source_path.relative_to(dist_dir).as_posix()
+ if source_path.is_symlink():
+ raise SystemExit(f"Nuitka output must not contain links: {relative}")
+ if source_path.is_dir():
+ continue
+ if not source_path.is_file():
+ raise SystemExit(f"Nuitka output must contain ordinary files: {relative}")
+ if source_path == compiled_executable:
+ continue
+ if relative in entrypoint_paths:
+ raise SystemExit(f"Nuitka output collides with entrypoint: {relative}")
+ destination = payload_root.joinpath(*relative.split("/"))
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(source_path, destination)
+
+
+def build_python_nuitka_standalone(
+ manifest: ToolManifest,
+ target_key: str,
+ source: PreparedSource,
+ out_dir: Path,
+ work_dir: Path,
+ *,
+ runner: Runner = run_command,
+ entrypoint_resolver: EntrypointResolver = resolve_entrypoint,
+) -> NuitkaBuildResult:
+ if not isinstance(manifest.builder, PythonNuitkaStandaloneSpec):
+ raise SystemExit(f"{manifest.name} is not a Python/Nuitka standalone tool")
+ if target_key not in manifest.targets:
+ raise SystemExit(f"unknown target {target_key}")
+ builder = manifest.builder
+ target = manifest.targets[target_key]
+ if len(builder.binaries) < 2:
+ raise SystemExit("Nuitka multidist requires at least two binaries")
+
+ uv_version = parse_uv_version(runner(["uv", "--version"], cwd=None, env=None))
+ runner(
+ [
+ "uv",
+ "sync",
+ "--frozen",
+ "--no-dev",
+ "--directory",
+ str(source.path),
+ "--python",
+ sys.executable,
+ ],
+ cwd=None,
+ env=None,
+ )
+ if os.name == "nt":
+ venv_python = source.path / ".venv" / "Scripts" / "python.exe"
+ else:
+ venv_python = source.path / ".venv" / "bin" / "python"
+ python_version = parse_python_version(
+ runner([str(venv_python), "--version"], cwd=None, env=None)
+ )
+ runner(
+ [
+ "uv",
+ "pip",
+ "install",
+ "--python",
+ str(venv_python),
+ f"Nuitka=={builder.nuitka_version}",
+ ],
+ cwd=None,
+ env=None,
+ )
+ nuitka_version = parse_nuitka_version(
+ runner(
+ [str(venv_python), "-m", "nuitka", "--version"],
+ cwd=None,
+ env=None,
+ )
+ )
+ _verify_identity(
+ python_version=python_version,
+ uv_version=uv_version,
+ nuitka_version=nuitka_version,
+ builder=builder,
+ )
+
+ build_root = work_dir / "nuitka"
+ entrypoint_root = build_root / "entrypoints"
+ output_root = build_root / "output"
+ report = build_root / "report.xml"
+ entrypoint_root.mkdir(parents=True, exist_ok=True)
+ main_args: list[str] = []
+ for binary in builder.binaries:
+ module, attr = entrypoint_resolver(venv_python, binary.source_name)
+ if module != binary.module:
+ raise SystemExit(
+ f"{binary.source_name} resolves to {module}, expected {binary.module}"
+ )
+ stub = entrypoint_root / f"{binary.asset_base}.py"
+ write_entrypoint_stub(
+ stub,
+ module,
+ attr,
+ configure_utf8_stdio=target_key == "win-x64",
+ )
+ main_args.append(f"--main={stub}")
+
+ runner(
+ [
+ str(venv_python),
+ "-m",
+ "nuitka",
+ "--mode=standalone",
+ "--assume-yes-for-downloads",
+ f"--output-filename={builder.binaries[0].asset_base}{target.exe}",
+ f"--include-package={builder.include_package}",
+ f"--include-package-data={builder.include_package}",
+ f"--output-dir={output_root}",
+ f"--report={report}",
+ *main_args,
+ ],
+ cwd=build_root,
+ env={**os.environ, "PYTHONHASHSEED": "0"},
+ )
+ compiler = parse_nuitka_report(report, expected_nuitka=builder.nuitka_version)
+ builder_identity: dict[str, Any] = {
+ "kind": builder.kind,
+ "python": python_version,
+ "uv": uv_version,
+ "nuitka": nuitka_version,
+ "compiler": compiler,
+ }
+
+ first = builder.binaries[0]
+ dist_dir = output_root / f"{first.asset_base}.dist"
+ compiled_executable = dist_dir / f"{first.asset_base}{target.exe}"
+ if not compiled_executable.is_file() or compiled_executable.is_symlink():
+ raise SystemExit(f"Nuitka output executable not found: {compiled_executable}")
+ payload_root = build_root / "payload"
+ if payload_root.exists():
+ shutil.rmtree(payload_root)
+ entrypoints = {
+ binary.asset_base: f"{binary.asset_base}{target.exe}"
+ for binary in builder.binaries
+ }
+ _copy_dist_payload(
+ dist_dir=dist_dir,
+ compiled_executable=compiled_executable,
+ payload_root=payload_root,
+ entrypoint_paths=set(entrypoints.values()),
+ )
+ for relative in entrypoints.values():
+ destination = payload_root / relative
+ shutil.copyfile(compiled_executable, destination)
+ destination.chmod(0o755)
+
+ out_dir.mkdir(parents=True, exist_ok=True)
+ archive = out_dir / f"{manifest.name}-{target_key}.tar.gz"
+ write_runtime_archive(
+ archive_path=archive,
+ payload_root=payload_root,
+ release_tag=release_tag(manifest),
+ source={
+ "ref": manifest.source.ref,
+ "commit": source.commit,
+ "tree": source.tree,
+ "patches": [
+ {"path": patch.path, "sha256": patch.sha256}
+ for patch in source.patches
+ ],
+ },
+ target={"key": target_key, "triple": target.target_triple},
+ entrypoints=entrypoints,
+ builder=builder_identity,
+ )
+ return NuitkaBuildResult(assets=(archive,), builder_identity=builder_identity)
diff --git a/toolchain/manifest.py b/toolchain/manifest.py
index 38a66d6..a5462d9 100644
--- a/toolchain/manifest.py
+++ b/toolchain/manifest.py
@@ -91,7 +91,18 @@ class PythonBuilderSpec:
binaries: tuple[BinarySpec, ...]
-BuilderSpec = CargoBuilderSpec | PythonBuilderSpec
+@dataclass(frozen=True)
+class PythonNuitkaStandaloneSpec:
+ kind: Literal["python-nuitka-standalone"]
+ python_version: str
+ uv_version: str
+ nuitka_version: str
+ lock_file: str
+ include_package: str
+ binaries: tuple[BinarySpec, ...]
+
+
+BuilderSpec = CargoBuilderSpec | PythonBuilderSpec | PythonNuitkaStandaloneSpec
@dataclass(frozen=True)
@@ -266,13 +277,22 @@ def _load_binary(value: Any, index: int, *, python: bool) -> BinarySpec:
)
-def _load_binaries(value: Any, *, python: bool) -> tuple[BinarySpec, ...]:
+def _load_binaries(
+ value: Any,
+ *,
+ python: bool,
+ unique_sources: bool = False,
+) -> tuple[BinarySpec, ...]:
if not isinstance(value, list) or not value:
raise SystemExit("builder.binaries must be a non-empty array")
binaries = tuple(_load_binary(item, index, python=python) for index, item in enumerate(value))
bases = [binary.asset_base for binary in binaries]
if len(set(bases)) != len(bases):
raise SystemExit("builder.binaries assetBase values must be unique")
+ if unique_sources:
+ sources = [binary.source_name for binary in binaries]
+ if len(set(sources)) != len(sources):
+ raise SystemExit("builder.binaries sourceName values must be unique")
return binaries
@@ -312,6 +332,31 @@ def _load_builder(value: Any) -> BuilderSpec:
collect_all=_name(data["collectAll"], "builder.collectAll"),
binaries=_load_binaries(data["binaries"], python=True),
)
+ if kind == "python-nuitka-standalone":
+ _fields(
+ data,
+ "builder",
+ required={
+ "kind",
+ "pythonVersion",
+ "uvVersion",
+ "nuitkaVersion",
+ "lockFile",
+ "includePackage",
+ "binaries",
+ },
+ )
+ return PythonNuitkaStandaloneSpec(
+ kind="python-nuitka-standalone",
+ python_version=_version(data["pythonVersion"], "builder.pythonVersion"),
+ uv_version=_version(data["uvVersion"], "builder.uvVersion"),
+ nuitka_version=_version(data["nuitkaVersion"], "builder.nuitkaVersion"),
+ lock_file=_string(data["lockFile"], "builder.lockFile"),
+ include_package=_name(data["includePackage"], "builder.includePackage"),
+ binaries=_load_binaries(
+ data["binaries"], python=True, unique_sources=True
+ ),
+ )
raise SystemExit(f"unsupported builder kind: {kind}")
@@ -408,14 +453,29 @@ def release_tag(manifest: ToolManifest) -> str:
)
-def expected_asset_names(manifest: ToolManifest) -> set[str]:
+def expected_target_asset_names(
+ manifest: ToolManifest,
+ target_key: str,
+) -> set[str]:
+ if target_key not in manifest.targets:
+ raise SystemExit(f"unknown target {target_key}")
+ if isinstance(manifest.builder, PythonNuitkaStandaloneSpec):
+ return {f"{manifest.name}-{target_key}.tar.gz"}
+ target = manifest.targets[target_key]
return {
f"{binary.asset_base}-{target_key}{target.exe}"
- for target_key, target in manifest.targets.items()
for binary in manifest.builder.binaries
}
+def expected_asset_names(manifest: ToolManifest) -> set[str]:
+ return {
+ asset
+ for target_key in manifest.targets
+ for asset in expected_target_asset_names(manifest, target_key)
+ }
+
+
def expected_release_files(manifest: ToolManifest) -> set[str]:
result = expected_asset_names(manifest)
result.update(item.asset_name for item in manifest.license.files)
diff --git a/toolchain/provenance.py b/toolchain/provenance.py
index 6a701ae..67b65bf 100644
--- a/toolchain/provenance.py
+++ b/toolchain/provenance.py
@@ -4,7 +4,7 @@
from pathlib import Path
from typing import Any
-from toolchain.manifest import ToolManifest, release_tag
+from toolchain.manifest import ToolManifest, expected_target_asset_names, release_tag
from toolchain.source import PreparedSource, sha256
@@ -20,9 +20,7 @@ def write_target_metadata(
if target_key not in manifest.targets:
raise SystemExit(f"unknown target {target_key}")
target = manifest.targets[target_key]
- expected = {
- f"{binary.asset_base}-{target_key}{target.exe}" for binary in manifest.builder.binaries
- }
+ expected = expected_target_asset_names(manifest, target_key)
actual = {asset.name for asset in assets}
if actual != expected:
raise SystemExit(
diff --git a/toolchain/runtime_archive.py b/toolchain/runtime_archive.py
new file mode 100644
index 0000000..3d0bc6b
--- /dev/null
+++ b/toolchain/runtime_archive.py
@@ -0,0 +1,473 @@
+from __future__ import annotations
+
+import gzip
+import hashlib
+import io
+import json
+import re
+import tarfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+SHA40 = re.compile(r"^[0-9a-f]{40}$")
+SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+@dataclass(frozen=True)
+class RuntimeFile:
+ path: str
+ sha256: str
+ size: int
+ executable: bool
+
+
+@dataclass(frozen=True)
+class RuntimeArchiveManifest:
+ schema_version: int
+ release_tag: str
+ source: dict[str, Any]
+ target: dict[str, str]
+ entrypoints: dict[str, str]
+ builder: dict[str, Any]
+ files: tuple[RuntimeFile, ...]
+
+
+@dataclass(frozen=True)
+class RuntimeArchiveFile:
+ path: str
+ sha256: str
+ size: int
+ executable: bool
+ payload: bytes
+
+
+@dataclass(frozen=True)
+class ValidatedRuntimeArchive:
+ manifest: RuntimeArchiveManifest
+ files: tuple[RuntimeArchiveFile, ...]
+
+
+def _sha256_bytes(payload: bytes) -> str:
+ return hashlib.sha256(payload).hexdigest()
+
+
+def _safe_relative_path(value: Any, *, field: str) -> str:
+ if not isinstance(value, str) or not value:
+ raise SystemExit(f"{field} must be a non-empty relative path")
+ if "\\" in value or value.startswith("/") or "\x00" in value:
+ raise SystemExit(f"unsafe {field}: {value!r}")
+ parts = value.split("/")
+ if any(part in ("", ".", "..") for part in parts):
+ raise SystemExit(f"unsafe {field}: {value!r}")
+ return value
+
+
+def _exact_fields(data: dict[str, Any], expected: set[str], *, field: str) -> None:
+ missing = sorted(expected - data.keys())
+ unknown = sorted(data.keys() - expected)
+ if missing or unknown:
+ raise SystemExit(
+ f"{field} fields mismatch: missing={missing}, unknown={unknown}"
+ )
+
+
+def _dict(value: Any, *, field: str) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ raise SystemExit(f"{field} must be an object")
+ return value
+
+
+def _validate_source(value: Any) -> dict[str, Any]:
+ source = _dict(value, field="source")
+ _exact_fields(source, {"ref", "commit", "tree", "patches"}, field="source")
+ if not isinstance(source["ref"], str) or not source["ref"]:
+ raise SystemExit("source.ref must be a non-empty string")
+ for field in ("commit", "tree"):
+ if not isinstance(source[field], str) or not SHA40.fullmatch(source[field]):
+ raise SystemExit(f"source.{field} must be 40 lowercase hexadecimal characters")
+ patches = source["patches"]
+ if not isinstance(patches, list):
+ raise SystemExit("source.patches must be an array")
+ normalized_patches: list[dict[str, str]] = []
+ for index, item in enumerate(patches):
+ patch = _dict(item, field=f"source.patches[{index}]")
+ _exact_fields(patch, {"path", "sha256"}, field=f"source.patches[{index}]")
+ path = _safe_relative_path(patch["path"], field=f"source.patches[{index}].path")
+ digest = patch["sha256"]
+ if not isinstance(digest, str) or not SHA256.fullmatch(digest):
+ raise SystemExit(
+ f"source.patches[{index}].sha256 must be 64 lowercase hexadecimal characters"
+ )
+ normalized_patches.append({"path": path, "sha256": digest})
+ return {
+ "ref": source["ref"],
+ "commit": source["commit"],
+ "tree": source["tree"],
+ "patches": normalized_patches,
+ }
+
+
+def _validate_target(value: Any) -> dict[str, str]:
+ target = _dict(value, field="target")
+ _exact_fields(target, {"key", "triple"}, field="target")
+ for field in ("key", "triple"):
+ if not isinstance(target[field], str) or not target[field]:
+ raise SystemExit(f"target.{field} must be a non-empty string")
+ return {"key": target["key"], "triple": target["triple"]}
+
+
+def _validate_entrypoints(value: Any) -> dict[str, str]:
+ entrypoints = _dict(value, field="entrypoints")
+ if len(entrypoints) < 2:
+ raise SystemExit("entrypoints must contain at least two programs")
+ normalized: dict[str, str] = {}
+ for name, path_value in entrypoints.items():
+ if not isinstance(name, str) or not name:
+ raise SystemExit("entrypoints names must be non-empty strings")
+ normalized[name] = _safe_relative_path(
+ path_value, field=f"entrypoints.{name}"
+ )
+ if len(set(normalized.values())) != len(normalized):
+ raise SystemExit("entrypoints paths must be unique")
+ return normalized
+
+
+def _validate_builder(value: Any) -> dict[str, Any]:
+ builder = _dict(value, field="builder")
+ if not builder or not isinstance(builder.get("kind"), str):
+ raise SystemExit("builder.kind must be a non-empty string")
+ try:
+ json.dumps(builder, ensure_ascii=False)
+ except (TypeError, ValueError) as exc:
+ raise SystemExit(f"builder must be JSON serializable: {exc}") from exc
+ return builder
+
+
+def _payload_files(payload_root: Path) -> tuple[RuntimeArchiveFile, ...]:
+ if not payload_root.is_dir() or payload_root.is_symlink():
+ raise SystemExit(f"payload root must be an ordinary directory: {payload_root}")
+ result: list[RuntimeArchiveFile] = []
+ for path in sorted(payload_root.rglob("*"), key=lambda item: item.as_posix()):
+ relative = path.relative_to(payload_root).as_posix()
+ _safe_relative_path(relative, field="payload path")
+ if path.is_symlink():
+ raise SystemExit(f"payload member must be an ordinary file: {relative}")
+ if path.is_dir():
+ continue
+ if not path.is_file():
+ raise SystemExit(f"payload member must be an ordinary file: {relative}")
+ payload = path.read_bytes()
+ executable = bool(path.stat().st_mode & 0o111)
+ result.append(
+ RuntimeArchiveFile(
+ path=relative,
+ sha256=_sha256_bytes(payload),
+ size=len(payload),
+ executable=executable,
+ payload=payload,
+ )
+ )
+ if not result:
+ raise SystemExit("payload must contain at least one ordinary file")
+ return tuple(result)
+
+
+def _require_entrypoints(
+ entrypoints: dict[str, str], files: tuple[RuntimeArchiveFile, ...]
+) -> None:
+ by_path = {item.path: item for item in files}
+ selected: list[RuntimeArchiveFile] = []
+ for name, path in entrypoints.items():
+ item = by_path.get(path)
+ if item is None:
+ raise SystemExit(f"entrypoints.{name} is missing from payload: {path}")
+ if not item.executable:
+ raise SystemExit(f"entrypoints.{name} is not executable: {path}")
+ selected.append(item)
+ if len({item.sha256 for item in selected}) != 1:
+ raise SystemExit("multidist entrypoints must be byte-identical")
+
+
+def _manifest_document(
+ *,
+ release_tag: str,
+ source: dict[str, Any],
+ target: dict[str, str],
+ entrypoints: dict[str, str],
+ builder: dict[str, Any],
+ files: tuple[RuntimeArchiveFile, ...],
+) -> dict[str, Any]:
+ return {
+ "schemaVersion": 1,
+ "releaseTag": release_tag,
+ "source": source,
+ "target": target,
+ "entrypoints": entrypoints,
+ "builder": builder,
+ "files": [
+ {
+ "path": item.path,
+ "sha256": item.sha256,
+ "size": item.size,
+ "executable": item.executable,
+ }
+ for item in files
+ ],
+ }
+
+
+def _add_bytes(
+ archive: tarfile.TarFile,
+ name: str,
+ payload: bytes,
+ executable: bool,
+) -> None:
+ info = tarfile.TarInfo(name)
+ info.size = len(payload)
+ info.mode = 0o755 if executable else 0o644
+ info.uid = 0
+ info.gid = 0
+ info.uname = ""
+ info.gname = ""
+ info.mtime = 0
+ archive.addfile(info, io.BytesIO(payload))
+
+
+def write_runtime_archive(
+ *,
+ archive_path: Path,
+ payload_root: Path,
+ release_tag: str,
+ source: dict[str, Any],
+ target: dict[str, Any],
+ entrypoints: dict[str, Any],
+ builder: dict[str, Any],
+) -> Path:
+ if not isinstance(release_tag, str) or not release_tag:
+ raise SystemExit("releaseTag must be a non-empty string")
+ normalized_source = _validate_source(source)
+ normalized_target = _validate_target(target)
+ normalized_entrypoints = _validate_entrypoints(entrypoints)
+ normalized_builder = _validate_builder(builder)
+ files = _payload_files(payload_root)
+ _require_entrypoints(normalized_entrypoints, files)
+ document = _manifest_document(
+ release_tag=release_tag,
+ source=normalized_source,
+ target=normalized_target,
+ entrypoints=normalized_entrypoints,
+ builder=normalized_builder,
+ files=files,
+ )
+ manifest_bytes = (
+ json.dumps(document, ensure_ascii=False, indent=2, sort_keys=False) + "\n"
+ ).encode("utf-8")
+ archive_path.parent.mkdir(parents=True, exist_ok=True)
+ with archive_path.open("wb") as raw:
+ with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed:
+ with tarfile.open(
+ fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT
+ ) as archive:
+ _add_bytes(archive, "manifest.json", manifest_bytes, False)
+ for item in files:
+ _add_bytes(
+ archive,
+ f"payload/{item.path}",
+ item.payload,
+ item.executable,
+ )
+ return archive_path
+
+
+def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+ result: dict[str, Any] = {}
+ for key, value in pairs:
+ if key in result:
+ raise SystemExit(f"duplicate JSON key in manifest: {key}")
+ result[key] = value
+ return result
+
+
+def _load_manifest_bytes(payload: bytes) -> dict[str, Any]:
+ try:
+ value = json.loads(
+ payload.decode("utf-8"), object_pairs_hook=_unique_json_object
+ )
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raise SystemExit(f"invalid UTF-8 JSON manifest: {exc}") from exc
+ return _dict(value, field="manifest")
+
+
+def _parse_manifest(document: dict[str, Any]) -> RuntimeArchiveManifest:
+ _exact_fields(
+ document,
+ {
+ "schemaVersion",
+ "releaseTag",
+ "source",
+ "target",
+ "entrypoints",
+ "builder",
+ "files",
+ },
+ field="manifest",
+ )
+ if document["schemaVersion"] != 1:
+ raise SystemExit(f"unsupported runtime archive schemaVersion: {document['schemaVersion']}")
+ release_tag = document["releaseTag"]
+ if not isinstance(release_tag, str) or not release_tag:
+ raise SystemExit("releaseTag must be a non-empty string")
+ source = _validate_source(document["source"])
+ target = _validate_target(document["target"])
+ entrypoints = _validate_entrypoints(document["entrypoints"])
+ builder = _validate_builder(document["builder"])
+ raw_files = document["files"]
+ if not isinstance(raw_files, list) or not raw_files:
+ raise SystemExit("files must be a non-empty array")
+ files: list[RuntimeFile] = []
+ paths: set[str] = set()
+ for index, raw_file in enumerate(raw_files):
+ item = _dict(raw_file, field=f"files[{index}]")
+ _exact_fields(
+ item,
+ {"path", "sha256", "size", "executable"},
+ field=f"files[{index}]",
+ )
+ path = _safe_relative_path(item["path"], field=f"files[{index}].path")
+ if path in paths:
+ raise SystemExit(f"duplicate file path in manifest: {path}")
+ paths.add(path)
+ digest = item["sha256"]
+ if not isinstance(digest, str) or not SHA256.fullmatch(digest):
+ raise SystemExit(f"files[{index}].sha256 must be 64 lowercase hexadecimal characters")
+ size = item["size"]
+ if not isinstance(size, int) or isinstance(size, bool) or size < 0:
+ raise SystemExit(f"files[{index}].size must be a non-negative integer")
+ executable = item["executable"]
+ if not isinstance(executable, bool):
+ raise SystemExit(f"files[{index}].executable must be a boolean")
+ files.append(RuntimeFile(path, digest, size, executable))
+ if [item.path for item in files] != sorted(item.path for item in files):
+ raise SystemExit("files must be sorted by path")
+ return RuntimeArchiveManifest(
+ schema_version=1,
+ release_tag=release_tag,
+ source=source,
+ target=target,
+ entrypoints=entrypoints,
+ builder=builder,
+ files=tuple(files),
+ )
+
+
+def validate_runtime_archive(
+ path: Path,
+ *,
+ expected_release_tag: str,
+ expected_source_ref: str,
+ expected_source_commit: str,
+ expected_target_key: str,
+ expected_target_triple: str,
+ expected_entrypoints: dict[str, str],
+) -> ValidatedRuntimeArchive:
+ member_payloads: dict[str, tuple[bytes, int]] = {}
+ try:
+ with tarfile.open(path, "r:gz") as archive:
+ for member in archive.getmembers():
+ name = _safe_relative_path(member.name, field="archive member")
+ if name in member_payloads:
+ raise SystemExit(f"duplicate archive member: {name}")
+ if not member.isreg():
+ raise SystemExit(f"archive member must be an ordinary file: {name}")
+ if name != "manifest.json" and not name.startswith("payload/"):
+ raise SystemExit(f"unsafe archive member outside payload: {name}")
+ if member.mode not in (0o644, 0o755):
+ raise SystemExit(f"unsafe archive member mode for {name}: {oct(member.mode)}")
+ stream = archive.extractfile(member)
+ if stream is None:
+ raise SystemExit(f"archive member cannot be read: {name}")
+ member_payloads[name] = (stream.read(), member.mode)
+ except (tarfile.TarError, OSError) as exc:
+ raise SystemExit(f"cannot read runtime archive: {exc}") from exc
+ manifest_entry = member_payloads.get("manifest.json")
+ if manifest_entry is None:
+ raise SystemExit("runtime archive is missing manifest.json")
+ if manifest_entry[1] != 0o644:
+ raise SystemExit("manifest.json mode must be 0644")
+ manifest = _parse_manifest(_load_manifest_bytes(manifest_entry[0]))
+ if manifest.release_tag != expected_release_tag:
+ raise SystemExit(
+ f"releaseTag mismatch: {manifest.release_tag} != {expected_release_tag}"
+ )
+ if manifest.source["ref"] != expected_source_ref:
+ raise SystemExit(
+ f"source.ref mismatch: {manifest.source['ref']} != {expected_source_ref}"
+ )
+ if manifest.source["commit"] != expected_source_commit:
+ raise SystemExit(
+ "source.commit mismatch: "
+ f"{manifest.source['commit']} != {expected_source_commit}"
+ )
+ if manifest.target["key"] != expected_target_key:
+ raise SystemExit(
+ f"target.key mismatch: {manifest.target['key']} != {expected_target_key}"
+ )
+ if manifest.target["triple"] != expected_target_triple:
+ raise SystemExit(
+ "target.triple mismatch: "
+ f"{manifest.target['triple']} != {expected_target_triple}"
+ )
+ if manifest.entrypoints != expected_entrypoints:
+ raise SystemExit(
+ f"entrypoints mismatch: {manifest.entrypoints} != {expected_entrypoints}"
+ )
+ expected_names = {f"payload/{item.path}" for item in manifest.files}
+ actual_names = set(member_payloads) - {"manifest.json"}
+ if actual_names != expected_names:
+ raise SystemExit(
+ "runtime archive file set mismatch: "
+ f"missing={sorted(expected_names - actual_names)}, "
+ f"unexpected={sorted(actual_names - expected_names)}"
+ )
+ files: list[RuntimeArchiveFile] = []
+ for item in manifest.files:
+ payload, mode = member_payloads[f"payload/{item.path}"]
+ if len(payload) != item.size:
+ raise SystemExit(f"size mismatch for payload/{item.path}")
+ digest = _sha256_bytes(payload)
+ if digest != item.sha256:
+ raise SystemExit(f"sha256 mismatch for payload/{item.path}")
+ expected_mode = 0o755 if item.executable else 0o644
+ if mode != expected_mode:
+ raise SystemExit(f"mode mismatch for payload/{item.path}")
+ files.append(
+ RuntimeArchiveFile(
+ path=item.path,
+ sha256=item.sha256,
+ size=item.size,
+ executable=item.executable,
+ payload=payload,
+ )
+ )
+ _require_entrypoints(manifest.entrypoints, tuple(files))
+ return ValidatedRuntimeArchive(manifest=manifest, files=tuple(files))
+
+
+def materialize_runtime_archive(
+ archive: ValidatedRuntimeArchive,
+ destination: Path,
+) -> dict[str, Path]:
+ if destination.exists():
+ raise SystemExit(f"runtime archive destination already exists: {destination}")
+ destination.mkdir(parents=True)
+ result: dict[str, Path] = {}
+ for item in archive.files:
+ path = destination.joinpath(*item.path.split("/"))
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("xb") as stream:
+ stream.write(item.payload)
+ path.chmod(0o755 if item.executable else 0o644)
+ result[item.path] = path
+ return result