From 30fa3e3d1ded33fcf01f729b65dc672d3f52d234 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Fri, 14 Aug 2026 20:22:16 +0300 Subject: [PATCH 1/6] feat(rlm): define Nuitka standalone assets --- tests/test_manifest.py | 64 ++++++++++++++++++++++++++++++++++++++++++ toolchain/manifest.py | 54 +++++++++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index b27954b..3a5b373 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -9,6 +9,7 @@ from toolchain.manifest import ( CargoBuilderSpec, PythonBuilderSpec, + PythonNuitkaStandaloneSpec, expected_asset_names, expected_release_files, load_manifest, @@ -110,6 +111,20 @@ def python_manifest() -> dict: return data +def python_nuitka_manifest() -> dict: + data = python_manifest() + data["builder"] = { + "kind": "python-nuitka-standalone", + "pythonVersion": "3.12.10", + "uvVersion": "0.11.29", + "nuitkaVersion": "4.1.3", + "lockFile": "uv.lock", + "includePackage": "rlm_tools_bsl", + "binaries": data["builder"]["binaries"], + } + return data + + class ManifestTests(unittest.TestCase): def write_manifest(self, data: dict) -> Path: root = Path(self.enterContext(tempfile.TemporaryDirectory())) @@ -151,6 +166,55 @@ def test_loads_python_builder(self) -> None: self.assertEqual(manifest.builder.python_version, "3.12.10") self.assertEqual(manifest.builder.binaries[0].module, "rlm_tools_bsl.server") + def test_loads_nuitka_standalone_builder_and_generates_one_archive_per_target(self) -> None: + manifest = load_manifest(self.write_manifest(python_nuitka_manifest())) + + self.assertIsInstance(manifest.builder, PythonNuitkaStandaloneSpec) + self.assertEqual(manifest.builder.python_version, "3.12.10") + self.assertEqual(manifest.builder.uv_version, "0.11.29") + self.assertEqual(manifest.builder.nuitka_version, "4.1.3") + self.assertEqual(manifest.builder.include_package, "rlm_tools_bsl") + self.assertEqual( + expected_asset_names(manifest), + { + "rlm-tools-bsl-darwin-arm64.tar.gz", + "rlm-tools-bsl-linux-x64.tar.gz", + "rlm-tools-bsl-win-x64.tar.gz", + }, + ) + + def test_nuitka_builder_rejects_pyinstaller_fields_and_invalid_identity(self) -> None: + cases: list[tuple[dict, str]] = [] + + missing_version = python_nuitka_manifest() + del missing_version["builder"]["nuitkaVersion"] + cases.append((missing_version, "missing fields: nuitkaVersion")) + + empty_package = python_nuitka_manifest() + empty_package["builder"]["includePackage"] = "" + cases.append((empty_package, "includePackage must be a non-empty string")) + + pyinstaller_field = python_nuitka_manifest() + pyinstaller_field["builder"]["collectAll"] = "rlm_tools_bsl" + cases.append((pyinstaller_field, "unknown fields: collectAll")) + + duplicate_source = python_nuitka_manifest() + duplicate_source["builder"]["binaries"].append( + dict(duplicate_source["builder"]["binaries"][0], assetBase="other") + ) + cases.append((duplicate_source, "sourceName values must be unique")) + + duplicate_asset = python_nuitka_manifest() + duplicate_asset["builder"]["binaries"].append( + dict(duplicate_asset["builder"]["binaries"][0], sourceName="other") + ) + cases.append((duplicate_asset, "assetBase values must be unique")) + + for data, message in cases: + with self.subTest(message=message): + with self.assertRaisesRegex(SystemExit, message): + load_manifest(self.write_manifest(data)) + def test_loads_captured_smoke_checks_with_literal_output_contracts(self) -> None: data = python_manifest() binary = data["builder"]["binaries"][0] diff --git a/toolchain/manifest.py b/toolchain/manifest.py index 38a66d6..c5d7414 100644 --- a/toolchain/manifest.py +++ b/toolchain/manifest.py @@ -91,7 +91,18 @@ class PythonBuilderSpec: binaries: tuple[BinarySpec, ...] -BuilderSpec = CargoBuilderSpec | PythonBuilderSpec +@dataclass(frozen=True) +class PythonNuitkaStandaloneSpec: + kind: Literal["python-nuitka-standalone"] + python_version: str + uv_version: str + nuitka_version: str + lock_file: str + include_package: str + binaries: tuple[BinarySpec, ...] + + +BuilderSpec = CargoBuilderSpec | PythonBuilderSpec | PythonNuitkaStandaloneSpec @dataclass(frozen=True) @@ -266,13 +277,22 @@ def _load_binary(value: Any, index: int, *, python: bool) -> BinarySpec: ) -def _load_binaries(value: Any, *, python: bool) -> tuple[BinarySpec, ...]: +def _load_binaries( + value: Any, + *, + python: bool, + unique_sources: bool = False, +) -> tuple[BinarySpec, ...]: if not isinstance(value, list) or not value: raise SystemExit("builder.binaries must be a non-empty array") binaries = tuple(_load_binary(item, index, python=python) for index, item in enumerate(value)) bases = [binary.asset_base for binary in binaries] if len(set(bases)) != len(bases): raise SystemExit("builder.binaries assetBase values must be unique") + if unique_sources: + sources = [binary.source_name for binary in binaries] + if len(set(sources)) != len(sources): + raise SystemExit("builder.binaries sourceName values must be unique") return binaries @@ -312,6 +332,31 @@ def _load_builder(value: Any) -> BuilderSpec: collect_all=_name(data["collectAll"], "builder.collectAll"), binaries=_load_binaries(data["binaries"], python=True), ) + if kind == "python-nuitka-standalone": + _fields( + data, + "builder", + required={ + "kind", + "pythonVersion", + "uvVersion", + "nuitkaVersion", + "lockFile", + "includePackage", + "binaries", + }, + ) + return PythonNuitkaStandaloneSpec( + kind="python-nuitka-standalone", + python_version=_version(data["pythonVersion"], "builder.pythonVersion"), + uv_version=_version(data["uvVersion"], "builder.uvVersion"), + nuitka_version=_version(data["nuitkaVersion"], "builder.nuitkaVersion"), + lock_file=_string(data["lockFile"], "builder.lockFile"), + include_package=_name(data["includePackage"], "builder.includePackage"), + binaries=_load_binaries( + data["binaries"], python=True, unique_sources=True + ), + ) raise SystemExit(f"unsupported builder kind: {kind}") @@ -409,6 +454,11 @@ def release_tag(manifest: ToolManifest) -> str: def expected_asset_names(manifest: ToolManifest) -> set[str]: + if isinstance(manifest.builder, PythonNuitkaStandaloneSpec): + return { + f"{manifest.name}-{target_key}.tar.gz" + for target_key in manifest.targets + } return { f"{binary.asset_base}-{target_key}{target.exe}" for target_key, target in manifest.targets.items() From 693a8189c3b5cd091529bdb6402018cb154e33f1 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Fri, 14 Aug 2026 20:26:44 +0300 Subject: [PATCH 2/6] feat(toolchain): add verified runtime archives --- tests/test_runtime_archive.py | 316 +++++++++++++++++++++++ toolchain/runtime_archive.py | 473 ++++++++++++++++++++++++++++++++++ 2 files changed, 789 insertions(+) create mode 100644 tests/test_runtime_archive.py create mode 100644 toolchain/runtime_archive.py diff --git a/tests/test_runtime_archive.py b/tests/test_runtime_archive.py new file mode 100644 index 0000000..5090d79 --- /dev/null +++ b/tests/test_runtime_archive.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import tarfile +import tempfile +import unittest +from pathlib import Path + +from toolchain.runtime_archive import ( + materialize_runtime_archive, + validate_runtime_archive, + write_runtime_archive, +) + + +RELEASE_TAG = "rlm-tools-bsl-v1.33.0-build.3" +SOURCE = { + "ref": "v1.33.0", + "commit": "3e6920cd015a61af4ba7aa1a5f1fedd8bc935549", + "tree": "4b321de0454d4d0998762659891374a3a1326cd0", + "patches": [], +} +TARGET = { + "key": "linux-x64", + "triple": "x86_64-unknown-linux-gnu", +} +ENTRYPOINTS = { + "rlm-bsl-index": "rlm-bsl-index", + "rlm-bsl-mcp": "rlm-bsl-mcp", +} +BUILDER = { + "kind": "python-nuitka-standalone", + "python": "3.12.10", + "uv": "0.11.29", + "nuitka": "4.1.3", + "compiler": {"cCompiler": "Clang", "ccName": "clang", "compiler": "clang"}, +} + + +def add_member( + archive: tarfile.TarFile, + name: str, + payload: bytes, + *, + mode: int = 0o644, + type_: bytes = tarfile.REGTYPE, + linkname: str = "", +) -> None: + info = tarfile.TarInfo(name) + info.size = len(payload) if type_ == tarfile.REGTYPE else 0 + info.mode = mode + info.type = type_ + info.linkname = linkname + archive.addfile(info, io.BytesIO(payload) if type_ == tarfile.REGTYPE else None) + + +def raw_archive(path: Path, members: list[tuple[str, bytes, int, bytes, str]]) -> None: + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w") as archive: + for name, payload, mode, type_, linkname in members: + add_member( + archive, + name, + payload, + mode=mode, + type_=type_, + linkname=linkname, + ) + + +def expected_manifest(binary: bytes = b"multidist") -> dict: + files = [ + { + "path": "libpython3.12.so.1.0", + "sha256": hashlib.sha256(b"shared").hexdigest(), + "size": len(b"shared"), + "executable": False, + }, + { + "path": "rlm-bsl-index", + "sha256": hashlib.sha256(binary).hexdigest(), + "size": len(binary), + "executable": True, + }, + { + "path": "rlm-bsl-mcp", + "sha256": hashlib.sha256(binary).hexdigest(), + "size": len(binary), + "executable": True, + }, + ] + return { + "schemaVersion": 1, + "releaseTag": RELEASE_TAG, + "source": SOURCE, + "target": TARGET, + "entrypoints": ENTRYPOINTS, + "builder": BUILDER, + "files": files, + } + + +class RuntimeArchiveTests(unittest.TestCase): + def make_payload(self, root: Path) -> Path: + payload = root / "payload" + payload.mkdir() + for name, data in ( + ("rlm-bsl-index", b"multidist"), + ("rlm-bsl-mcp", b"multidist"), + ("libpython3.12.so.1.0", b"shared"), + ): + path = payload / name + path.write_bytes(data) + path.chmod(0o755 if name.startswith("rlm-bsl-") else 0o644) + return payload + + def validate(self, path: Path): + return validate_runtime_archive( + path, + expected_release_tag=RELEASE_TAG, + expected_source_ref=SOURCE["ref"], + expected_source_commit=SOURCE["commit"], + expected_target_key=TARGET["key"], + expected_target_triple=TARGET["triple"], + expected_entrypoints=ENTRYPOINTS, + ) + + def test_archive_is_deterministic_complete_and_materializes_exact_payload(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + payload = self.make_payload(root) + first = root / "first.tar.gz" + second = root / "second.tar.gz" + + for archive in (first, second): + write_runtime_archive( + archive_path=archive, + payload_root=payload, + release_tag=RELEASE_TAG, + source=SOURCE, + target=TARGET, + entrypoints=ENTRYPOINTS, + builder=BUILDER, + ) + + self.assertEqual(first.read_bytes(), second.read_bytes()) + with tarfile.open(first, "r:gz") as archive: + members = archive.getmembers() + self.assertEqual( + [member.name for member in members], + [ + "manifest.json", + "payload/libpython3.12.so.1.0", + "payload/rlm-bsl-index", + "payload/rlm-bsl-mcp", + ], + ) + self.assertTrue(all(member.uid == 0 and member.gid == 0 for member in members)) + self.assertTrue(all(member.uname == "" and member.gname == "" for member in members)) + self.assertTrue(all(member.mtime == 0 for member in members)) + manifest = json.loads(archive.extractfile("manifest.json").read()) + self.assertEqual(manifest, expected_manifest()) + + validated = self.validate(first) + destination = root / "materialized" + paths = materialize_runtime_archive(validated, destination) + self.assertEqual(set(paths), set(ENTRYPOINTS.values()) | {"libpython3.12.so.1.0"}) + self.assertEqual((destination / "rlm-bsl-index").read_bytes(), b"multidist") + self.assertEqual((destination / "rlm-bsl-mcp").read_bytes(), b"multidist") + self.assertEqual((destination / "libpython3.12.so.1.0").read_bytes(), b"shared") + self.assertTrue((destination / "rlm-bsl-index").stat().st_mode & 0o111) + self.assertFalse((destination / "libpython3.12.so.1.0").stat().st_mode & 0o111) + + def test_rejects_unsafe_duplicate_and_non_regular_archive_members(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + manifest = json.dumps(expected_manifest(), separators=(",", ":")).encode() + base = [ + ("manifest.json", manifest, 0o644, tarfile.REGTYPE, ""), + ("payload/libpython3.12.so.1.0", b"shared", 0o644, tarfile.REGTYPE, ""), + ("payload/rlm-bsl-index", b"multidist", 0o755, tarfile.REGTYPE, ""), + ("payload/rlm-bsl-mcp", b"multidist", 0o755, tarfile.REGTYPE, ""), + ] + cases = { + "absolute": base + [("/absolute", b"x", 0o644, tarfile.REGTYPE, "")], + "parent": base + [("payload/../escape", b"x", 0o644, tarfile.REGTYPE, "")], + "backslash": base + [("payload\\escape", b"x", 0o644, tarfile.REGTYPE, "")], + "duplicate": base + [("payload/rlm-bsl-index", b"multidist", 0o755, tarfile.REGTYPE, "")], + "symlink": base + [("payload/link", b"", 0o777, tarfile.SYMTYPE, "rlm-bsl-index")], + "hardlink": base + [("payload/link", b"", 0o777, tarfile.LNKTYPE, "rlm-bsl-index")], + "fifo": base + [("payload/fifo", b"", 0o644, tarfile.FIFOTYPE, "")], + } + + for label, members in cases.items(): + with self.subTest(label=label): + archive = root / f"{label}.tar.gz" + raw_archive(archive, members) + with self.assertRaisesRegex(SystemExit, "unsafe|duplicate|ordinary"): + self.validate(archive) + + def test_rejects_manifest_identity_file_set_and_payload_drift(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + mutations: list[tuple[str, dict, list[tuple[str, bytes, int, bytes, str]], str]] = [] + base_members = [ + ("payload/libpython3.12.so.1.0", b"shared", 0o644, tarfile.REGTYPE, ""), + ("payload/rlm-bsl-index", b"multidist", 0o755, tarfile.REGTYPE, ""), + ("payload/rlm-bsl-mcp", b"multidist", 0o755, tarfile.REGTYPE, ""), + ] + + wrong_release = expected_manifest() + wrong_release["releaseTag"] = "other" + mutations.append(("release", wrong_release, base_members, "releaseTag")) + + wrong_source = expected_manifest() + wrong_source["source"] = dict(SOURCE, commit="a" * 40) + mutations.append(("source", wrong_source, base_members, "source.commit")) + + wrong_target = expected_manifest() + wrong_target["target"] = dict(TARGET, key="win-x64") + mutations.append(("target", wrong_target, base_members, "target.key")) + + wrong_entrypoint = expected_manifest() + wrong_entrypoint["entrypoints"] = {"rlm-bsl-index": "rlm-bsl-index"} + mutations.append(("entrypoint", wrong_entrypoint, base_members, "entrypoints")) + + wrong_digest = expected_manifest() + wrong_digest["files"][1]["sha256"] = "0" * 64 + mutations.append(("digest", wrong_digest, base_members, "sha256")) + + wrong_size = expected_manifest() + wrong_size["files"][1]["size"] = 99 + mutations.append(("size", wrong_size, base_members, "size")) + + missing_file = expected_manifest() + mutations.append(("missing", missing_file, base_members[:-1], "file set")) + + extra_file = expected_manifest() + mutations.append( + ( + "extra", + extra_file, + base_members + [("payload/extra", b"x", 0o644, tarfile.REGTYPE, "")], + "file set", + ) + ) + + unequal = expected_manifest(binary=b"index") + unequal["files"][2] = { + "path": "rlm-bsl-mcp", + "sha256": hashlib.sha256(b"mcp").hexdigest(), + "size": 3, + "executable": True, + } + mutations.append( + ( + "unequal", + unequal, + [ + base_members[0], + ("payload/rlm-bsl-index", b"index", 0o755, tarfile.REGTYPE, ""), + ("payload/rlm-bsl-mcp", b"mcp", 0o755, tarfile.REGTYPE, ""), + ], + "byte-identical", + ) + ) + + for label, manifest, payload_members, message in mutations: + with self.subTest(label=label): + archive = root / f"{label}.tar.gz" + members = [ + ( + "manifest.json", + json.dumps(manifest, separators=(",", ":")).encode(), + 0o644, + tarfile.REGTYPE, + "", + ), + *payload_members, + ] + raw_archive(archive, members) + with self.assertRaisesRegex(SystemExit, message): + self.validate(archive) + + def test_writer_rejects_symlink_and_different_entrypoint_bytes(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + payload = self.make_payload(root) + (payload / "rlm-bsl-mcp").write_bytes(b"different") + with self.assertRaisesRegex(SystemExit, "byte-identical"): + write_runtime_archive( + archive_path=root / "different.tar.gz", + payload_root=payload, + release_tag=RELEASE_TAG, + source=SOURCE, + target=TARGET, + entrypoints=ENTRYPOINTS, + builder=BUILDER, + ) + + (payload / "rlm-bsl-mcp").unlink() + (payload / "rlm-bsl-mcp").symlink_to(payload / "rlm-bsl-index") + with self.assertRaisesRegex(SystemExit, "ordinary"): + write_runtime_archive( + archive_path=root / "symlink.tar.gz", + payload_root=payload, + release_tag=RELEASE_TAG, + source=SOURCE, + target=TARGET, + entrypoints=ENTRYPOINTS, + builder=BUILDER, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/toolchain/runtime_archive.py b/toolchain/runtime_archive.py new file mode 100644 index 0000000..3d0bc6b --- /dev/null +++ b/toolchain/runtime_archive.py @@ -0,0 +1,473 @@ +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import re +import tarfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +SHA40 = re.compile(r"^[0-9a-f]{40}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +@dataclass(frozen=True) +class RuntimeFile: + path: str + sha256: str + size: int + executable: bool + + +@dataclass(frozen=True) +class RuntimeArchiveManifest: + schema_version: int + release_tag: str + source: dict[str, Any] + target: dict[str, str] + entrypoints: dict[str, str] + builder: dict[str, Any] + files: tuple[RuntimeFile, ...] + + +@dataclass(frozen=True) +class RuntimeArchiveFile: + path: str + sha256: str + size: int + executable: bool + payload: bytes + + +@dataclass(frozen=True) +class ValidatedRuntimeArchive: + manifest: RuntimeArchiveManifest + files: tuple[RuntimeArchiveFile, ...] + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _safe_relative_path(value: Any, *, field: str) -> str: + if not isinstance(value, str) or not value: + raise SystemExit(f"{field} must be a non-empty relative path") + if "\\" in value or value.startswith("/") or "\x00" in value: + raise SystemExit(f"unsafe {field}: {value!r}") + parts = value.split("/") + if any(part in ("", ".", "..") for part in parts): + raise SystemExit(f"unsafe {field}: {value!r}") + return value + + +def _exact_fields(data: dict[str, Any], expected: set[str], *, field: str) -> None: + missing = sorted(expected - data.keys()) + unknown = sorted(data.keys() - expected) + if missing or unknown: + raise SystemExit( + f"{field} fields mismatch: missing={missing}, unknown={unknown}" + ) + + +def _dict(value: Any, *, field: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise SystemExit(f"{field} must be an object") + return value + + +def _validate_source(value: Any) -> dict[str, Any]: + source = _dict(value, field="source") + _exact_fields(source, {"ref", "commit", "tree", "patches"}, field="source") + if not isinstance(source["ref"], str) or not source["ref"]: + raise SystemExit("source.ref must be a non-empty string") + for field in ("commit", "tree"): + if not isinstance(source[field], str) or not SHA40.fullmatch(source[field]): + raise SystemExit(f"source.{field} must be 40 lowercase hexadecimal characters") + patches = source["patches"] + if not isinstance(patches, list): + raise SystemExit("source.patches must be an array") + normalized_patches: list[dict[str, str]] = [] + for index, item in enumerate(patches): + patch = _dict(item, field=f"source.patches[{index}]") + _exact_fields(patch, {"path", "sha256"}, field=f"source.patches[{index}]") + path = _safe_relative_path(patch["path"], field=f"source.patches[{index}].path") + digest = patch["sha256"] + if not isinstance(digest, str) or not SHA256.fullmatch(digest): + raise SystemExit( + f"source.patches[{index}].sha256 must be 64 lowercase hexadecimal characters" + ) + normalized_patches.append({"path": path, "sha256": digest}) + return { + "ref": source["ref"], + "commit": source["commit"], + "tree": source["tree"], + "patches": normalized_patches, + } + + +def _validate_target(value: Any) -> dict[str, str]: + target = _dict(value, field="target") + _exact_fields(target, {"key", "triple"}, field="target") + for field in ("key", "triple"): + if not isinstance(target[field], str) or not target[field]: + raise SystemExit(f"target.{field} must be a non-empty string") + return {"key": target["key"], "triple": target["triple"]} + + +def _validate_entrypoints(value: Any) -> dict[str, str]: + entrypoints = _dict(value, field="entrypoints") + if len(entrypoints) < 2: + raise SystemExit("entrypoints must contain at least two programs") + normalized: dict[str, str] = {} + for name, path_value in entrypoints.items(): + if not isinstance(name, str) or not name: + raise SystemExit("entrypoints names must be non-empty strings") + normalized[name] = _safe_relative_path( + path_value, field=f"entrypoints.{name}" + ) + if len(set(normalized.values())) != len(normalized): + raise SystemExit("entrypoints paths must be unique") + return normalized + + +def _validate_builder(value: Any) -> dict[str, Any]: + builder = _dict(value, field="builder") + if not builder or not isinstance(builder.get("kind"), str): + raise SystemExit("builder.kind must be a non-empty string") + try: + json.dumps(builder, ensure_ascii=False) + except (TypeError, ValueError) as exc: + raise SystemExit(f"builder must be JSON serializable: {exc}") from exc + return builder + + +def _payload_files(payload_root: Path) -> tuple[RuntimeArchiveFile, ...]: + if not payload_root.is_dir() or payload_root.is_symlink(): + raise SystemExit(f"payload root must be an ordinary directory: {payload_root}") + result: list[RuntimeArchiveFile] = [] + for path in sorted(payload_root.rglob("*"), key=lambda item: item.as_posix()): + relative = path.relative_to(payload_root).as_posix() + _safe_relative_path(relative, field="payload path") + if path.is_symlink(): + raise SystemExit(f"payload member must be an ordinary file: {relative}") + if path.is_dir(): + continue + if not path.is_file(): + raise SystemExit(f"payload member must be an ordinary file: {relative}") + payload = path.read_bytes() + executable = bool(path.stat().st_mode & 0o111) + result.append( + RuntimeArchiveFile( + path=relative, + sha256=_sha256_bytes(payload), + size=len(payload), + executable=executable, + payload=payload, + ) + ) + if not result: + raise SystemExit("payload must contain at least one ordinary file") + return tuple(result) + + +def _require_entrypoints( + entrypoints: dict[str, str], files: tuple[RuntimeArchiveFile, ...] +) -> None: + by_path = {item.path: item for item in files} + selected: list[RuntimeArchiveFile] = [] + for name, path in entrypoints.items(): + item = by_path.get(path) + if item is None: + raise SystemExit(f"entrypoints.{name} is missing from payload: {path}") + if not item.executable: + raise SystemExit(f"entrypoints.{name} is not executable: {path}") + selected.append(item) + if len({item.sha256 for item in selected}) != 1: + raise SystemExit("multidist entrypoints must be byte-identical") + + +def _manifest_document( + *, + release_tag: str, + source: dict[str, Any], + target: dict[str, str], + entrypoints: dict[str, str], + builder: dict[str, Any], + files: tuple[RuntimeArchiveFile, ...], +) -> dict[str, Any]: + return { + "schemaVersion": 1, + "releaseTag": release_tag, + "source": source, + "target": target, + "entrypoints": entrypoints, + "builder": builder, + "files": [ + { + "path": item.path, + "sha256": item.sha256, + "size": item.size, + "executable": item.executable, + } + for item in files + ], + } + + +def _add_bytes( + archive: tarfile.TarFile, + name: str, + payload: bytes, + executable: bool, +) -> None: + info = tarfile.TarInfo(name) + info.size = len(payload) + info.mode = 0o755 if executable else 0o644 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + info.mtime = 0 + archive.addfile(info, io.BytesIO(payload)) + + +def write_runtime_archive( + *, + archive_path: Path, + payload_root: Path, + release_tag: str, + source: dict[str, Any], + target: dict[str, Any], + entrypoints: dict[str, Any], + builder: dict[str, Any], +) -> Path: + if not isinstance(release_tag, str) or not release_tag: + raise SystemExit("releaseTag must be a non-empty string") + normalized_source = _validate_source(source) + normalized_target = _validate_target(target) + normalized_entrypoints = _validate_entrypoints(entrypoints) + normalized_builder = _validate_builder(builder) + files = _payload_files(payload_root) + _require_entrypoints(normalized_entrypoints, files) + document = _manifest_document( + release_tag=release_tag, + source=normalized_source, + target=normalized_target, + entrypoints=normalized_entrypoints, + builder=normalized_builder, + files=files, + ) + manifest_bytes = ( + json.dumps(document, ensure_ascii=False, indent=2, sort_keys=False) + "\n" + ).encode("utf-8") + archive_path.parent.mkdir(parents=True, exist_ok=True) + with archive_path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + with tarfile.open( + fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT + ) as archive: + _add_bytes(archive, "manifest.json", manifest_bytes, False) + for item in files: + _add_bytes( + archive, + f"payload/{item.path}", + item.payload, + item.executable, + ) + return archive_path + + +def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise SystemExit(f"duplicate JSON key in manifest: {key}") + result[key] = value + return result + + +def _load_manifest_bytes(payload: bytes) -> dict[str, Any]: + try: + value = json.loads( + payload.decode("utf-8"), object_pairs_hook=_unique_json_object + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SystemExit(f"invalid UTF-8 JSON manifest: {exc}") from exc + return _dict(value, field="manifest") + + +def _parse_manifest(document: dict[str, Any]) -> RuntimeArchiveManifest: + _exact_fields( + document, + { + "schemaVersion", + "releaseTag", + "source", + "target", + "entrypoints", + "builder", + "files", + }, + field="manifest", + ) + if document["schemaVersion"] != 1: + raise SystemExit(f"unsupported runtime archive schemaVersion: {document['schemaVersion']}") + release_tag = document["releaseTag"] + if not isinstance(release_tag, str) or not release_tag: + raise SystemExit("releaseTag must be a non-empty string") + source = _validate_source(document["source"]) + target = _validate_target(document["target"]) + entrypoints = _validate_entrypoints(document["entrypoints"]) + builder = _validate_builder(document["builder"]) + raw_files = document["files"] + if not isinstance(raw_files, list) or not raw_files: + raise SystemExit("files must be a non-empty array") + files: list[RuntimeFile] = [] + paths: set[str] = set() + for index, raw_file in enumerate(raw_files): + item = _dict(raw_file, field=f"files[{index}]") + _exact_fields( + item, + {"path", "sha256", "size", "executable"}, + field=f"files[{index}]", + ) + path = _safe_relative_path(item["path"], field=f"files[{index}].path") + if path in paths: + raise SystemExit(f"duplicate file path in manifest: {path}") + paths.add(path) + digest = item["sha256"] + if not isinstance(digest, str) or not SHA256.fullmatch(digest): + raise SystemExit(f"files[{index}].sha256 must be 64 lowercase hexadecimal characters") + size = item["size"] + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + raise SystemExit(f"files[{index}].size must be a non-negative integer") + executable = item["executable"] + if not isinstance(executable, bool): + raise SystemExit(f"files[{index}].executable must be a boolean") + files.append(RuntimeFile(path, digest, size, executable)) + if [item.path for item in files] != sorted(item.path for item in files): + raise SystemExit("files must be sorted by path") + return RuntimeArchiveManifest( + schema_version=1, + release_tag=release_tag, + source=source, + target=target, + entrypoints=entrypoints, + builder=builder, + files=tuple(files), + ) + + +def validate_runtime_archive( + path: Path, + *, + expected_release_tag: str, + expected_source_ref: str, + expected_source_commit: str, + expected_target_key: str, + expected_target_triple: str, + expected_entrypoints: dict[str, str], +) -> ValidatedRuntimeArchive: + member_payloads: dict[str, tuple[bytes, int]] = {} + try: + with tarfile.open(path, "r:gz") as archive: + for member in archive.getmembers(): + name = _safe_relative_path(member.name, field="archive member") + if name in member_payloads: + raise SystemExit(f"duplicate archive member: {name}") + if not member.isreg(): + raise SystemExit(f"archive member must be an ordinary file: {name}") + if name != "manifest.json" and not name.startswith("payload/"): + raise SystemExit(f"unsafe archive member outside payload: {name}") + if member.mode not in (0o644, 0o755): + raise SystemExit(f"unsafe archive member mode for {name}: {oct(member.mode)}") + stream = archive.extractfile(member) + if stream is None: + raise SystemExit(f"archive member cannot be read: {name}") + member_payloads[name] = (stream.read(), member.mode) + except (tarfile.TarError, OSError) as exc: + raise SystemExit(f"cannot read runtime archive: {exc}") from exc + manifest_entry = member_payloads.get("manifest.json") + if manifest_entry is None: + raise SystemExit("runtime archive is missing manifest.json") + if manifest_entry[1] != 0o644: + raise SystemExit("manifest.json mode must be 0644") + manifest = _parse_manifest(_load_manifest_bytes(manifest_entry[0])) + if manifest.release_tag != expected_release_tag: + raise SystemExit( + f"releaseTag mismatch: {manifest.release_tag} != {expected_release_tag}" + ) + if manifest.source["ref"] != expected_source_ref: + raise SystemExit( + f"source.ref mismatch: {manifest.source['ref']} != {expected_source_ref}" + ) + if manifest.source["commit"] != expected_source_commit: + raise SystemExit( + "source.commit mismatch: " + f"{manifest.source['commit']} != {expected_source_commit}" + ) + if manifest.target["key"] != expected_target_key: + raise SystemExit( + f"target.key mismatch: {manifest.target['key']} != {expected_target_key}" + ) + if manifest.target["triple"] != expected_target_triple: + raise SystemExit( + "target.triple mismatch: " + f"{manifest.target['triple']} != {expected_target_triple}" + ) + if manifest.entrypoints != expected_entrypoints: + raise SystemExit( + f"entrypoints mismatch: {manifest.entrypoints} != {expected_entrypoints}" + ) + expected_names = {f"payload/{item.path}" for item in manifest.files} + actual_names = set(member_payloads) - {"manifest.json"} + if actual_names != expected_names: + raise SystemExit( + "runtime archive file set mismatch: " + f"missing={sorted(expected_names - actual_names)}, " + f"unexpected={sorted(actual_names - expected_names)}" + ) + files: list[RuntimeArchiveFile] = [] + for item in manifest.files: + payload, mode = member_payloads[f"payload/{item.path}"] + if len(payload) != item.size: + raise SystemExit(f"size mismatch for payload/{item.path}") + digest = _sha256_bytes(payload) + if digest != item.sha256: + raise SystemExit(f"sha256 mismatch for payload/{item.path}") + expected_mode = 0o755 if item.executable else 0o644 + if mode != expected_mode: + raise SystemExit(f"mode mismatch for payload/{item.path}") + files.append( + RuntimeArchiveFile( + path=item.path, + sha256=item.sha256, + size=item.size, + executable=item.executable, + payload=payload, + ) + ) + _require_entrypoints(manifest.entrypoints, tuple(files)) + return ValidatedRuntimeArchive(manifest=manifest, files=tuple(files)) + + +def materialize_runtime_archive( + archive: ValidatedRuntimeArchive, + destination: Path, +) -> dict[str, Path]: + if destination.exists(): + raise SystemExit(f"runtime archive destination already exists: {destination}") + destination.mkdir(parents=True) + result: dict[str, Path] = {} + for item in archive.files: + path = destination.joinpath(*item.path.split("/")) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as stream: + stream.write(item.payload) + path.chmod(0o755 if item.executable else 0o644) + result[item.path] = path + return result From 76ae0cf8cba34e59155a6b3400b3bcda9e2f3a9b Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Fri, 14 Aug 2026 20:29:50 +0300 Subject: [PATCH 3/6] feat(rlm): build standalone multidist runtime --- tests/test_python_nuitka_standalone.py | 286 ++++++++++++++++++ .../builders/python_nuitka_standalone.py | 275 +++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100644 tests/test_python_nuitka_standalone.py create mode 100644 toolchain/builders/python_nuitka_standalone.py diff --git a/tests/test_python_nuitka_standalone.py b/tests/test_python_nuitka_standalone.py new file mode 100644 index 0000000..336bf56 --- /dev/null +++ b/tests/test_python_nuitka_standalone.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from tests.test_manifest import python_nuitka_manifest +from toolchain.builders.python_nuitka_standalone import ( + build_python_nuitka_standalone, + parse_nuitka_report, + parse_nuitka_version, +) +from toolchain.manifest import load_manifest, release_tag +from toolchain.runtime_archive import validate_runtime_archive +from toolchain.source import PreparedSource + + +class PythonNuitkaStandaloneTests(unittest.TestCase): + def make_manifest(self, root: Path, *, target: str = "darwin-arm64"): + data = python_nuitka_manifest() + data["builder"]["binaries"] = [ + { + "package": "rlm_tools_bsl", + "sourceName": "rlm-bsl-index", + "module": "rlm_tools_bsl.cli", + "assetBase": "rlm-bsl-index", + "smokeArgs": ["--help"], + }, + { + "package": "rlm_tools_bsl", + "sourceName": "rlm-tools-bsl", + "module": "rlm_tools_bsl.server", + "assetBase": "rlm-bsl-mcp", + "smokeArgs": ["--help"], + }, + ] + path = root / "manifest.json" + path.write_text(json.dumps(data), encoding="utf-8") + return load_manifest(path), target + + def test_builds_one_archive_from_two_main_programs_and_records_compiler(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + manifest, target_key = self.make_manifest(root) + source_dir = root / "source" + source_dir.mkdir() + source = PreparedSource( + source_dir, + manifest.source.commit, + "b" * 40, + (), + ) + out_dir = root / "out" + work_dir = root / "work" + commands: list[list[str]] = [] + + def fake_runner(command: list[str], *, cwd=None, env=None) -> str: + commands.append(command) + if command == ["uv", "--version"]: + return "uv 0.11.29 (test)" + if command[-1:] == ["--version"] and "nuitka" in command: + return "4.1.3" + if command[-1:] == ["--version"]: + return "Python 3.12.10" + if "--mode=standalone" in command: + output_arg = next(item for item in command if item.startswith("--output-dir=")) + report_arg = next(item for item in command if item.startswith("--report=")) + output = Path(output_arg.split("=", 1)[1]) + report = Path(report_arg.split("=", 1)[1]) + dist = output / "rlm-bsl-index.dist" + nested = dist / "package" + nested.mkdir(parents=True) + executable = dist / "rlm-bsl-index" + executable.write_bytes(b"compiled multidist") + executable.chmod(0o755) + (dist / "libpython3.12.dylib").write_bytes(b"python") + (nested / "data.json").write_bytes(b"{}") + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text( + '' + '' + "", + encoding="utf-8", + ) + return "" + + modules = { + "rlm-bsl-index": ("rlm_tools_bsl.cli", "main"), + "rlm-tools-bsl": ("rlm_tools_bsl.server", "main"), + } + result = build_python_nuitka_standalone( + manifest, + target_key, + source, + out_dir, + work_dir, + runner=fake_runner, + entrypoint_resolver=lambda _python, name: modules[name], + ) + + self.assertEqual( + [path.name for path in result.assets], + ["rlm-tools-bsl-darwin-arm64.tar.gz"], + ) + compile_commands = [command for command in commands if "--mode=standalone" in command] + self.assertEqual(len(compile_commands), 1) + compile_command = compile_commands[0] + main_args = [item for item in compile_command if item.startswith("--main=")] + self.assertEqual(len(main_args), 2) + self.assertTrue(main_args[0].endswith("/rlm-bsl-index.py")) + self.assertTrue(main_args[1].endswith("/rlm-bsl-mcp.py")) + self.assertIn("--include-package=rlm_tools_bsl", compile_command) + self.assertIn("--include-package-data=rlm_tools_bsl", compile_command) + self.assertIn("--assume-yes-for-downloads", compile_command) + self.assertFalse(any("onefile" in item for item in compile_command)) + self.assertFalse(any("tempdir" in item for item in compile_command)) + self.assertEqual( + result.builder_identity, + { + "kind": "python-nuitka-standalone", + "python": "3.12.10", + "uv": "0.11.29", + "nuitka": "4.1.3", + "compiler": { + "cCompiler": "Clang", + "ccName": "clang", + "compiler": "clang", + }, + }, + ) + + archive = validate_runtime_archive( + result.assets[0], + expected_release_tag=release_tag(manifest), + expected_source_ref=manifest.source.ref, + expected_source_commit=manifest.source.commit, + expected_target_key=target_key, + expected_target_triple=manifest.targets[target_key].target_triple, + expected_entrypoints={ + "rlm-bsl-index": "rlm-bsl-index", + "rlm-bsl-mcp": "rlm-bsl-mcp", + }, + ) + by_path = {item.path: item for item in archive.files} + self.assertEqual( + by_path["rlm-bsl-index"].sha256, + by_path["rlm-bsl-mcp"].sha256, + ) + self.assertIn("libpython3.12.dylib", by_path) + self.assertIn("package/data.json", by_path) + self.assertEqual(archive.manifest.builder, result.builder_identity) + + def test_windows_build_uses_exe_names_and_utf8_entrypoints(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + manifest, target_key = self.make_manifest(root, target="win-x64") + source_dir = root / "source" + source_dir.mkdir() + source = PreparedSource(source_dir, manifest.source.commit, "b" * 40, ()) + + def fake_runner(command: list[str], *, cwd=None, env=None) -> str: + if command == ["uv", "--version"]: + return "uv 0.11.29" + if command[-1:] == ["--version"] and "nuitka" in command: + return "4.1.3" + if command[-1:] == ["--version"]: + return "Python 3.12.10" + if "--mode=standalone" in command: + output = Path(next(item for item in command if item.startswith("--output-dir=")).split("=", 1)[1]) + report = Path(next(item for item in command if item.startswith("--report=")).split("=", 1)[1]) + dist = output / "rlm-bsl-index.dist" + dist.mkdir(parents=True) + (dist / "rlm-bsl-index.exe").write_bytes(b"compiled") + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text( + '' + '' + "", + encoding="utf-8", + ) + return "" + + result = build_python_nuitka_standalone( + manifest, + target_key, + source, + root / "out", + root / "work", + runner=fake_runner, + entrypoint_resolver=lambda _python, name: { + "rlm-bsl-index": ("rlm_tools_bsl.cli", "main"), + "rlm-tools-bsl": ("rlm_tools_bsl.server", "main"), + }[name], + ) + + archive = validate_runtime_archive( + result.assets[0], + expected_release_tag=release_tag(manifest), + expected_source_ref=manifest.source.ref, + expected_source_commit=manifest.source.commit, + expected_target_key=target_key, + expected_target_triple=manifest.targets[target_key].target_triple, + expected_entrypoints={ + "rlm-bsl-index": "rlm-bsl-index.exe", + "rlm-bsl-mcp": "rlm-bsl-mcp.exe", + }, + ) + self.assertEqual( + {item.path for item in archive.files}, + {"rlm-bsl-index.exe", "rlm-bsl-mcp.exe"}, + ) + stubs = root / "work" / "nuitka" / "entrypoints" + for name in ("rlm-bsl-index.py", "rlm-bsl-mcp.py"): + source_text = (stubs / name).read_text(encoding="utf-8") + self.assertLess( + source_text.index("configure_utf8_stdio()"), + source_text.index("importlib.import_module"), + ) + + def test_rejects_wrong_versions_modules_and_incomplete_compiler_report(self) -> None: + self.assertEqual(parse_nuitka_version("4.1.3\nCommercial: not installed"), "4.1.3") + with self.assertRaisesRegex(SystemExit, "cannot parse Nuitka version"): + parse_nuitka_version("Nuitka unknown") + + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + report = root / "report.xml" + report.write_text( + '' + "" + "", + encoding="utf-8", + ) + with self.assertRaisesRegex(SystemExit, "compiler identity"): + parse_nuitka_report(report, expected_nuitka="4.1.3") + + manifest, target_key = self.make_manifest(root) + source_dir = root / "source" + source_dir.mkdir() + source = PreparedSource(source_dir, manifest.source.commit, "b" * 40, ()) + + def wrong_version(command: list[str], *, cwd=None, env=None) -> str: + if command == ["uv", "--version"]: + return "uv 0.11.29" + if command[-1:] == ["--version"] and "nuitka" in command: + return "4.1.2" + if command[-1:] == ["--version"]: + return "Python 3.12.10" + return "" + + with self.assertRaisesRegex(SystemExit, "Nuitka 4.1.2, expected 4.1.3"): + build_python_nuitka_standalone( + manifest, + target_key, + source, + root / "out", + root / "work-version", + runner=wrong_version, + entrypoint_resolver=lambda _python, _name: ("ignored", "main"), + ) + + with self.assertRaisesRegex( + SystemExit, + "rlm-bsl-index resolves to other.module, expected rlm_tools_bsl.cli", + ): + build_python_nuitka_standalone( + manifest, + target_key, + source, + root / "out-module", + root / "work-module", + runner=lambda command, cwd=None, env=None: ( + "uv 0.11.29" + if command == ["uv", "--version"] + else "4.1.3" + if command[-1:] == ["--version"] and "nuitka" in command + else "Python 3.12.10" + if command[-1:] == ["--version"] + else "" + ), + entrypoint_resolver=lambda _python, _name: ("other.module", "main"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/toolchain/builders/python_nuitka_standalone.py b/toolchain/builders/python_nuitka_standalone.py new file mode 100644 index 0000000..b6b4a80 --- /dev/null +++ b/toolchain/builders/python_nuitka_standalone.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import os +import re +import shutil +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +from toolchain.builders.python_pyinstaller import ( + parse_python_version, + parse_uv_version, + resolve_entrypoint, + run_command, + write_entrypoint_stub, +) +from toolchain.manifest import ( + PythonNuitkaStandaloneSpec, + ToolManifest, + release_tag, +) +from toolchain.runtime_archive import write_runtime_archive +from toolchain.source import PreparedSource + + +Runner = Callable[..., str] +EntrypointResolver = Callable[[Path, str], tuple[str, str]] +NUITKA_VERSION = re.compile(r"^([0-9]+\.[0-9]+\.[0-9]+)(?:\s|$)") + + +@dataclass(frozen=True) +class NuitkaBuildResult: + assets: tuple[Path, ...] + builder_identity: dict[str, Any] + + +def parse_nuitka_version(output: str) -> str: + match = NUITKA_VERSION.match(output.strip()) + if match is None: + raise SystemExit(f"cannot parse Nuitka version: {output.strip()}") + return match.group(1) + + +def parse_nuitka_report( + path: Path, + *, + expected_nuitka: str, +) -> dict[str, str]: + try: + root = ET.parse(path).getroot() + except (OSError, ET.ParseError) as exc: + raise SystemExit(f"cannot read Nuitka report: {exc}") from exc + if root.tag != "nuitka-compilation-report": + raise SystemExit(f"unexpected Nuitka report root: {root.tag}") + report_version = root.get("nuitka_version") + if report_version != expected_nuitka: + raise SystemExit( + f"Nuitka report version is {report_version}, expected {expected_nuitka}" + ) + environment = root.find("scons_environment") + attributes = ( + environment.attrib if environment is not None else {} + ) + required = ("c_compiler", "the_cc_name", "the_compiler") + if any(not attributes.get(name) for name in required): + raise SystemExit("Nuitka report is missing compiler identity") + return { + "cCompiler": attributes["c_compiler"], + "ccName": attributes["the_cc_name"], + "compiler": attributes["the_compiler"], + } + + +def _verify_identity( + *, + python_version: str, + uv_version: str, + nuitka_version: str, + builder: PythonNuitkaStandaloneSpec, +) -> None: + checks = ( + ("Python", python_version, builder.python_version), + ("uv", uv_version, builder.uv_version), + ("Nuitka", nuitka_version, builder.nuitka_version), + ) + for name, actual, expected in checks: + if actual != expected: + raise SystemExit(f"builder uses {name} {actual}, expected {expected}") + + +def _copy_dist_payload( + *, + dist_dir: Path, + compiled_executable: Path, + payload_root: Path, + entrypoint_paths: set[str], +) -> None: + if not dist_dir.is_dir() or dist_dir.is_symlink(): + raise SystemExit(f"Nuitka standalone directory not found: {dist_dir}") + payload_root.mkdir(parents=True) + for source_path in sorted(dist_dir.rglob("*"), key=lambda item: item.as_posix()): + relative = source_path.relative_to(dist_dir).as_posix() + if source_path.is_symlink(): + raise SystemExit(f"Nuitka output must not contain links: {relative}") + if source_path.is_dir(): + continue + if not source_path.is_file(): + raise SystemExit(f"Nuitka output must contain ordinary files: {relative}") + if source_path == compiled_executable: + continue + if relative in entrypoint_paths: + raise SystemExit(f"Nuitka output collides with entrypoint: {relative}") + destination = payload_root.joinpath(*relative.split("/")) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, destination) + + +def build_python_nuitka_standalone( + manifest: ToolManifest, + target_key: str, + source: PreparedSource, + out_dir: Path, + work_dir: Path, + *, + runner: Runner = run_command, + entrypoint_resolver: EntrypointResolver = resolve_entrypoint, +) -> NuitkaBuildResult: + if not isinstance(manifest.builder, PythonNuitkaStandaloneSpec): + raise SystemExit(f"{manifest.name} is not a Python/Nuitka standalone tool") + if target_key not in manifest.targets: + raise SystemExit(f"unknown target {target_key}") + builder = manifest.builder + target = manifest.targets[target_key] + if len(builder.binaries) < 2: + raise SystemExit("Nuitka multidist requires at least two binaries") + + uv_version = parse_uv_version(runner(["uv", "--version"], cwd=None, env=None)) + runner( + [ + "uv", + "sync", + "--frozen", + "--no-dev", + "--directory", + str(source.path), + "--python", + sys.executable, + ], + cwd=None, + env=None, + ) + if os.name == "nt": + venv_python = source.path / ".venv" / "Scripts" / "python.exe" + else: + venv_python = source.path / ".venv" / "bin" / "python" + python_version = parse_python_version( + runner([str(venv_python), "--version"], cwd=None, env=None) + ) + runner( + [ + "uv", + "pip", + "install", + "--python", + str(venv_python), + f"Nuitka=={builder.nuitka_version}", + ], + cwd=None, + env=None, + ) + nuitka_version = parse_nuitka_version( + runner( + [str(venv_python), "-m", "nuitka", "--version"], + cwd=None, + env=None, + ) + ) + _verify_identity( + python_version=python_version, + uv_version=uv_version, + nuitka_version=nuitka_version, + builder=builder, + ) + + build_root = work_dir / "nuitka" + entrypoint_root = build_root / "entrypoints" + output_root = build_root / "output" + report = build_root / "report.xml" + entrypoint_root.mkdir(parents=True, exist_ok=True) + main_args: list[str] = [] + for binary in builder.binaries: + module, attr = entrypoint_resolver(venv_python, binary.source_name) + if module != binary.module: + raise SystemExit( + f"{binary.source_name} resolves to {module}, expected {binary.module}" + ) + stub = entrypoint_root / f"{binary.asset_base}.py" + write_entrypoint_stub( + stub, + module, + attr, + configure_utf8_stdio=target_key == "win-x64", + ) + main_args.append(f"--main={stub}") + + runner( + [ + str(venv_python), + "-m", + "nuitka", + "--mode=standalone", + "--assume-yes-for-downloads", + f"--include-package={builder.include_package}", + f"--include-package-data={builder.include_package}", + f"--output-dir={output_root}", + f"--report={report}", + *main_args, + ], + cwd=build_root, + env={**os.environ, "PYTHONHASHSEED": "0"}, + ) + compiler = parse_nuitka_report(report, expected_nuitka=builder.nuitka_version) + builder_identity: dict[str, Any] = { + "kind": builder.kind, + "python": python_version, + "uv": uv_version, + "nuitka": nuitka_version, + "compiler": compiler, + } + + first = builder.binaries[0] + dist_dir = output_root / f"{first.asset_base}.dist" + compiled_executable = dist_dir / f"{first.asset_base}{target.exe}" + if not compiled_executable.is_file() or compiled_executable.is_symlink(): + raise SystemExit(f"Nuitka output executable not found: {compiled_executable}") + payload_root = build_root / "payload" + if payload_root.exists(): + shutil.rmtree(payload_root) + entrypoints = { + binary.asset_base: f"{binary.asset_base}{target.exe}" + for binary in builder.binaries + } + _copy_dist_payload( + dist_dir=dist_dir, + compiled_executable=compiled_executable, + payload_root=payload_root, + entrypoint_paths=set(entrypoints.values()), + ) + for relative in entrypoints.values(): + destination = payload_root / relative + shutil.copyfile(compiled_executable, destination) + destination.chmod(0o755) + + out_dir.mkdir(parents=True, exist_ok=True) + archive = out_dir / f"{manifest.name}-{target_key}.tar.gz" + write_runtime_archive( + archive_path=archive, + payload_root=payload_root, + release_tag=release_tag(manifest), + source={ + "ref": manifest.source.ref, + "commit": source.commit, + "tree": source.tree, + "patches": [ + {"path": patch.path, "sha256": patch.sha256} + for patch in source.patches + ], + }, + target={"key": target_key, "triple": target.target_triple}, + entrypoints=entrypoints, + builder=builder_identity, + ) + return NuitkaBuildResult(assets=(archive,), builder_identity=builder_identity) From 154a2afc90acae527af84296471457c53e5253ca Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Fri, 14 Aug 2026 20:33:46 +0300 Subject: [PATCH 4/6] feat(toolchain): smoke archived RLM payloads --- scripts/toolchain.py | 110 +++++++++++++++-- tests/test_provenance.py | 51 +++++++- tests/test_toolchain_cli.py | 233 +++++++++++++++++++++++++++++++++++- toolchain/manifest.py | 22 +++- toolchain/provenance.py | 6 +- 5 files changed, 401 insertions(+), 21 deletions(-) diff --git a/scripts/toolchain.py b/scripts/toolchain.py index bdf5f2f..dc73aed 100644 --- a/scripts/toolchain.py +++ b/scripts/toolchain.py @@ -18,15 +18,23 @@ WINDOWS_STDIO_POLICY, build_python_pyinstaller, ) +from toolchain.builders.python_nuitka_standalone import ( # noqa: E402 + build_python_nuitka_standalone, +) from toolchain.manifest import ( # noqa: E402 CargoBuilderSpec, PythonBuilderSpec, + PythonNuitkaStandaloneSpec, ToolManifest, expected_release_files, load_manifest, release_tag, ) from toolchain.provenance import write_target_metadata # noqa: E402 +from toolchain.runtime_archive import ( # noqa: E402 + materialize_runtime_archive, + validate_runtime_archive, +) from toolchain.source import ( # noqa: E402 checkout_source, copy_license_assets, @@ -43,10 +51,33 @@ def builder_versions(manifest: ToolManifest) -> dict[str, str]: "uv": manifest.builder.uv_version, "pyinstaller": manifest.builder.pyinstaller_version, } + if isinstance(manifest.builder, PythonNuitkaStandaloneSpec): + return { + "python": manifest.builder.python_version, + "uv": manifest.builder.uv_version, + "nuitka": manifest.builder.nuitka_version, + } raise SystemExit(f"unsupported builder: {manifest.builder}") -def builder_identity(manifest: ToolManifest, target_key: str) -> dict: +def builder_identity( + manifest: ToolManifest, + target_key: str, + observed: dict | None = None, +) -> dict: + if isinstance(manifest.builder, PythonNuitkaStandaloneSpec): + if observed is None: + raise SystemExit("Nuitka builder identity must come from the build report") + expected = {"kind": manifest.builder.kind, **builder_versions(manifest)} + for key, value in expected.items(): + if observed.get(key) != value: + raise SystemExit( + f"observed Nuitka builder {key} is {observed.get(key)}, expected {value}" + ) + compiler = observed.get("compiler") + if not isinstance(compiler, dict) or not compiler: + raise SystemExit("observed Nuitka builder is missing compiler identity") + return observed identity: dict = {"kind": manifest.builder.kind, **builder_versions(manifest)} if isinstance(manifest.builder, PythonBuilderSpec) and target_key == "win-x64": identity["stdio"] = dict(WINDOWS_STDIO_POLICY) @@ -112,20 +143,65 @@ def validate_source( ) -def _smoke(manifest: ToolManifest, target_key: str, assets: list[Path]) -> None: - by_name = {path.name: path for path in assets} +def _smoke( + manifest: ToolManifest, + target_key: str, + assets: list[Path], + *, + smoke_root: Path | None = None, + expected_builder_identity: dict | None = None, +) -> None: target = manifest.targets[target_key] + if isinstance(manifest.builder, PythonNuitkaStandaloneSpec): + if len(assets) != 1: + raise SystemExit( + f"Nuitka build must produce one archive, got {[path.name for path in assets]}" + ) + if smoke_root is None: + raise SystemExit("Nuitka archive smoke requires an isolated destination") + expected_entrypoints = { + binary.asset_base: f"{binary.asset_base}{target.exe}" + for binary in manifest.builder.binaries + } + validated = validate_runtime_archive( + assets[0], + expected_release_tag=release_tag(manifest), + expected_source_ref=manifest.source.ref, + expected_source_commit=manifest.source.commit, + expected_target_key=target_key, + expected_target_triple=target.target_triple, + expected_entrypoints=expected_entrypoints, + ) + if validated.manifest.builder != expected_builder_identity: + raise SystemExit( + "archive builder identity mismatch: " + f"{validated.manifest.builder} != {expected_builder_identity}" + ) + materialized = materialize_runtime_archive(validated, smoke_root) + by_base = { + name: materialized[relative] + for name, relative in validated.manifest.entrypoints.items() + } + else: + by_name = {path.name: path for path in assets} + by_base = { + binary.asset_base: by_name[ + f"{binary.asset_base}-{target_key}{target.exe}" + ] + for binary in manifest.builder.binaries + } for binary in manifest.builder.binaries: - name = f"{binary.asset_base}-{target_key}{target.exe}" + executable = by_base[binary.asset_base] + name = executable.name if not binary.smoke_checks: - subprocess.run([str(by_name[name]), *binary.smoke_args], check=True) + subprocess.run([str(executable), *binary.smoke_args], check=True) continue checks = [(binary.smoke_args, ())] checks.extend( (check.args, check.expected_output) for check in binary.smoke_checks ) for args, expected_output in checks: - command = [str(by_name[name]), *args] + command = [str(executable), *args] result = subprocess.run( command, check=False, @@ -168,18 +244,36 @@ def build( prepared = _prepare(manifest, repo_root, work_dir) if isinstance(manifest.builder, CargoBuilderSpec): assets = build_cargo(manifest, target_key, prepared, out_dir, work_dir) + observed_identity = None elif isinstance(manifest.builder, PythonBuilderSpec): assets = build_python_pyinstaller(manifest, target_key, prepared, out_dir, work_dir) + observed_identity = None + elif isinstance(manifest.builder, PythonNuitkaStandaloneSpec): + result = build_python_nuitka_standalone( + manifest, target_key, prepared, out_dir, work_dir + ) + assets = list(result.assets) + observed_identity = result.builder_identity else: raise SystemExit(f"unsupported builder: {manifest.builder}") - _smoke(manifest, target_key, assets) + _smoke( + manifest, + target_key, + assets, + smoke_root=(work_dir / "smoke-runtime") + if isinstance(manifest.builder, PythonNuitkaStandaloneSpec) + else None, + expected_builder_identity=observed_identity, + ) write_target_metadata( manifest, target_key, prepared, assets, out_dir, - builder_identity=builder_identity(manifest, target_key), + builder_identity=builder_identity( + manifest, target_key, observed=observed_identity + ), ) diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 3cb9464..0615ac7 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -6,7 +6,7 @@ import unittest from pathlib import Path -from tests.test_manifest import cargo_manifest +from tests.test_manifest import cargo_manifest, python_nuitka_manifest from toolchain.manifest import load_manifest from toolchain.provenance import write_target_metadata from toolchain.source import AppliedPatch, PreparedSource @@ -56,6 +56,55 @@ def test_records_source_patches_builder_target_and_assets_with_lf(self) -> None: self.assertNotIn(b"\r\n", checksums_path.read_bytes()) self.assertIn(asset.name.encode(), checksums_path.read_bytes()) + def test_nuitka_provenance_hashes_only_the_target_archive(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + manifest_data = python_nuitka_manifest() + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(manifest_data), encoding="utf-8") + manifest = load_manifest(manifest_path) + source = PreparedSource(root / "source", "a" * 40, "b" * 40, ()) + out_dir = root / "out" + out_dir.mkdir() + archive = out_dir / "rlm-tools-bsl-linux-x64.tar.gz" + archive.write_bytes(b"verified archive") + identity = { + "kind": "python-nuitka-standalone", + "python": "3.12.10", + "uv": "0.11.29", + "nuitka": "4.1.3", + "compiler": { + "cCompiler": "GCC", + "ccName": "gcc", + "compiler": "gcc", + }, + } + + provenance_path, checksums_path = write_target_metadata( + manifest, + "linux-x64", + source, + [archive], + out_dir, + builder_identity=identity, + ) + + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + self.assertEqual(provenance["builder"], identity) + self.assertEqual( + provenance["assets"], + [ + { + "name": archive.name, + "sha256": hashlib.sha256(b"verified archive").hexdigest(), + "size": len(b"verified archive"), + } + ], + ) + self.assertEqual( + checksums_path.read_text(encoding="utf-8"), + f"{hashlib.sha256(b'verified archive').hexdigest()} {archive.name}\n", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_toolchain_cli.py b/tests/test_toolchain_cli.py index 8a7b168..60c9305 100644 --- a/tests/test_toolchain_cli.py +++ b/tests/test_toolchain_cli.py @@ -11,8 +11,10 @@ from contextlib import redirect_stdout from unittest.mock import Mock, patch -from tests.test_manifest import cargo_manifest, python_manifest -from toolchain.manifest import load_manifest +from tests.test_manifest import cargo_manifest, python_manifest, python_nuitka_manifest +from toolchain.builders.python_nuitka_standalone import NuitkaBuildResult +from toolchain.manifest import load_manifest, release_tag +from toolchain.runtime_archive import write_runtime_archive from toolchain.source import PreparedSource @@ -276,6 +278,233 @@ def test_python_builder_identity_records_only_applied_windows_stdio_policy(self) }, ) + def test_nuitka_build_smokes_extracted_archive_before_writing_metadata(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + data = python_nuitka_manifest() + data["builder"]["binaries"] = [ + { + "package": "rlm_tools_bsl", + "sourceName": "rlm-bsl-index", + "module": "rlm_tools_bsl.cli", + "assetBase": "rlm-bsl-index", + "smokeArgs": ["--help"], + "smokeChecks": [ + { + "args": ["index", "build", "--help"], + "expectedOutput": ["Строить неполный индекс"], + }, + { + "args": ["index", "update", "--help"], + "expectedOutput": ["usage: rlm-bsl-index index update"], + }, + { + "args": ["index", "info", "--help"], + "expectedOutput": ["usage: rlm-bsl-index index info"], + }, + ], + }, + { + "package": "rlm_tools_bsl", + "sourceName": "rlm-tools-bsl", + "module": "rlm_tools_bsl.server", + "assetBase": "rlm-bsl-mcp", + "smokeArgs": ["--help"], + }, + ] + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(data), encoding="utf-8") + manifest = load_manifest(manifest_path) + source = PreparedSource(root / "source", manifest.source.commit, "b" * 40, ()) + source.path.mkdir() + payload = root / "payload" + payload.mkdir() + calls = root / "calls.txt" + executable = ( + "#!/usr/bin/env python3\n" + "from pathlib import Path\n" + "import sys\n" + f"log = Path({str(calls)!r})\n" + "with log.open('a', encoding='utf-8') as stream:\n" + " stream.write(Path(sys.argv[0]).name + ' ' + ' '.join(sys.argv[1:]) + '\\n')\n" + "name = Path(sys.argv[0]).name\n" + "args = sys.argv[1:]\n" + "if name == 'rlm-bsl-mcp':\n" + " print('usage: rlm-bsl-mcp')\n" + "elif args == ['--help']:\n" + " print('usage: rlm-bsl-index')\n" + "elif args == ['index', 'build', '--help']:\n" + " print('Строить неполный индекс')\n" + "elif args == ['index', 'update', '--help']:\n" + " print('usage: rlm-bsl-index index update')\n" + "elif args == ['index', 'info', '--help']:\n" + " print('usage: rlm-bsl-index index info')\n" + "else:\n" + " raise SystemExit(9)\n" + ).encode() + for name in ("rlm-bsl-index", "rlm-bsl-mcp"): + path = payload / name + path.write_bytes(executable) + path.chmod(0o755) + archive = root / "out" / "rlm-tools-bsl-darwin-arm64.tar.gz" + builder_identity = { + "kind": "python-nuitka-standalone", + "python": "3.12.10", + "uv": "0.11.29", + "nuitka": "4.1.3", + "compiler": { + "cCompiler": "Clang", + "ccName": "clang", + "compiler": "clang", + }, + } + write_runtime_archive( + archive_path=archive, + payload_root=payload, + release_tag=release_tag(manifest), + source={ + "ref": manifest.source.ref, + "commit": source.commit, + "tree": source.tree, + "patches": [], + }, + target={ + "key": "darwin-arm64", + "triple": manifest.targets["darwin-arm64"].target_triple, + }, + entrypoints={ + "rlm-bsl-index": "rlm-bsl-index", + "rlm-bsl-mcp": "rlm-bsl-mcp", + }, + builder=builder_identity, + ) + module = load_script() + metadata = Mock() + + with ( + patch.object(module, "_prepare", return_value=source), + patch.object( + module, + "build_python_nuitka_standalone", + return_value=NuitkaBuildResult((archive,), builder_identity), + create=True, + ), + patch.object(module, "write_target_metadata", metadata), + ): + module.build( + manifest, + root, + "darwin-arm64", + root / "work", + root / "out", + ) + + self.assertEqual( + calls.read_text(encoding="utf-8").splitlines(), + [ + "rlm-bsl-index --help", + "rlm-bsl-index index build --help", + "rlm-bsl-index index update --help", + "rlm-bsl-index index info --help", + "rlm-bsl-mcp --help", + ], + ) + metadata.assert_called_once() + self.assertEqual(metadata.call_args.args[3], [archive]) + self.assertEqual( + metadata.call_args.kwargs["builder_identity"], builder_identity + ) + + def test_nuitka_build_rejects_corrupt_archive_before_metadata(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + data = python_nuitka_manifest() + data["builder"]["binaries"] = [ + dict(data["builder"]["binaries"][0], sourceName="rlm-bsl-index", module="rlm_tools_bsl.cli", assetBase="rlm-bsl-index"), + dict(data["builder"]["binaries"][0], sourceName="rlm-tools-bsl", module="rlm_tools_bsl.server", assetBase="rlm-bsl-mcp"), + ] + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(data), encoding="utf-8") + manifest = load_manifest(manifest_path) + source = PreparedSource(root / "source", manifest.source.commit, "b" * 40, ()) + archive = root / "out" / "rlm-tools-bsl-darwin-arm64.tar.gz" + archive.parent.mkdir() + archive.write_bytes(b"not a tar archive") + identity = { + "kind": "python-nuitka-standalone", + "python": "3.12.10", + "uv": "0.11.29", + "nuitka": "4.1.3", + "compiler": {"cCompiler": "Clang", "ccName": "clang", "compiler": "clang"}, + } + module = load_script() + metadata = Mock() + + with ( + patch.object(module, "_prepare", return_value=source), + patch.object( + module, + "build_python_nuitka_standalone", + return_value=NuitkaBuildResult((archive,), identity), + create=True, + ), + patch.object(module, "write_target_metadata", metadata), + ): + with self.assertRaisesRegex(SystemExit, "cannot read runtime archive"): + module.build(manifest, root, "darwin-arm64", root / "work", root / "out") + + metadata.assert_not_called() + + def test_nuitka_build_rejects_archive_builder_identity_drift_before_metadata(self) -> None: + root = Path(self.enterContext(tempfile.TemporaryDirectory())) + data = python_nuitka_manifest() + data["builder"]["binaries"] = [ + dict(data["builder"]["binaries"][0], sourceName="rlm-bsl-index", module="rlm_tools_bsl.cli", assetBase="rlm-bsl-index"), + dict(data["builder"]["binaries"][0], sourceName="rlm-tools-bsl", module="rlm_tools_bsl.server", assetBase="rlm-bsl-mcp"), + ] + manifest_path = root / "manifest.json" + manifest_path.write_text(json.dumps(data), encoding="utf-8") + manifest = load_manifest(manifest_path) + source = PreparedSource(root / "source", manifest.source.commit, "b" * 40, ()) + payload = root / "payload" + payload.mkdir() + for name in ("rlm-bsl-index", "rlm-bsl-mcp"): + executable = payload / name + executable.write_bytes(b"#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + archive = root / "out" / "rlm-tools-bsl-darwin-arm64.tar.gz" + write_runtime_archive( + archive_path=archive, + payload_root=payload, + release_tag=release_tag(manifest), + source={"ref": manifest.source.ref, "commit": source.commit, "tree": source.tree, "patches": []}, + target={"key": "darwin-arm64", "triple": manifest.targets["darwin-arm64"].target_triple}, + entrypoints={"rlm-bsl-index": "rlm-bsl-index", "rlm-bsl-mcp": "rlm-bsl-mcp"}, + builder={"kind": "tampered-builder"}, + ) + observed = { + "kind": "python-nuitka-standalone", + "python": "3.12.10", + "uv": "0.11.29", + "nuitka": "4.1.3", + "compiler": {"cCompiler": "Clang", "ccName": "clang", "compiler": "clang"}, + } + module = load_script() + metadata = Mock() + + with ( + patch.object(module, "_prepare", return_value=source), + patch.object( + module, + "build_python_nuitka_standalone", + return_value=NuitkaBuildResult((archive,), observed), + create=True, + ), + patch.object(module, "write_target_metadata", metadata), + ): + with self.assertRaisesRegex(SystemExit, "archive builder identity mismatch"): + module.build(manifest, root, "darwin-arm64", root / "work", root / "out") + + metadata.assert_not_called() + def test_main_resolves_relative_work_and_output_paths_before_build(self) -> None: root = Path(self.enterContext(tempfile.TemporaryDirectory())) manifest_path = self.write_manifest(root) diff --git a/toolchain/manifest.py b/toolchain/manifest.py index c5d7414..a5462d9 100644 --- a/toolchain/manifest.py +++ b/toolchain/manifest.py @@ -453,19 +453,29 @@ def release_tag(manifest: ToolManifest) -> str: ) -def expected_asset_names(manifest: ToolManifest) -> set[str]: +def expected_target_asset_names( + manifest: ToolManifest, + target_key: str, +) -> set[str]: + if target_key not in manifest.targets: + raise SystemExit(f"unknown target {target_key}") if isinstance(manifest.builder, PythonNuitkaStandaloneSpec): - return { - f"{manifest.name}-{target_key}.tar.gz" - for target_key in manifest.targets - } + return {f"{manifest.name}-{target_key}.tar.gz"} + target = manifest.targets[target_key] return { f"{binary.asset_base}-{target_key}{target.exe}" - for target_key, target in manifest.targets.items() for binary in manifest.builder.binaries } +def expected_asset_names(manifest: ToolManifest) -> set[str]: + return { + asset + for target_key in manifest.targets + for asset in expected_target_asset_names(manifest, target_key) + } + + def expected_release_files(manifest: ToolManifest) -> set[str]: result = expected_asset_names(manifest) result.update(item.asset_name for item in manifest.license.files) diff --git a/toolchain/provenance.py b/toolchain/provenance.py index 6a701ae..67b65bf 100644 --- a/toolchain/provenance.py +++ b/toolchain/provenance.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any -from toolchain.manifest import ToolManifest, release_tag +from toolchain.manifest import ToolManifest, expected_target_asset_names, release_tag from toolchain.source import PreparedSource, sha256 @@ -20,9 +20,7 @@ def write_target_metadata( if target_key not in manifest.targets: raise SystemExit(f"unknown target {target_key}") target = manifest.targets[target_key] - expected = { - f"{binary.asset_base}-{target_key}{target.exe}" for binary in manifest.builder.binaries - } + expected = expected_target_asset_names(manifest, target_key) actual = {asset.name for asset in assets} if actual != expected: raise SystemExit( From f23bb5ae1c15df4a6e3f339662616b0e7172c0a0 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Fri, 14 Aug 2026 20:50:23 +0300 Subject: [PATCH 5/6] build(rlm): select Nuitka standalone archives --- .github/workflows/ci.yml | 24 ++++++++++----- .github/workflows/release-tool.yml | 10 +++++-- manifests/rlm-tools-bsl.json | 22 +++++++------- tests/test_python_nuitka_standalone.py | 9 ++++++ tests/test_repository_contract.py | 30 ++++++++++++------- tests/test_rlm_manifest.py | 25 ++++++++-------- .../builders/python_nuitka_standalone.py | 1 + 7 files changed, 78 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 595183b..a2ba417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,10 +39,20 @@ jobs: - name: Validate GitHub Actions syntax run: docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:1.7.7 - rlm-windows-frozen-smoke: + rlm-standalone-smoke: needs: test - runs-on: windows-latest - timeout-minutes: 30 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: darwin-arm64 + runner: macos-14 + - target: linux-x64 + runner: ubuntu-latest + - target: win-x64 + runner: windows-latest steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v6 @@ -52,11 +62,11 @@ jobs: with: version: "0.11.29" enable-cache: false - - name: Build and smoke the frozen Windows RLM CLIs + - name: Build, extract, and smoke the frozen RLM archive shell: bash run: >- python scripts/toolchain.py build --manifest manifests/rlm-tools-bsl.json --repo-root . - --target win-x64 - --work-dir .build/ci-rlm-tools-bsl-win-x64 - --out-dir dist/ci-rlm-tools-bsl-win-x64 + --target "${{ matrix.target }}" + --work-dir ".build/ci-rlm-tools-bsl-${{ matrix.target }}" + --out-dir "dist/ci-rlm-tools-bsl-${{ matrix.target }}" diff --git a/.github/workflows/release-tool.yml b/.github/workflows/release-tool.yml index b2d1bba..99e2975 100644 --- a/.github/workflows/release-tool.yml +++ b/.github/workflows/release-tool.yml @@ -93,7 +93,7 @@ jobs: build: needs: metadata runs-on: ${{ matrix.runner }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: ${{ fromJSON(needs.metadata.outputs.matrix) }} @@ -103,7 +103,7 @@ jobs: with: python-version: ${{ needs.metadata.outputs.python_version }} - uses: astral-sh/setup-uv@v7 - if: needs.metadata.outputs.builder_kind == 'python-pyinstaller' + if: needs.metadata.outputs.builder_kind == 'python-pyinstaller' || needs.metadata.outputs.builder_kind == 'python-nuitka-standalone' with: version: ${{ needs.metadata.outputs.uv_version }} enable-cache: false @@ -192,12 +192,18 @@ jobs: exit 1 fi - name: Attest native executables + if: needs.metadata.outputs.builder_kind != 'python-nuitka-standalone' uses: actions/attest-build-provenance@v2 with: subject-path: | dist/*-darwin-arm64 dist/*-linux-x64 dist/*-win-x64.exe + - name: Attest standalone runtime archives + if: needs.metadata.outputs.builder_kind == 'python-nuitka-standalone' + uses: actions/attest-build-provenance@v2 + with: + subject-path: dist/*.tar.gz - uses: softprops/action-gh-release@v3 with: tag_name: ${{ needs.metadata.outputs.release_tag }} diff --git a/manifests/rlm-tools-bsl.json b/manifests/rlm-tools-bsl.json index d70bbb1..2d57a3a 100644 --- a/manifests/rlm-tools-bsl.json +++ b/manifests/rlm-tools-bsl.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "name": "rlm-tools-bsl", "version": "1.33.0", - "buildRevision": 2, + "buildRevision": 3, "source": { "kind": "release", "repository": "https://github.com/Dach-Coin/rlm-tools-bsl", @@ -20,20 +20,13 @@ }, "patches": [], "builder": { - "kind": "python-pyinstaller", + "kind": "python-nuitka-standalone", "pythonVersion": "3.12.10", "uvVersion": "0.11.29", - "pyinstallerVersion": "6.21.0", + "nuitkaVersion": "4.1.3", "lockFile": "uv.lock", - "collectAll": "rlm_tools_bsl", + "includePackage": "rlm_tools_bsl", "binaries": [ - { - "package": "rlm_tools_bsl", - "sourceName": "rlm-tools-bsl", - "module": "rlm_tools_bsl.server", - "assetBase": "rlm-bsl-mcp", - "smokeArgs": ["--help"] - }, { "package": "rlm_tools_bsl", "sourceName": "rlm-bsl-index", @@ -57,6 +50,13 @@ "expectedOutput": ["usage: rlm-bsl-index index info"] } ] + }, + { + "package": "rlm_tools_bsl", + "sourceName": "rlm-tools-bsl", + "module": "rlm_tools_bsl.server", + "assetBase": "rlm-bsl-mcp", + "smokeArgs": ["--help"] } ] }, diff --git a/tests/test_python_nuitka_standalone.py b/tests/test_python_nuitka_standalone.py index 336bf56..484ef07 100644 --- a/tests/test_python_nuitka_standalone.py +++ b/tests/test_python_nuitka_standalone.py @@ -112,6 +112,7 @@ def fake_runner(command: list[str], *, cwd=None, env=None) -> str: self.assertTrue(main_args[1].endswith("/rlm-bsl-mcp.py")) self.assertIn("--include-package=rlm_tools_bsl", compile_command) self.assertIn("--include-package-data=rlm_tools_bsl", compile_command) + self.assertIn("--output-filename=rlm-bsl-index", compile_command) self.assertIn("--assume-yes-for-downloads", compile_command) self.assertFalse(any("onefile" in item for item in compile_command)) self.assertFalse(any("tempdir" in item for item in compile_command)) @@ -158,7 +159,10 @@ def test_windows_build_uses_exe_names_and_utf8_entrypoints(self) -> None: source_dir.mkdir() source = PreparedSource(source_dir, manifest.source.commit, "b" * 40, ()) + commands: list[list[str]] = [] + def fake_runner(command: list[str], *, cwd=None, env=None) -> str: + commands.append(command) if command == ["uv", "--version"]: return "uv 0.11.29" if command[-1:] == ["--version"] and "nuitka" in command: @@ -194,6 +198,11 @@ def fake_runner(command: list[str], *, cwd=None, env=None) -> str: }[name], ) + compile_command = next( + command for command in commands if "--mode=standalone" in command + ) + self.assertIn("--output-filename=rlm-bsl-index.exe", compile_command) + archive = validate_runtime_archive( result.assets[0], expected_release_tag=release_tag(manifest), diff --git a/tests/test_repository_contract.py b/tests/test_repository_contract.py index d614238..eba7a90 100644 --- a/tests/test_repository_contract.py +++ b/tests/test_repository_contract.py @@ -5,7 +5,7 @@ from toolchain.manifest import ( CargoBuilderSpec, - PythonBuilderSpec, + PythonNuitkaStandaloneSpec, expected_asset_names, load_manifest, release_tag, @@ -26,7 +26,7 @@ def test_checked_in_tools_have_independent_release_identities(self) -> None: "release", "v1.33.0", "3e6920cd015a61af4ba7aa1a5f1fedd8bc935549", - "rlm-tools-bsl-v1.33.0-build.2", + "rlm-tools-bsl-v1.33.0-build.3", ), "bsl-analyzer": ( "release", @@ -84,18 +84,19 @@ def test_cargo_tools_pin_rust_and_normalize_assets(self) -> None: }, ) - def test_rlm_pins_python_builder_and_both_entrypoints(self) -> None: + def test_rlm_pins_nuitka_builder_and_one_archive_per_target(self) -> None: manifest = self.load("rlm-tools-bsl") - self.assertIsInstance(manifest.builder, PythonBuilderSpec) + self.assertIsInstance(manifest.builder, PythonNuitkaStandaloneSpec) self.assertEqual(manifest.builder.python_version, "3.12.10") self.assertEqual(manifest.builder.uv_version, "0.11.29") - self.assertEqual(manifest.builder.pyinstaller_version, "6.21.0") + self.assertEqual(manifest.builder.nuitka_version, "4.1.3") + self.assertEqual(manifest.builder.include_package, "rlm_tools_bsl") self.assertEqual( [binary.module for binary in manifest.builder.binaries], - ["rlm_tools_bsl.server", "rlm_tools_bsl.cli"], + ["rlm_tools_bsl.cli", "rlm_tools_bsl.server"], ) - self.assertEqual(len(expected_asset_names(manifest)), 6) + self.assertEqual(len(expected_asset_names(manifest)), 3) def test_rlm_only_implementation_is_removed(self) -> None: self.assertFalse((REPO_ROOT / "scripts" / "build_rlm.py").exists()) @@ -122,6 +123,8 @@ def test_one_generic_release_workflow_builds_only_on_dispatch(self) -> None: self.assertIn("refs/tags/", text) self.assertIn("expected_release_files", text) self.assertIn("actions/attest-build-provenance@v2", text) + self.assertIn("dist/*.tar.gz", text) + self.assertIn("python-nuitka-standalone", text) self.assertIn("softprops/action-gh-release@v3", text) self.assertIn("make_latest: false", text) @@ -135,10 +138,14 @@ def test_pull_request_ci_validates_sources_without_release_operations(self) -> N self.assertNotIn("softprops/action-gh-release", text) self.assertNotIn("gh release", text) - def test_pull_request_ci_builds_and_smokes_frozen_rlm_on_windows(self) -> None: + def test_pull_request_ci_builds_and_smokes_frozen_rlm_archives_on_three_targets(self) -> None: text = (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") - self.assertIn("rlm-windows-frozen-smoke:", text) - self.assertIn("runs-on: windows-latest", text) + self.assertIn("rlm-standalone-smoke:", text) + self.assertIn("runs-on: ${{ matrix.runner }}", text) + self.assertIn("fail-fast: false", text) + self.assertIn("macos-14", text) + self.assertIn("ubuntu-latest", text) + self.assertIn("windows-latest", text) self.assertIn('python-version: "3.12.10"', text) self.assertIn("uses: astral-sh/setup-uv@v7", text) self.assertIn('version: "0.11.29"', text) @@ -146,7 +153,8 @@ def test_pull_request_ci_builds_and_smokes_frozen_rlm_on_windows(self) -> None: "python scripts/toolchain.py build --manifest manifests/rlm-tools-bsl.json", text, ) - self.assertIn("--target win-x64", text) + self.assertIn('--target "${{ matrix.target }}"', text) + self.assertNotIn("softprops/action-gh-release", text) if __name__ == "__main__": diff --git a/tests/test_rlm_manifest.py b/tests/test_rlm_manifest.py index c8935af..6489cc1 100644 --- a/tests/test_rlm_manifest.py +++ b/tests/test_rlm_manifest.py @@ -2,7 +2,7 @@ import unittest from toolchain.manifest import ( - PythonBuilderSpec, + PythonNuitkaStandaloneSpec, expected_asset_names, expected_release_files, load_manifest, @@ -17,17 +17,17 @@ class RlmManifestTests(unittest.TestCase): def test_v1_33_release_keeps_upstream_entrypoints_and_renames_assets(self) -> None: manifest = load_manifest(REPO_ROOT / "manifests" / "rlm-tools-bsl.json") - self.assertIsInstance(manifest.builder, PythonBuilderSpec) + self.assertIsInstance(manifest.builder, PythonNuitkaStandaloneSpec) self.assertEqual(manifest.name, "rlm-tools-bsl") self.assertEqual(manifest.version, "1.33.0") - self.assertEqual(manifest.build_revision, 2) + self.assertEqual(manifest.build_revision, 3) self.assertEqual(manifest.source.kind, "release") self.assertEqual(manifest.source.ref, "v1.33.0") self.assertEqual( manifest.source.commit, "3e6920cd015a61af4ba7aa1a5f1fedd8bc935549", ) - self.assertEqual(release_tag(manifest), "rlm-tools-bsl-v1.33.0-build.2") + self.assertEqual(release_tag(manifest), "rlm-tools-bsl-v1.33.0-build.3") self.assertEqual(manifest.patches, ()) self.assertEqual( @@ -36,11 +36,15 @@ def test_v1_33_release_keeps_upstream_entrypoints_and_renames_assets(self) -> No for binary in manifest.builder.binaries ], [ - ("rlm-tools-bsl", "rlm-bsl-mcp", "rlm_tools_bsl", "rlm_tools_bsl.server"), ("rlm-bsl-index", "rlm-bsl-index", "rlm_tools_bsl", "rlm_tools_bsl.cli"), + ("rlm-tools-bsl", "rlm-bsl-mcp", "rlm_tools_bsl", "rlm_tools_bsl.server"), ], ) - index = manifest.builder.binaries[1] + self.assertEqual(manifest.builder.python_version, "3.12.10") + self.assertEqual(manifest.builder.uv_version, "0.11.29") + self.assertEqual(manifest.builder.nuitka_version, "4.1.3") + self.assertEqual(manifest.builder.include_package, "rlm_tools_bsl") + index = manifest.builder.binaries[0] self.assertEqual(index.smoke_args, ("--help",)) self.assertEqual( [(check.args, check.expected_output) for check in index.smoke_checks], @@ -62,12 +66,9 @@ def test_v1_33_release_keeps_upstream_entrypoints_and_renames_assets(self) -> No self.assertEqual( expected_asset_names(manifest), { - "rlm-bsl-mcp-darwin-arm64", - "rlm-bsl-mcp-linux-x64", - "rlm-bsl-mcp-win-x64.exe", - "rlm-bsl-index-darwin-arm64", - "rlm-bsl-index-linux-x64", - "rlm-bsl-index-win-x64.exe", + "rlm-tools-bsl-darwin-arm64.tar.gz", + "rlm-tools-bsl-linux-x64.tar.gz", + "rlm-tools-bsl-win-x64.tar.gz", }, ) self.assertEqual( diff --git a/toolchain/builders/python_nuitka_standalone.py b/toolchain/builders/python_nuitka_standalone.py index b6b4a80..fc16bec 100644 --- a/toolchain/builders/python_nuitka_standalone.py +++ b/toolchain/builders/python_nuitka_standalone.py @@ -212,6 +212,7 @@ def build_python_nuitka_standalone( "nuitka", "--mode=standalone", "--assume-yes-for-downloads", + f"--output-filename={builder.binaries[0].asset_base}{target.exe}", f"--include-package={builder.include_package}", f"--include-package-data={builder.include_package}", f"--output-dir={output_root}", From 48378b14d57978c4aad8ff39c6e8d1b2576db479 Mon Sep 17 00:00:00 2001 From: Igor Apresov Date: Fri, 14 Aug 2026 21:21:56 +0300 Subject: [PATCH 6/6] ci(rlm): expose standalone provenance evidence --- .github/workflows/ci.yml | 6 ++++++ tests/test_repository_contract.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2ba417..bd6fa9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,3 +70,9 @@ jobs: --target "${{ matrix.target }}" --work-dir ".build/ci-rlm-tools-bsl-${{ matrix.target }}" --out-dir "dist/ci-rlm-tools-bsl-${{ matrix.target }}" + - name: Emit target provenance and checksum + shell: bash + run: | + set -euo pipefail + python -m json.tool "dist/ci-rlm-tools-bsl-${{ matrix.target }}/provenance-rlm-tools-bsl-${{ matrix.target }}.json" + cat "dist/ci-rlm-tools-bsl-${{ matrix.target }}/checksums-rlm-tools-bsl-${{ matrix.target }}.txt" diff --git a/tests/test_repository_contract.py b/tests/test_repository_contract.py index eba7a90..8482883 100644 --- a/tests/test_repository_contract.py +++ b/tests/test_repository_contract.py @@ -154,6 +154,15 @@ def test_pull_request_ci_builds_and_smokes_frozen_rlm_archives_on_three_targets( text, ) self.assertIn('--target "${{ matrix.target }}"', text) + self.assertIn("Emit target provenance and checksum", text) + self.assertIn( + 'python -m json.tool "dist/ci-rlm-tools-bsl-${{ matrix.target }}/provenance-rlm-tools-bsl-${{ matrix.target }}.json"', + text, + ) + self.assertIn( + 'cat "dist/ci-rlm-tools-bsl-${{ matrix.target }}/checksums-rlm-tools-bsl-${{ matrix.target }}.txt"', + text, + ) self.assertNotIn("softprops/action-gh-release", text)