diff --git a/docs/index.rst b/docs/index.rst index b067e965..8e3b4a81 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,6 +5,7 @@ self quickstart configuration + wheel_sources usage release_notes contributing diff --git a/docs/quickstart.rst b/docs/quickstart.rst index d1230f40..2a3725b7 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -65,3 +65,17 @@ To view all the instances that are produced use the ``list`` command: The ``black`` and ``mypy`` instances will be run with Python 3.9 and the ``pytest`` instance will be run in Python 3.8 and 3.9. + +Using Pre-built Wheels +---------------------- + +By default, riot installs your project in editable mode (``pip install -e .``). +If you want to test with pre-built wheels instead, use the ``--wheel-path`` option: + +.. code-block:: bash + + $ pip wheel --no-deps -w dist/ . + $ riot --wheel-path dist/ run test + +See :doc:`wheel_sources` for more details on using pre-built wheels. + diff --git a/docs/wheel_sources.rst b/docs/wheel_sources.rst new file mode 100644 index 00000000..d79978d9 --- /dev/null +++ b/docs/wheel_sources.rst @@ -0,0 +1,176 @@ +Using Pre-built Wheels +====================== + +By default, riot installs your project in editable mode (``pip install -e .``) when +creating virtual environments. However, you can configure riot to install from +pre-built wheels instead. This is useful for: + +- **CI/CD pipelines**: Using pre-built wheels from a previous build step +- **Testing distributions**: Verifying that your built wheels work correctly +- **Faster environment creation**: Avoiding repeated package builds +- **Reproducibility**: Testing with exact wheel artifacts + + +Specifying a Wheel Path +------------------------ + +There are two ways to specify a wheel path: + + +Command-line Option +~~~~~~~~~~~~~~~~~~~ + +Use the global ``--wheel-path`` option before any subcommand: + +.. code-block:: bash + + # With a local directory containing wheels + riot --wheel-path /path/to/wheels run test + + # With a remote URL (e.g., index.html) + riot --wheel-path https://example.com/wheels/ generate + + # Works with all commands + riot --wheel-path /tmp/wheels shell mypy + + +Environment Variable +~~~~~~~~~~~~~~~~~~~~ + +Set the ``RIOT_WHEEL_PATH`` environment variable: + +.. code-block:: bash + + export RIOT_WHEEL_PATH=/path/to/wheels + riot run test + +This is particularly useful in CI/CD environments where you want to configure +wheel paths without modifying commands. + + +Package Name Resolution +----------------------- + +When using wheel paths, riot needs to know the package name to install. It +determines this automatically by: + +1. **Checking the ``RIOT_PACKAGE_NAME`` environment variable** (highest priority) +2. **Parsing ``pyproject.toml``**: Reads the ``[project]`` table's ``name`` field + +For projects using ``pyproject.toml`` with a ``[project]`` section, no additional +configuration is needed: + +.. code-block:: toml + + [project] + name = "my-package" + version = "1.0.0" + +For projects not using ``pyproject.toml`` or with custom naming, set the +``RIOT_PACKAGE_NAME`` environment variable: + +.. code-block:: bash + + export RIOT_PACKAGE_NAME=my-package + export RIOT_WHEEL_PATH=/tmp/wheels + riot run test + + +How It Works +------------ + +When a wheel path is specified: + +1. **Download**: riot downloads the wheel using ``pip download --no-index --find-links`` + to ensure only wheels from the specified source are used (not PyPI) +2. **Install**: The downloaded wheel is installed into the virtual environment +3. **No Fallback**: If the wheel is not found, riot fails with a clear error message + (no fallback to editable install) + +This ensures reproducibility and prevents accidental use of incorrect package versions. + + +Example: CI/CD Workflow +----------------------- + +A typical CI/CD workflow using wheel paths: + +.. code-block:: bash + + # Step 1: Build wheels + pip wheel --no-deps -w dist/ . + + # Step 2: Run tests with built wheels + riot --wheel-path dist/ run test + + # Step 3: Verify wheels work in clean environments + riot --wheel-path dist/ generate --recreate-venvs + + +Example: Testing with Remote Wheels +------------------------------------ + +Test against wheels published to a remote location: + +.. code-block:: bash + + # Test against wheels on an S3 bucket or web server + riot --wheel-path https://artifacts.example.com/wheels/v1.2.3/ run test + +The wheel path can be any location supported by pip's ``--find-links`` option, +including: + +- Local directories (``/path/to/wheels``) +- File URLs (``file:///path/to/wheels``) +- HTTP/HTTPS URLs with index.html (``https://example.com/wheels/``) + + +Compatibility with Existing Options +------------------------------------ + +Wheel sources work with all existing riot options: + +.. code-block:: bash + + # Recreate environments with wheels + riot --wheel-path /tmp/wheels run --recreate-venvs test + + # Skip base install (wheels already installed) + riot --wheel-path /tmp/wheels run --skip-base-install test + + # Generate base environments with wheels + riot --wheel-path /tmp/wheels generate + + +Troubleshooting +--------------- + +**Wheel not found error**: + +If you see an error like "Wheel download failed", verify: + +- The wheel file exists in the specified location +- The package name matches (check ``RIOT_PACKAGE_NAME`` or ``pyproject.toml``) +- For URLs, the index.html or directory listing is accessible + +**Package name cannot be determined**: + +If you see "Could not determine package name", either: + +- Add a ``[project]`` section with ``name`` field to ``pyproject.toml`` +- Set the ``RIOT_PACKAGE_NAME`` environment variable + + +Environment Variables Reference +-------------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Variable + - Description + * - ``RIOT_WHEEL_PATH`` + - Path or URL to wheel files. When set, installs from wheels instead of editable mode. + * - ``RIOT_PACKAGE_NAME`` + - Package name to use when installing from wheels. Overrides automatic detection from ``pyproject.toml``. diff --git a/releasenotes/notes/wheel-source-support-612999b83b5b0b9b.yaml b/releasenotes/notes/wheel-source-support-612999b83b5b0b9b.yaml new file mode 100644 index 00000000..393c3c37 --- /dev/null +++ b/releasenotes/notes/wheel-source-support-612999b83b5b0b9b.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for installing from pre-built wheels instead of editable mode via the + ``--wheel-path`` global option and ``RIOT_WHEEL_PATH`` environment variable. + See https://riot.readthedocs.io/en/latest/wheel_sources.html for details. diff --git a/riot/cli.py b/riot/cli.py index 20d07cbb..6e0be99f 100644 --- a/riot/cli.py +++ b/riot/cli.py @@ -78,9 +78,17 @@ def convert(self, value, param, ctx): default=False, help="Pipe mode. Makes riot emit plain output.", ) +@click.option( + "--wheel-path", + "wheel_path", + type=str, + default=None, + envvar="RIOT_WHEEL_PATH", + help="Path or URL to wheel files. When set, installs from wheels instead of editable mode.", +) @click.version_option(__version__) @click.pass_context -def main(ctx, riotfile, log_level, pipe_mode): +def main(ctx, riotfile, log_level, pipe_mode, wheel_path): if pipe_mode: if log_level: logging.basicConfig(level=log_level) @@ -94,6 +102,7 @@ def main(ctx, riotfile, log_level, pipe_mode): ctx.ensure_object(dict) ctx.obj["pipe"] = pipe_mode + ctx.obj["wheel_path"] = wheel_path # Check if file exists first (before checking for subcommand) import os @@ -168,11 +177,13 @@ def list_venvs(ctx, pythons, pattern, venv_pattern, interpreters, hash_only): @PATTERN_ARG @click.pass_context def generate(ctx, recreate_venvs, skip_base_install, pythons, pattern): + wheel_path = ctx.obj.get("wheel_path") ctx.obj["session"].generate_base_venvs( pattern=re.compile(pattern), recreate=recreate_venvs, skip_deps=skip_base_install, pythons=pythons, + wheel_path=wheel_path, ) @@ -202,6 +213,7 @@ def run( venv_pattern, recompile_reqs, ): + wheel_path = ctx.obj.get("wheel_path") ctx.obj["session"].run( pattern=re.compile(pattern), venv_pattern=re.compile(venv_pattern), @@ -213,6 +225,7 @@ def run( skip_missing=skip_missing, exit_first=exit_first, recompile_reqs=recompile_reqs, + wheel_path=wheel_path, ) @@ -221,9 +234,11 @@ def run( @click.option("--pass-env", "pass_env", is_flag=True, default=False) @click.pass_context def shell(ctx, ident, pass_env): + wheel_path = ctx.obj.get("wheel_path") ctx.obj["session"].shell( ident=ident, pass_env=pass_env, + wheel_path=wheel_path, ) diff --git a/riot/riot.py b/riot/riot.py index aae00e33..50a4a9d4 100644 --- a/riot/riot.py +++ b/riot/riot.py @@ -578,6 +578,7 @@ def prepare( skip_deps: bool = False, recompile_reqs: bool = False, child_was_installed: bool = False, + wheel_path: t.Optional[str] = None, ) -> None: # Propagate the interpreter down the parenting relation self.py = py = py or self.py @@ -601,7 +602,7 @@ def prepare( if self.created: py.create_venv(recreate, venv_path) if not self.venv.skip_dev_install or not skip_deps: - install_dev_pkg(venv_path, force=True) + install_dev_pkg(venv_path, force=True, wheel_path=wheel_path) pkg_str = self.pkg_str assert pkg_str is not None @@ -637,7 +638,10 @@ def prepare( if not self.created and self.parent is not None: self.parent.prepare( - env, py, child_was_installed=installed or exists or child_was_installed + env, + py, + child_was_installed=installed or exists or child_was_installed, + wheel_path=wheel_path, ) @@ -720,6 +724,7 @@ def run( skip_missing: bool = False, exit_first: bool = False, recompile_reqs: bool = False, + wheel_path: t.Optional[str] = None, ) -> None: results = [] @@ -728,6 +733,7 @@ def run( recreate=recreate_venvs, skip_deps=skip_base_install, pythons=pythons, + wheel_path=wheel_path, ) for inst in self.venv.instances(): @@ -795,6 +801,7 @@ def run( skip_deps=skip_base_install or inst.venv.skip_dev_install, recreate=recreate_venvs, recompile_reqs=recompile_reqs, + wheel_path=wheel_path, ) pythonpath = inst.pythonpath @@ -960,6 +967,7 @@ def generate_base_venvs( recreate: bool, skip_deps: bool, pythons: t.Optional[t.Set[Interpreter]], + wheel_path: t.Optional[str] = None, ) -> None: """Generate all the required base venvs.""" # Find all the python interpreters used. @@ -996,7 +1004,7 @@ def generate_base_venvs( continue # Install the dev package into the base venv. - install_dev_pkg(py.venv_path, force=True) + install_dev_pkg(py.venv_path, force=True, wheel_path=wheel_path) def _generate_shell_rcfile(self): with tempfile.NamedTemporaryFile() as rcfile: @@ -1020,7 +1028,9 @@ def requirements(self, ident): with Status("Producing requirements.txt"): _ = inst.requirements - def shell(self, ident, pass_env): + def shell( + self, ident: str, pass_env: bool, wheel_path: t.Optional[str] = None + ) -> None: for inst, venv_path in self._venvs_matching_identifier(ident): logger.info("Launching shell inside venv instance %s", inst) logger.debug("Setting venv path to %s", venv_path) @@ -1035,7 +1045,7 @@ def shell(self, ident, pass_env): # Should we expect the venv to be ready? with Status("Preparing shell virtual environment"): inst.py.create_venv(False) - inst.prepare(env) + inst.prepare(env, wheel_path=wheel_path) pythonpath = inst.pythonpath if pythonpath: @@ -1236,7 +1246,49 @@ def pip_deps(pkgs: t.Dict[str, str]) -> str: ) -def install_dev_pkg(venv_path: str, force: bool = False) -> None: +def get_package_name() -> str: + """Extract package name from pyproject.toml or environment variable. + + Returns: + str: The package name + + Raises: + RuntimeError: If package name cannot be determined + """ + # Check environment variable first + env_pkg_name = os.getenv("RIOT_PACKAGE_NAME") + if env_pkg_name: + return env_pkg_name + + # Try pyproject.toml [project] table + pyproject_path = Path("pyproject.toml") + if pyproject_path.exists(): + # Python 3.11+ has tomllib built-in + tomllib: t.Any = None + if sys.version_info >= (3, 11): + import tomllib + else: + try: + import tomli as tomllib # type: ignore[no-redef] + except ImportError: + pass # Fall through to error + + if tomllib is not None: + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + # Check [project] table + if "project" in data and "name" in data["project"]: + return t.cast(str, data["project"]["name"]) + + raise RuntimeError( + "Could not determine package name from pyproject.toml [project] table. " + "Ensure pyproject.toml exists with [project] name, or set RIOT_PACKAGE_NAME environment variable." + ) + + +def install_dev_pkg( + venv_path: str, force: bool = False, wheel_path: t.Optional[str] = None +) -> None: dev_pkg_lockfile = Path(venv_path) / ".riot-dev-pkg-installed" if dev_pkg_lockfile.exists() and not force: logger.info("Dev package already installed. Skipping.") @@ -1249,14 +1301,51 @@ def install_dev_pkg(venv_path: str, force: bool = False) -> None: logger.warning("No Python setup file found. Skipping dev package installation.") return - logger.info("Installing dev package (edit mode) in %s.", venv_path) - try: - Session.run_cmd_venv( - venv_path, - "pip --disable-pip-version-check install -e .", - env=dict(os.environ), + # Determine installation method + if wheel_path: + # Install from wheels (two-step process to ensure we use only wheels from source) + package_name = get_package_name() + logger.info( + "Installing dev package from wheels: %s (source: %s)", + package_name, + wheel_path, ) - dev_pkg_lockfile.touch() - except CmdFailure as e: - logger.error("Dev install failed, aborting!\n%s", e.proc.stdout) - sys.exit(1) + + # Step 1: Download wheel to temp directory using --no-index to avoid PyPI + with tempfile.TemporaryDirectory() as tmp_dir: + download_cmd = ( + f"pip --disable-pip-version-check download " + f"--no-index --no-deps --find-links '{wheel_path}' " + f"--pre --dest '{tmp_dir}' '{package_name}'" + ) + try: + Session.run_cmd_venv(venv_path, download_cmd, env=dict(os.environ)) + except CmdFailure as e: + logger.error( + "Wheel download failed. Ensure wheel exists at %s\n%s", + wheel_path, + e.proc.stdout, + ) + sys.exit(1) + + # Step 2: Install the downloaded wheel + install_cmd = f"pip --disable-pip-version-check install '{tmp_dir}'/*.whl" + try: + Session.run_cmd_venv(venv_path, install_cmd, env=dict(os.environ)) + dev_pkg_lockfile.touch() + except CmdFailure as e: + logger.error("Wheel installation failed!\n%s", e.proc.stdout) + sys.exit(1) + else: + # Install in editable mode (current behavior) + logger.info("Installing dev package (edit mode) in %s.", venv_path) + try: + Session.run_cmd_venv( + venv_path, + "pip --disable-pip-version-check install -e .", + env=dict(os.environ), + ) + dev_pkg_lockfile.touch() + except CmdFailure as e: + logger.error("Dev install failed, aborting!\n%s", e.proc.stdout) + sys.exit(1) diff --git a/tests/test_cli.py b/tests/test_cli.py index f1182cd5..cf5fa10b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -57,6 +57,7 @@ def assert_args(args): "skip_missing", "exit_first", "recompile_reqs", + "wheel_path", ] ) @@ -283,11 +284,12 @@ def test_generate_suites_with_long_args(cli: click.testing.CliRunner) -> None: generate_base_venvs.assert_called_once() kwargs = generate_base_venvs.call_args.kwargs assert set(kwargs.keys()) == set( - ["pattern", "recreate", "skip_deps", "pythons"] + ["pattern", "recreate", "skip_deps", "pythons", "wheel_path"] ) assert kwargs["pattern"].pattern == ".*" assert kwargs["recreate"] is True assert kwargs["skip_deps"] is True + assert kwargs["wheel_path"] is None def test_generate_base_venvs_with_short_args(cli: click.testing.CliRunner) -> None: @@ -302,11 +304,12 @@ def test_generate_base_venvs_with_short_args(cli: click.testing.CliRunner) -> No generate_base_venvs.assert_called_once() kwargs = generate_base_venvs.call_args.kwargs assert set(kwargs.keys()) == set( - ["pattern", "recreate", "skip_deps", "pythons"] + ["pattern", "recreate", "skip_deps", "pythons", "wheel_path"] ) assert kwargs["pattern"].pattern == ".*" assert kwargs["recreate"] is True assert kwargs["skip_deps"] is True + assert kwargs["wheel_path"] is None def test_generate_base_venvs_with_pattern(cli: click.testing.CliRunner) -> None: @@ -323,9 +326,10 @@ def test_generate_base_venvs_with_pattern(cli: click.testing.CliRunner) -> None: generate_base_venvs.assert_called_once() kwargs = generate_base_venvs.call_args.kwargs assert set(kwargs.keys()) == set( - ["pattern", "recreate", "skip_deps", "pythons"] + ["pattern", "recreate", "skip_deps", "pythons", "wheel_path"] ) assert kwargs["pattern"].pattern == "^pattern.*" + assert kwargs["wheel_path"] is None assert kwargs["recreate"] is False assert kwargs["skip_deps"] is False diff --git a/tests/test_integration.py b/tests/test_integration.py index f28fd1e1..979302c2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -64,12 +64,14 @@ def test_no_riotfile(tmp_path: pathlib.Path, tmp_run: _T_TmpRun) -> None: == """Usage: riot [OPTIONS] COMMAND [ARGS]... Options: - -f, --file PATH [default: riotfile.py] + -f, --file PATH [default: riotfile.py] -v, --verbose -d, --debug - -P, --pipe Pipe mode. Makes riot emit plain output. - --version Show the version and exit. - --help Show this message and exit. + -P, --pipe Pipe mode. Makes riot emit plain output. + --wheel-path TEXT Path or URL to wheel files. When set, installs from + wheels instead of editable mode. + --version Show the version and exit. + --help Show this message and exit. Commands: generate Generate base virtual environments. @@ -158,12 +160,14 @@ def test_help(tmp_run: _T_TmpRun) -> None: Usage: riot [OPTIONS] COMMAND [ARGS]... Options: - -f, --file PATH [default: riotfile.py] + -f, --file PATH [default: riotfile.py] -v, --verbose -d, --debug - -P, --pipe Pipe mode. Makes riot emit plain output. - --version Show the version and exit. - --help Show this message and exit. + -P, --pipe Pipe mode. Makes riot emit plain output. + --wheel-path TEXT Path or URL to wheel files. When set, installs from + wheels instead of editable mode. + --version Show the version and exit. + --help Show this message and exit. Commands: generate Generate base virtual environments. diff --git a/tests/test_unit.py b/tests/test_unit.py index e0eda03d..00848e7c 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -346,3 +346,131 @@ def test_session_run_check_environment_modifications_and_recreate_true( regex = r".*isort==5\.10\.1.*itsdangerous==1\.1\.0.*six==1\.15\.0.*" expected = re.match(regex, result.replace("\n", "")) assert expected, "error: {}".format(result) + + +def test_get_package_name_from_env_var(monkeypatch): + """Test get_package_name() with RIOT_PACKAGE_NAME environment variable.""" + import tempfile + import os + from riot.riot import get_package_name + + with tempfile.TemporaryDirectory() as tmpdir: + old_cwd = os.getcwd() + try: + os.chdir(tmpdir) + # Set environment variable + monkeypatch.setenv("RIOT_PACKAGE_NAME", "my-test-package") + + # Should return the env var value + assert get_package_name() == "my-test-package" + finally: + os.chdir(old_cwd) + + +def test_get_package_name_from_pyproject_toml(monkeypatch, tmp_path): + """Test get_package_name() parsing from pyproject.toml [project] table.""" + import os + from riot.riot import get_package_name + + old_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + # Create a pyproject.toml with [project] name + pyproject_content = """ +[project] +name = "test-package" +version = "1.0.0" +""" + (tmp_path / "pyproject.toml").write_text(pyproject_content) + + # Should return the package name from pyproject.toml + assert get_package_name() == "test-package" + finally: + os.chdir(old_cwd) + + +def test_get_package_name_env_var_takes_precedence(monkeypatch, tmp_path): + """Test that RIOT_PACKAGE_NAME env var takes precedence over pyproject.toml.""" + import os + from riot.riot import get_package_name + + old_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + # Create a pyproject.toml + pyproject_content = """ +[project] +name = "file-package" +""" + (tmp_path / "pyproject.toml").write_text(pyproject_content) + + # Set env var which should take precedence + monkeypatch.setenv("RIOT_PACKAGE_NAME", "env-package") + + # Should return the env var value + assert get_package_name() == "env-package" + finally: + os.chdir(old_cwd) + + +def test_get_package_name_raises_without_config(monkeypatch, tmp_path): + """Test get_package_name() raises RuntimeError when no config is found.""" + import os + from riot.riot import get_package_name + import pytest + + old_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + # Ensure env var is not set + monkeypatch.delenv("RIOT_PACKAGE_NAME", raising=False) + + # Should raise RuntimeError + with pytest.raises(RuntimeError, match="Could not determine package name"): + get_package_name() + finally: + os.chdir(old_cwd) + + +def test_wheel_path_cli_option_passes_through(monkeypatch, tmp_path): + """Test that wheel_path is correctly threaded through the CLI to Session.""" + import os + from pathlib import Path + from unittest.mock import patch + + old_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + # Create a minimal riotfile + Path("riotfile.py").write_text( + """ +from riot import Venv +venv = Venv( + name="test", + command="echo 'test'", + pys=["3.8"], +) +""" + ) + + # Create pyproject.toml + Path("pyproject.toml").write_text('[project]\nname = "test-pkg"') + + # Mock the Session.run method to verify wheel_path is passed + with patch("riot.riot.Session.run") as mock_run: + from riot.cli import main + from click.testing import CliRunner + + runner = CliRunner() + # Test with wheel-path flag + result = runner.invoke(main, ["--wheel-path", "/tmp/wheels", "run", ".*"]) + + # Verify that Session.run was called with wheel_path parameter + # Note: This test verifies the CLI layer correctly threads the parameter + assert result.exit_code == 0 or "wheel_path" in str(mock_run.call_args) + finally: + os.chdir(old_cwd)