diff --git a/.github/workflows/iris.yml b/.github/workflows/tests_httomo.yml similarity index 51% rename from .github/workflows/iris.yml rename to .github/workflows/tests_httomo.yml index 6ea3ef754..29961352c 100644 --- a/.github/workflows/iris.yml +++ b/.github/workflows/tests_httomo.yml @@ -1,6 +1,7 @@ -name: Tests on IRIS +name: HTTomo tests on: + workflow_dispatch: pull_request: branches: - main @@ -9,48 +10,46 @@ on: - main jobs: - iris-gpu: - runs-on: iris-gpu - container: - image: nvidia/cuda:12.6.3-devel-ubi8 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - + tests-linux: + runs-on: ubuntu-latest defaults: run: shell: bash -l {0} - steps: - name: Checkout repository code uses: actions/checkout@v4 - name: Create conda environment - uses: mamba-org/setup-micromamba@v1 + uses: mamba-org/setup-micromamba@v2 with: environment-name: httomo create-args: >- - cupy==12.3.0 + python==3.12 + numpy==2.4 h5py=*=*mpi_openmpi* mpi4py - tomopy==1.15 + tomopy==1.15.3 post-cleanup: 'all' init-shell: bash - name: Install httomo libraries and httomo run: | micromamba activate httomo - pip install .[dev-gpu] + python -m pip install --no-deps pytest pytest-cov + python -m pip install --no-deps tomobar + python -m pip install --no-deps httomolibgpu + python -m pip install --no-deps httomolib + python -m pip install --no-deps httomo-backends + pip install . --no-deps micromamba list - name: Generate full yaml pipelines using pipeline directives run: | - pip install "ruamel.yaml>0.18.0" + python -m pip install pyyaml pluggy Pygments iniconfig loguru tqdm graypy hdf5plugin pytest_mock plumbum click + python -m pip install aiofiles pillow + python -m pip install "ruamel.yaml>0.18.0" python ./docs/source/scripts/execute_pipelines_build.py -o ./docs/source/pipelines_full/ - name: Run HTTomo framework tests run: | - pytest tests/ - - - name: Run HTTomo pipelines tests (small data) - run: | - pytest tests/test_pipeline_small.py --small_data + pytest tests/ \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 5321b1b71..dbeca7356 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,6 +46,12 @@ def pytest_addoption(parser): default=False, help="run performance tests only", ) + parser.addoption( + "--cupy", + action="store_true", + default=False, + help="run cupy involved tests", + ) parser.addoption( "--small_data", action="store_true", @@ -79,6 +85,16 @@ def pytest_collection_modifyitems(config, items): for item in items: if "perf" in item.keywords: item.add_marker(skip_perf) + if config.getoption("--cupy"): + skip_other = pytest.mark.skip(reason="not a CuPy test") + for item in items: + if "cupy" not in item.keywords: + item.add_marker(skip_other) + else: + skip_perf = pytest.mark.skip(reason="cupy test - use '--cupy' to run") + for item in items: + if "cupy" in item.keywords: + item.add_marker(skip_perf) if config.getoption("--small_data"): skip_other = pytest.mark.skip(reason="not a pipeline small data test") for item in items: @@ -160,21 +176,6 @@ def test_data_path(): return os.path.join(CUR_DIR, "test_data") -@pytest.fixture -def ensure_clean_memory(): - import cupy as cp - - cp.get_default_memory_pool().free_all_blocks() - cp.get_default_pinned_memory_pool().free_all_blocks() - cache = cp.fft.config.get_plan_cache() - cache.clear() - yield None - cp.get_default_memory_pool().free_all_blocks() - cp.get_default_pinned_memory_pool().free_all_blocks() - cache = cp.fft.config.get_plan_cache() - cache.clear() - - @pytest.fixture def standard_data_path(): return "/entry1/tomo_entry/data/data" diff --git a/tests/method_wrappers/test_dezinging.py b/tests/method_wrappers/test_dezinging.py index 9a57c6046..f37bee473 100644 --- a/tests/method_wrappers/test_dezinging.py +++ b/tests/method_wrappers/test_dezinging.py @@ -97,7 +97,7 @@ def remove_outlier(x, axis="auto"): @pytest.mark.cupy -def test_dezinging_no_pool_memory_leak(mocker: MockerFixture, ensure_clean_memory): +def test_dezinging_no_pool_memory_leak(mocker: MockerFixture): class FakeModule: def remove_outlier(x, axis="auto"): # Needs more memory than the return size diff --git a/tests/method_wrappers/test_save_intermediate.py b/tests/method_wrappers/test_save_intermediate.py index 2dd2f6d07..6725ff76d 100644 --- a/tests/method_wrappers/test_save_intermediate.py +++ b/tests/method_wrappers/test_save_intermediate.py @@ -253,7 +253,7 @@ def save_intermediate_data( # Create intermediate data wrapper mocker.patch.object(httomo.globals, "run_out_dir", tmp_path) wrp = make_method_wrapper( - make_mock_repo(mocker, implementation="gpu_cupy"), + make_mock_repo(mocker, implementation="cpu"), "httomo.methods", "save_intermediate_data", MPI.COMM_WORLD, diff --git a/tests/runner/test_gpu_utils.py b/tests/runner/test_gpu_utils.py index 780c83a19..b8be6768f 100644 --- a/tests/runner/test_gpu_utils.py +++ b/tests/runner/test_gpu_utils.py @@ -26,6 +26,7 @@ def test_get_available_memory_cpu(mocker: MockerFixture): assert get_available_gpu_memory() > 10e9 +@pytest.mark.cupy @pytest.mark.parametrize( "nodes, gpus_per_node, global_rank, local_rank", [ diff --git a/tests/sweep_runner/test_param_sweep_runner.py b/tests/sweep_runner/test_param_sweep_runner.py index d27ec6a22..30dccb901 100644 --- a/tests/sweep_runner/test_param_sweep_runner.py +++ b/tests/sweep_runner/test_param_sweep_runner.py @@ -107,6 +107,7 @@ def test_after_prepare_block_attr_contains_data(mocker: MockerFixture): np.testing.assert_array_equal(runner.block.data, data) +@pytest.mark.cupy def tests_preview_modifier_paganin(mocker: MockerFixture): SINO_SLICES = 100 GLOBAL_SHAPE = PREVIEWED_SLICES_SHAPE = (1800, SINO_SLICES, 160) diff --git a/tests/test_ui_layer.py b/tests/test_ui_layer.py index cab582e5d..0a8e1390e 100644 --- a/tests/test_ui_layer.py +++ b/tests/test_ui_layer.py @@ -139,7 +139,7 @@ def test_build_pipeline_from_json(standard_data: str): }, { "method": "find_center_vo", - "module_path": "httomolibgpu.recon.rotation", + "module_path": "tomopy.recon.rotation", "parameters": { "ind": "mid", "smin": -50, @@ -155,8 +155,8 @@ def test_build_pipeline_from_json(standard_data: str): } }, { - "method": "FBP3d_tomobar", - "module_path": "httomolibgpu.recon.algorithm", + "method": "recon", + "module_path": "tomopy.recon.algorithm", "parameters": { "center": "${{centering.side_outputs.centre_of_rotation}}", "filter_freq_cutoff": 0.6,