Merge branch 'main' into frames_average

Author: dkazanc

docs/source/howto/how_to_run/at_diamond.rst

@@ -36,8 +36,17 @@ This will allow HTTomo to be run on the local machine like so:
 Parallel
 ~~~~~~~~
 
-A parallel run of HTTomo at Diamond would usually be done on a compute cluster.
-However, there are cases where a parallel run on a local machine on cropped data
-is also useful, so that has also been described below.
+Parallel execution of HTTomo at Diamond is typically performed on a compute cluster. 
+The HTTomo launcher is integrated with the SLURM workload manager via REST APIs and is 
+configured to submit jobs to the :code:`wilson` compute cluster.
 
-TODO (:code:`httomo_mpi_local`?)
+To run HTTomo, either log in to a Wilson compute node directly or load the HTTomo environment on a workstation
+with :code:`module load httomo`.
+
+Once the environment is loaded, HTTomo jobs can be submitted and executed in parallel as described below.
+
+.. code-block:: console
+
+  $ httomo_mpi IN_FILE YAML_CONFIG OUT_DIR
+`
+Also look for help with :code:`httomo_mpi --help`.

docs/source/howto/run_httomo.rst

@@ -419,18 +419,23 @@ block size happens to improve performance for the CPU-only section(s).
 :code:`--max-memory`
 ~~~~~~~~~~~~~~~~~~~~
 
-HTTomo supports running on both:
+HTTomo supports execution on both:
 
-- a compute cluster, where RAM on the host machine is often quite large
-- a personal machine, where RAM is not nearly as large
+-  compute clusters, where large amounts of system RAM are typically available
+-  personal workstations or laptops, where available RAM is often more limited
 
-This is done by a mechanism within HTTomo to hold data in RAM wherever there is
-enough RAM to do the required processing, and write data to a file if there is not
-enough RAM.
+To accommodate these different environments, HTTomo dynamically manages intermediate 
+data during pipeline execution. Whenever sufficient RAM is available, data is kept 
+in memory to maximise performance. If memory requirements exceed the available 
+RAM, HTTomo automatically stores intermediate data on disk and reloads it as needed.
+
+The :code:`--max-memory` option specifies the amount of RAM available to HTTomo.
+It sets the limit in terms of the maximum amount of the CPU memory per process available
+on the system. For instance, if you're running HTTomo on 1 process serially, you need
+to pass the value of the CPU memory available on your system, e.g., :code:`--max-memory 32G` would 
+set it to 32 Gigabyte. Providing an accurate value allows HTTomo to optimise memory usage 
+while avoiding out-of-memory errors.
 
-The :code:`--max-memory` flag is for telling HTTomo how much RAM the machine has,
-so then it can switch to using a file during execution of the pipeline if
-necessary.
 
 :code:`--save-snapshots`
 ~~~~~~~~~~~~~~~~~~~~~~~~

httomo/cli.py

@@ -171,7 +171,7 @@ def check(pipeline: Union[Path, str], in_data_file: Optional[Path] = None):
     "--max-memory",
     type=click.STRING,
     default="0",
-    help="Limit the amount of memory used by the pipeline to the given memory (supports strings like 3.2G or bytes)",
+    help="The maximum amount of the CPU memory per process available on the system (supports strings like 3.2G or bytes)",
 )
 @click.option(
     "--save-snapshots",
@@ -324,6 +324,10 @@ def run(
         format_enum,
     )
 
+    for wrapper in pipeline_object:
+        wrapper.check()
+        wrapper.prep()
+
     if not does_contain_sweep:
         execute_high_throughput_run(
             pipeline_object,
@@ -539,7 +543,12 @@ def execute_sweep_run(pipeline: Pipeline, global_comm: MPI.Comm) -> None:
     ParamSweepRunner(pipeline, global_comm).execute()
 
 
-def estimate_cpu_memory(in_data_file: Path, pipeline_file: Path, nprocs: int) -> int:
+def estimate_cpu_memory(
+    in_data_file: Path,
+    pipeline_file: Path,
+    nprocs: int,
+    shape: Optional[tuple[int, int, int]] = None,
+) -> int:
     pipeline = generate_pipeline(
         in_data_file, pipeline_file, False, MPI.COMM_WORLD, PipelineFormat.Yaml
     )
@@ -551,14 +560,17 @@ def estimate_cpu_memory(in_data_file: Path, pipeline_file: Path, nprocs: int) ->
         dtype = dataset.dtype
         full_shape = dataset.shape
 
-    preview_config = parse_preview(
-        config[0]["parameters"].get("preview", None), full_shape
-    )
-    previewed_shape = (
-        preview_config.angles.stop - preview_config.angles.start,
-        preview_config.detector_y.stop - preview_config.detector_y.start,
-        preview_config.detector_x.stop - preview_config.detector_x.start,
-    )
+    if shape is None:
+        preview_config = parse_preview(
+            config[0]["parameters"].get("preview", None), full_shape
+        )
+        previewed_shape = (
+            preview_config.angles.stop - preview_config.angles.start,
+            preview_config.detector_y.stop - preview_config.detector_y.start,
+            preview_config.detector_x.stop - preview_config.detector_x.start,
+        )
+    else:
+        previewed_shape = shape
 
     section_memory_peak = 0
     for idx in range(len(sections)):

httomo/method_wrappers/generic.py

@@ -138,13 +138,21 @@ def __init__(
             self._query.save_result_default() if save_result is None else save_result
         )
 
-        if self.is_gpu and not gpu_enabled:
-            raise ValueError("GPU is not available, please use only CPU methods")
-
         self._side_output: Dict[str, Any] = dict()
 
         self._gpu_time_info = GpuTimeInfo()
 
+    def check(self) -> None:
+        """
+        Raises
+        ------
+        ValueError
+            If the method is a GPU method and the machine is not GPU-enabled.
+        """
+        if self.is_gpu and not gpu_enabled:
+            raise ValueError("GPU is not available, please use only CPU methods")
+
+    def prep(self) -> None:
         if gpu_enabled:
             _id = httomo.globals.gpu_id
             self._gpu_id = get_gpu_id() if _id == -1 else _id

httomo/runner/method_wrapper.py

@@ -206,3 +206,17 @@ def calculate_max_slices(
         something persists afterwards.
         """
         ...  # pragma: nocover
+
+    def check(self) -> None:
+        """
+        Perform any checks necessary to ensure that the method is capable of executing on the
+        machine (for example, if it's a GPU method, check that execution is on a machine with a
+        GPU that is accessible).
+        """
+        ...  # pragma: nocover
+
+    def prep(self) -> None:
+        """
+        Perform any preparation needed prior to beginning pipeline execution.
+        """
+        ...  # pragma: nocover

httomo/sweep_runner/param_sweep_runner.py

@@ -233,7 +233,7 @@ def execute_sweep(self):
             unit="value",
             ascii=True,
         )
-        for val, _ in zip(self._sweep_values, progress):
+        for val in progress:
             # Blocks are modified in-place by method wrappers, so a new block must be created
             # that contains a copy of the input data to the sweep stage
             block = ParamSweepBlock(

tests/method_wrappers/test_generic.py

@@ -213,7 +213,7 @@ def fake_method(data):
             "fake_method",
             MPI.COMM_WORLD,
             make_mock_preview_config(mocker),
-        )
+        ).check()
 
     assert "GPU is not available" in str(e)