another cpu generated pipeline added, tests reorganisation, a new CI …

…for pipelines

.github/workflows/httomo_docs.yml

@@ -35,9 +35,10 @@ jobs:
           run: |
             pip install --no-deps httomo-backends
 
-        - name: Generate full yaml pipelines using directives
+        - name: Generate full yaml pipelines using pipeline directives
           run: |
             python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml
+            python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/cpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/cpu_pipeline1.yaml
 
         - name: Build docs
           run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/

.github/workflows/run_tests_iris.yml → .github/workflows/run_tests_framework_iris.yml

@@ -1,4 +1,4 @@
-name: HTTomo tests
+name: HTTomo framework tests
 
 on:
   pull_request:

.github/workflows/run_tests_pipelines_iris.yml

@@ -0,0 +1,52 @@
+name: HTTomo pipelines tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  iris-gpu:
+    runs-on: iris-gpu
+    container:
+      image: nvidia/cuda:12.6.3-devel-ubi8
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+
+      - name: Create conda environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: conda/environment.yml
+          environment-name: httomo
+          post-cleanup: 'all'
+          init-shell: bash
+
+      - name: Install httomo libraries, httomo-backends and httomo
+        run: |
+          micromamba activate httomo
+          pip install --upgrade --force-reinstall pillow
+          pip install httomolibgpu tomobar
+          pip install --no-deps httomo-backends
+          pip install .   
+          micromamba list
+                
+      - name: Generate full yaml pipelines using pipeline directives
+        run: |
+          pip install "ruamel.yaml>0.18.0"
+          python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml
+          python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/cpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/cpu_pipeline1.yaml
+
+      - name: Run HTTomo pipelines tests (small data)
+        run: |
+          pytest tests/test_pipeline_small.py --pipeline_small

docs/source/pipelines/yaml.rst

@@ -3,50 +3,47 @@
 Full YAML pipelines
 ==============================
 
-This is a collection of ready to be used pipeline templates aka process lists for HTTomo.
+This is a collection of ready to be used full pipelines or process lists for HTTomo.
 See more on :ref:`explanation_process_list` and how to :ref:`howto_process_list`.
 
-.. _tutorials_pl_templates_cpu:
+HTTomo mainly targets GPU computations, therefore the use of :ref:`tutorials_pl_templates_gpu` is 
+preferable. However, when the GPU device is not available or a GPU method is not implemented, the use of 
+:ref:`tutorials_pl_templates_cpu` is possible. 
 
-CPU Pipeline templates
-----------------------------
+.. note:: The combination of both GPU and CPU methods is possible. If one expects to achieve the faster performance, please use the GPU methods provided, where possible.
 
-CPU-pipelines mostly use TomoPy methods that are executed on the CPU and expected to be slower.
+.. _tutorials_pl_templates_gpu:
 
-.. dropdown:: Basic TomoPy's (CPU-only) pipeline for the classical 180-degrees scan
+GPU Pipeline templates
+-----------------------
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu1.yaml
-        :language: yaml
+The GPU-pipelines consist of methods from httomolibgpu (GPU) and httomolib (CPU) backend :ref:`backends_list`. Those libraries are supported directly by the HTTomo development team.
 
-.. dropdown:: TomoPy's pipeline where :ref:`previewing` is demonstrated
+.. dropdown:: Basic GPU pipeline which uses functions from the httomolib/gpu libraries.
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu2.yaml
+    .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml
         :language: yaml
 
-.. dropdown:: This pipeline shows how "calculate_stats" module extracts global statistics in order to rescale data for saving 8-bit images
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_cpu3.yaml
-        :language: yaml
-
-.. _tutorials_pl_templates_gpu:
+.. _tutorials_pl_templates_cpu:
 
-GPU Pipeline templates
-----------------------------
+CPU Pipeline templates
+-----------------------
 
-It is recommended to use GPU-based pipelines and methods from the httomolib and httomolibgpu libraries. Those libraries are supported directly by HTTomo development team. 
+The CPU-pipelines mostly use TomoPy methods. They are executed solely on the CPU and therefore expected to be slower than the GPU pipelines.
 
-.. dropdown:: Basic GPU pipeline which uses functions from the httomolibgpu library.
+.. dropdown:: Basic pipeline using TomoPy.
 
-    .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml
+    .. literalinclude:: ../pipelines_full/cpu_pipeline1.yaml
         :language: yaml
 
 
 .. _tutorials_pl_templates_dls:
 
 DLS-specific templates
-----------------------------
+----------------------
 
-Those pipelines will use the methods from the httomolib and httomolibgpu libraries. 
+Those pipelines are specific to Diamond Light Source processing strategies and can differ for different tomographic beamlines. 
 
 .. dropdown:: An example of a typical DIAD (k11) beamline piepeline.
 
@@ -66,7 +63,7 @@ Those pipelines will use the methods from the httomolib and httomolibgpu librari
 .. _tutorials_pl_templates_sweeps:
 
 Parameter Sweeps templates
-----------------------------
+--------------------------
 
 These templates demonstrate how to perform a sweep across multiple values of a
 single parameter (see :ref:`parameter_sweeping` for more details).

docs/source/pipelines_full/cpu_pipeline1_directive.yaml

@@ -0,0 +1,18 @@
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+- method: find_center_vo
+  module_path: tomopy.recon.rotation
+- method: remove_outlier
+  module_path: tomopy.misc.corr
+- method: normalize
+  module_path: tomopy.prep.normalize
+- method: minus_log
+  module_path: tomopy.prep.normalize
+- method: recon
+  module_path: tomopy.recon.algorithm
+- method: calculate_stats
+  module_path: httomo.methods
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+- method: save_to_images
+  module_path: httomolib.misc.images

docs/source/scripts/yaml_pipelines_generator.py

@@ -124,6 +124,11 @@ def yaml_pipelines_generator(
                     indent=0,
                 )
                 pipeline_full += yaml_template_method
+                try:
+                    # fix for a default parameter (REAUIRED) in TomoPy's algorithm
+                    pipeline_full[i]["parameters"]["dif"] = 0.1
+                except:
+                    pass                
                 pipeline_full[i]["parameters"].yaml_add_eol_comment(
                     key="dif",
                     comment="A difference between the outlier value and the median value of neighbouring pixels.",
@@ -161,6 +166,14 @@ def yaml_pipelines_generator(
                     key="recon_mask_radius",
                     comment="Zero pixels outside the mask-circle radius.",
                 )
+                try:
+                    # fix for a default parameter (None) in TomoPy's algorithm
+                    pipeline_full[i]["parameters"]["algorithm"] = "gridrec"
+                except:
+                    pipeline_full[i]["parameters"].yaml_add_eol_comment(
+                        key="algorithm",
+                        comment="Select the required algorithm, e.g. `gridrec`",
+                    )
             elif "calculate_stats" in method_name:
                 pipeline_full.yaml_set_comment_before_after_key(
                     i,

tests/conftest.py

@@ -24,6 +24,7 @@ def pytest_configure(config):
     config.addinivalue_line("markers", "mpi: mark test to run in an MPI environment")
     config.addinivalue_line("markers", "perf: mark test as performance test")
     config.addinivalue_line("markers", "cupy: needs cupy to run")
+    config.addinivalue_line("markers", "pipesmall: mark tests to run full pipelines on small data")
     config.addinivalue_line(
         "markers", "preview: mark test to run with `httomo preview`"
     )
@@ -36,7 +37,9 @@ def pytest_addoption(parser):
         default=False,
         help="run performance tests only",
     )
-
+    parser.addoption(
+        "--pipeline_small", action="store_true", default=False, help="run full pipelines on small data"
+    )    
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--performance"):
@@ -51,7 +54,18 @@ def pytest_collection_modifyitems(config, items):
         for item in items:
             if "perf" in item.keywords:
                 item.add_marker(skip_perf)
-
+    if config.getoption("--pipeline_small"):
+        skip_other = pytest.mark.skip(reason="not a pipeline small data test")
+        for item in items:
+            if "pipesmall" not in item.keywords:
+                item.add_marker(skip_other)
+    else:
+        skip_perf = pytest.mark.skip(
+            reason="pipeline small data test - use '--pipeline_small' to run"
+        )
+        for item in items:
+            if "pipesmall" in item.keywords:
+                item.add_marker(skip_perf)
 
 @pytest.fixture
 def output_folder():
@@ -223,7 +237,7 @@ def standard_loader():
 
 @pytest.fixture
 def sample_pipelines():
-    return "tests/samples/pipeline_template_examples/"
+    return "docs/source/pipelines_full/"
 
 
 @pytest.fixture
@@ -233,32 +247,11 @@ def gpu_pipeline():
 
 @pytest.fixture
 def yaml_cpu_pipeline1():
-    return "tests/samples/pipeline_template_examples/pipeline_cpu1.yaml"
-
-
-@pytest.fixture
-def yaml_cpu_pipeline2():
-    return "tests/samples/pipeline_template_examples/pipeline_cpu2.yaml"
-
-
-@pytest.fixture
-def yaml_cpu_pipeline3():
-    return "tests/samples/pipeline_template_examples/pipeline_cpu3.yaml"
-
-
-@pytest.fixture
-def yaml_cpu_pipeline4():
-    return "tests/samples/pipeline_template_examples/pipeline_cpu4.yaml"
-
-
-@pytest.fixture
-def yaml_cpu_pipeline5():
-    return "tests/samples/pipeline_template_examples/pipeline_cpu5.yaml"
-
+    return "docs/source/pipelines_full/cpu_pipeline1.yaml"
 
 @pytest.fixture
 def yaml_gpu_pipeline1():
-    return "tests/samples/pipeline_template_examples/pipeline_gpu1.yaml"
+    return "docs/source/pipelines_full/gpu_pipeline1.yaml"
 
 
 @pytest.fixture
@@ -296,6 +289,21 @@ def _merge_yamls(*yamls) -> None:
     return _merge_yamls
 
 
+# @pytest.fixture
+# def change_value_piepeline_yaml(load_yaml: Callable):
+#     def _merge_yamls(*yamls) -> None:
+#         """Merge multiple yaml files into one"""
+#         data: List = []
+#         for y in yamls:
+#             curr_yaml_list = load_yaml(y)
+#             for x in curr_yaml_list:
+#                 data.append(x)
+#         with open("temp.yaml", "w") as file_descriptor:
+#             yaml.dump(data, file_descriptor)
+
+#     return new_pipeline_yaml
+
+
 @pytest.fixture
 def standard_data_darks_flats_config() -> DarksFlatsFileConfig:
     return DarksFlatsFileConfig(