Merge pull request #541 from DiamondLightSource/pipelinebuilder

Pipeline auto generator
DiamondLightSource · Jan 28, 2025 · fdafb95 · fdafb95
2 parents 89edde6 + ddd1240
commit fdafb95
Show file tree

Hide file tree

Showing 9 changed files with 266 additions and 141 deletions.
diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml
@@ -31,6 +31,14 @@ jobs:
             activate-environment: httomo-docs
             environment-file: ./docs/source/doc-conda-requirements.yml
 
+        - name: Install httomo-backends
+          run: |
+            pip install --no-deps httomo-backends
+
+        - name: Generate full yaml pipelines using directives
+          run: |
+            python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml
+
         - name: Build docs
           run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/
 

diff --git a/.github/workflows/main_conda_package_test.yml b/.github/workflows/main_conda_package_test.yml
diff --git a/.github/workflows/main_weekly_build.yml b/.github/workflows/main_weekly_build.yml
diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_iris.yml
@@ -36,7 +36,7 @@ jobs:
         run: |
           micromamba activate httomo
           pip install --upgrade --force-reinstall pillow
-          pip install httomolibgpu tomobar ccpi-regularisation-cupy
+          pip install httomolibgpu tomobar
           pip install --no-deps httomo-backends
           pip install .         
           micromamba list

diff --git a/conda/environment.yml b/conda/environment.yml
@@ -1,6 +1,7 @@
 name: httomo
 channels:
   - conda-forge
+  - httomo
 dependencies:
   - astra-toolbox
   - click
@@ -23,4 +24,5 @@ dependencies:
   - pytest-cov
   - pytest-mock
   - tomopy=1.15
+  - ccpi-regulariser
 
diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml
@@ -17,3 +17,5 @@ dependencies:
   - loguru
   - graypy
   - tqdm
+  - ruamel.yaml>=0.18 
+  - pip
diff --git a/docs/source/pipelines/yaml.rst b/docs/source/pipelines/yaml.rst
@@ -37,7 +37,7 @@ It is recommended to use GPU-based pipelines and methods from the httomolib and
 
 .. dropdown:: Basic GPU pipeline which uses functions from the httomolibgpu library.
 
-    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_gpu1.yaml
+    .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml
         :language: yaml
 
 

diff --git a/docs/source/pipelines_full/gpu_pipeline1_directive.yaml b/docs/source/pipelines_full/gpu_pipeline1_directive.yaml
@@ -0,0 +1,18 @@
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+- method: find_center_vo
+  module_path: httomolibgpu.recon.rotation
+- method: remove_outlier
+  module_path: httomolibgpu.misc.corr
+- method: normalize
+  module_path: httomolibgpu.prep.normalize
+- method: remove_all_stripe
+  module_path: httomolibgpu.prep.stripe
+- method: FBP
+  module_path: httomolibgpu.recon.algorithm
+- method: calculate_stats
+  module_path: httomo.methods
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+- method: save_to_images
+  module_path: httomolib.misc.images
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,3 +17,5 @@ dependencies: @@
       - loguru
       - graypy
       - tqdm
+      - ruamel.yaml>=0.18
+      - pip