From db7455f9544f080d9da08004e8b941bb91989b32 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 22 Jan 2025 13:44:58 +0000 Subject: [PATCH 1/8] starting with autogenerator for pipelines --- .../pipelines_full/pipelines_full_list.yaml | 41 +++++++++ .../scripts/yaml_pipelines_generator.py | 83 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 docs/source/pipelines_full/pipelines_full_list.yaml create mode 100644 docs/source/scripts/yaml_pipelines_generator.py diff --git a/docs/source/pipelines_full/pipelines_full_list.yaml b/docs/source/pipelines_full/pipelines_full_list.yaml new file mode 100644 index 000000000..4fc217f35 --- /dev/null +++ b/docs/source/pipelines_full/pipelines_full_list.yaml @@ -0,0 +1,41 @@ +- pipeline_name: diad_pipeline1 + methods: + name: standard_tomo + module_path: httomo.data.hdf.loaders + name: find_center_vo + module_path: httomolibgpu.recon.rotation + name: remove_outlier + module_path: httomolibgpu.misc.corr + name: normalize + module_path: httomolibgpu.prep.normalize + name: remove_all_stripe + module_path: httomolibgpu.prep.stripe + name: FBP + module_path: httomolibgpu.recon.algorithm + name: calculate_stats + module_path: httomo.methods + name: rescale_to_int + module_path: httomolibgpu.misc.rescale + name: save_to_images + module_path: httomolib.misc.images +# -----------------------------------------------# +- pipeline_name: diad_pipeline2 + methods: + name: standard_tomo + module_path: httomo.data.hdf.loaders + name: find_center_vo + module_path: httomolibgpu.recon.rotation + name: remove_outlier + module_path: httomolibgpu.misc.corr + name: normalize + module_path: httomolibgpu.prep.normalize + name: remove_stripe_based_sorting + module_path: httomolibgpu.prep.stripe + name: FBP + module_path: httomolibgpu.recon.algorithm + name: calculate_stats + module_path: httomo.methods + name: rescale_to_int + module_path: httomolibgpu.misc.rescale + name: save_to_images + module_path: httomolib.misc.images \ No newline at end of file diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py new file mode 100644 index 000000000..945acd8f2 --- /dev/null +++ b/docs/source/scripts/yaml_pipelines_generator.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------- +# Copyright 2022 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +# Created By : Tomography Team +# Created Date: 22/January/2025 +# version ='0.1' +# --------------------------------------------------------------------------- +"""Script that generates YAML pipelines for HTTomo using YAML templates from httomo-backends +(should be installed in environment). + +Please run the generator as: + python -m yaml_pipelines_generator -i /path/to/pipelines.yml -o /path/to/output/ +""" +import argparse +import importlib +import inspect +import os +import re +from typing import Any, List, Dict +import yaml + +import httomo_backends + + +def yaml_pipelines_generator(path_to_pipelines: str, path_to_httomobackends: str, output_folder: str) -> int: + """function that builds YAML pipelines using YAML templates from httomo-backends + + Args: + path_to_pipelines: path to the YAML file which contains a high-level description of the required pipeline to be built. + path_to_httomobackends: path to httomo-backends on the system, where YAML templates stored. + output_folder: path to output folder with saved pipelines + + Returns: + returns zero if the processing is successful + """ + + return 0 + + +def get_args(): + parser = argparse.ArgumentParser( + description="Script that generates YAML pipelines for HTTomo " + "using YAML templates from httomo-backends." + ) + parser.add_argument( + "-i", + "--input", + type=str, + default=None, + help="A path to the list of pipelines needed to be built within a yaml file", + ) + parser.add_argument( + "-o", + "--output", + type=str, + default="./", + help="Directory to save the yaml pipelines in.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + path_to_httomobackends = os.path.dirname(httomo_backends.__file__) + args = get_args() + path_to_pipelines = args.input + output_folder = args.output + return_val = yaml_pipelines_generator(path_to_pipelines, path_to_httomobackends, output_folder) + if return_val == 0: + print("YAML pipelines have been successfully generated!") From 4bdb523c58d2701801dcc30853061b20f828474b Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 22 Jan 2025 16:56:55 +0000 Subject: [PATCH 2/8] work continues --- .../pipelines_full/pipelines_full_list.yaml | 56 ++++++++++++------- .../scripts/yaml_pipelines_generator.py | 14 +++++ 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/docs/source/pipelines_full/pipelines_full_list.yaml b/docs/source/pipelines_full/pipelines_full_list.yaml index 4fc217f35..66764ddb3 100644 --- a/docs/source/pipelines_full/pipelines_full_list.yaml +++ b/docs/source/pipelines_full/pipelines_full_list.yaml @@ -1,41 +1,57 @@ - pipeline_name: diad_pipeline1 - methods: + method: name: standard_tomo - module_path: httomo.data.hdf.loaders + module_path: httomo.data.hdf.loaders + method: name: find_center_vo - module_path: httomolibgpu.recon.rotation + module_path: httomolibgpu.recon.rotation + method: name: remove_outlier - module_path: httomolibgpu.misc.corr + module_path: httomolibgpu.misc.corr + method: name: normalize - module_path: httomolibgpu.prep.normalize + module_path: httomolibgpu.prep.normalize + method: name: remove_all_stripe - module_path: httomolibgpu.prep.stripe + module_path: httomolibgpu.prep.stripe + method: name: FBP - module_path: httomolibgpu.recon.algorithm + module_path: httomolibgpu.recon.algorithm + method: name: calculate_stats - module_path: httomo.methods + module_path: httomo.methods + method: name: rescale_to_int - module_path: httomolibgpu.misc.rescale + module_path: httomolibgpu.misc.rescale + method: name: save_to_images - module_path: httomolib.misc.images + module_path: httomolib.misc.images # -----------------------------------------------# - pipeline_name: diad_pipeline2 - methods: + method: name: standard_tomo - module_path: httomo.data.hdf.loaders + module_path: httomo.data.hdf.loaders + method: name: find_center_vo - module_path: httomolibgpu.recon.rotation + module_path: httomolibgpu.recon.rotation + method: name: remove_outlier - module_path: httomolibgpu.misc.corr + module_path: httomolibgpu.misc.corr + method: name: normalize - module_path: httomolibgpu.prep.normalize + module_path: httomolibgpu.prep.normalize + method: name: remove_stripe_based_sorting - module_path: httomolibgpu.prep.stripe + module_path: httomolibgpu.prep.stripe + method: name: FBP - module_path: httomolibgpu.recon.algorithm + module_path: httomolibgpu.recon.algorithm + method: name: calculate_stats - module_path: httomo.methods + module_path: httomo.methods + method: name: rescale_to_int - module_path: httomolibgpu.misc.rescale + module_path: httomolibgpu.misc.rescale + method: name: save_to_images - module_path: httomolib.misc.images \ No newline at end of file + module_path: httomolib.misc.images diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py index 945acd8f2..525bfce03 100644 --- a/docs/source/scripts/yaml_pipelines_generator.py +++ b/docs/source/scripts/yaml_pipelines_generator.py @@ -48,6 +48,20 @@ def yaml_pipelines_generator(path_to_pipelines: str, path_to_httomobackends: str returns zero if the processing is successful """ + # open YAML file to inspect + with open(path_to_pipelines, "r") as stream: + try: + pipeline_file_content = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + # a loop over pipelines in the file + pipelines_no = len(pipeline_file_content) + for i in range(pipelines_no): + pipeline = pipeline_file_content[i] + pipeline_name = pipeline['pipeline_name'] + print(pipeline_name) + return 0 From 220c679b610ec0b77b79c17d77413f517ecdb51b Mon Sep 17 00:00:00 2001 From: dkazanc Date: Thu, 23 Jan 2025 17:37:06 +0000 Subject: [PATCH 3/8] working script, more work needed to complete --- .../pipelines_full/pipelines_full_list.yaml | 75 +++------- .../scripts/yaml_pipelines_generator.py | 135 +++++++++++++++--- 2 files changed, 133 insertions(+), 77 deletions(-) diff --git a/docs/source/pipelines_full/pipelines_full_list.yaml b/docs/source/pipelines_full/pipelines_full_list.yaml index 66764ddb3..9b6d2533a 100644 --- a/docs/source/pipelines_full/pipelines_full_list.yaml +++ b/docs/source/pipelines_full/pipelines_full_list.yaml @@ -1,57 +1,18 @@ -- pipeline_name: diad_pipeline1 - method: - name: standard_tomo - module_path: httomo.data.hdf.loaders - method: - name: find_center_vo - module_path: httomolibgpu.recon.rotation - method: - name: remove_outlier - module_path: httomolibgpu.misc.corr - method: - name: normalize - module_path: httomolibgpu.prep.normalize - method: - name: remove_all_stripe - module_path: httomolibgpu.prep.stripe - method: - name: FBP - module_path: httomolibgpu.recon.algorithm - method: - name: calculate_stats - module_path: httomo.methods - method: - name: rescale_to_int - module_path: httomolibgpu.misc.rescale - method: - name: save_to_images - module_path: httomolib.misc.images -# -----------------------------------------------# -- pipeline_name: diad_pipeline2 - method: - name: standard_tomo - module_path: httomo.data.hdf.loaders - method: - name: find_center_vo - module_path: httomolibgpu.recon.rotation - method: - name: remove_outlier - module_path: httomolibgpu.misc.corr - method: - name: normalize - module_path: httomolibgpu.prep.normalize - method: - name: remove_stripe_based_sorting - module_path: httomolibgpu.prep.stripe - method: - name: FBP - module_path: httomolibgpu.recon.algorithm - method: - name: calculate_stats - module_path: httomo.methods - method: - name: rescale_to_int - module_path: httomolibgpu.misc.rescale - method: - name: save_to_images - module_path: httomolib.misc.images +- method: standard_tomo + module_path: httomo.data.hdf.loaders +- method: find_center_vo + module_path: httomolibgpu.recon.rotation +- method: remove_outlier + module_path: httomolibgpu.misc.corr +- method: normalize + module_path: httomolibgpu.prep.normalize +- method: remove_all_stripe + module_path: httomolibgpu.prep.stripe +- method: FBP + module_path: httomolibgpu.recon.algorithm +- method: calculate_stats + module_path: httomo.methods +- method: rescale_to_int + module_path: httomolibgpu.misc.rescale +- method: save_to_images + module_path: httomolib.misc.images diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py index 525bfce03..e4aef50f6 100644 --- a/docs/source/scripts/yaml_pipelines_generator.py +++ b/docs/source/scripts/yaml_pipelines_generator.py @@ -19,30 +19,39 @@ # Created Date: 22/January/2025 # version ='0.1' # --------------------------------------------------------------------------- -"""Script that generates YAML pipelines for HTTomo using YAML templates from httomo-backends +"""Script that generates YAML pipeline for HTTomo using YAML templates from httomo-backends (should be installed in environment). Please run the generator as: python -m yaml_pipelines_generator -i /path/to/pipelines.yml -o /path/to/output/ """ import argparse -import importlib -import inspect import os -import re from typing import Any, List, Dict import yaml +import ruamel.yaml + +CS = ruamel.yaml.comments.CommentedSeq # defaults to block style + + +def FS(x): # flow style list + res = CS(x) + res.fa.set_flow_style() + return res + import httomo_backends -def yaml_pipelines_generator(path_to_pipelines: str, path_to_httomobackends: str, output_folder: str) -> int: - """function that builds YAML pipelines using YAML templates from httomo-backends +def yaml_pipelines_generator( + path_to_pipelines: str, path_to_httomobackends: str, path_to_output_file: str +) -> int: + """function that builds YAML pipeline using YAML templates from httomo-backends Args: - path_to_pipelines: path to the YAML file which contains a high-level description of the required pipeline to be built. - path_to_httomobackends: path to httomo-backends on the system, where YAML templates stored. - output_folder: path to output folder with saved pipelines + path_to_pipelines: path to the YAML file which contains a high-level description of the required pipeline to be built. + path_to_httomobackends: path to httomo-backends on the system, where YAML templates stored. + path_to_output_file: path to output file with the generated pipeline Returns: returns zero if the processing is successful @@ -54,13 +63,97 @@ def yaml_pipelines_generator(path_to_pipelines: str, path_to_httomobackends: str pipeline_file_content = yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc) - - # a loop over pipelines in the file - pipelines_no = len(pipeline_file_content) - for i in range(pipelines_no): - pipeline = pipeline_file_content[i] - pipeline_name = pipeline['pipeline_name'] - print(pipeline_name) + + with open(path_to_output_file, "w") as f: + # a loop over methods in the high-level pipeline file + methods_no = len(pipeline_file_content) + pipeline_full = CS() + for i in range(methods_no): + method_content = pipeline_file_content[i] + method_name = method_content["method"] + module_name = method_content["module_path"] + # get the corresponding yaml template from httomo-backends + backend_name = module_name[0 : module_name.find(".")] + full_path_to_yamls = ( + path_to_httomobackends + + "/yaml_templates/" + + backend_name + + "/" + + module_name + + "/" + + method_name + + ".yaml" + ) + with open(full_path_to_yamls, "r") as stream: + try: + yaml_template_method = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + if "loaders" in module_name: + # should be the first method in the list + pipeline_full.yaml_set_comment_before_after_key( + i, + "Standard tomography loader for NeXus files", + indent=0, + ) + elif "rotation" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Center of Rotation method for automatic center finding. Required for reconstruction", + indent=0, + ) + elif "corr" in module_name and "remove_outlier" in method_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Removing dead pixels in the data, aka zingers. Only required if there are sharp streaks in the reconstruction.", + indent=0, + ) + elif "normalize" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Normalisation of projection data using collected flats/darks and taking negative log (not needed with Paganin). ", + indent=0, + ) + elif "stripe" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Method to remove stripe artefacts in the data that lead to ring artefacts in the reconstruction. ", + indent=0, + ) + elif "algorithm" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Reconstruction method. Use reference to the center if the method is used above or set to an ", + indent=0, + ) + # pipeline_full.yaml_add_eol_comment( + # "End-of-line comment", + # "center", + # column=10, + # ) + elif "calculate_stats" in method_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Calculate global statistics on the reconstructed volume (min/max needed specifically for data rescaling) ", + indent=0, + ) + else: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--------------------------------------------------------#", + indent=0, + ) + pipeline_full += yaml_template_method + + ruamel.yaml.dump( + pipeline_full, + f, + Dumper=ruamel.yaml.RoundTripDumper, + default_flow_style=False, + width=50, + indent=0, + ) return 0 @@ -82,7 +175,7 @@ def get_args(): "--output", type=str, default="./", - help="Directory to save the yaml pipelines in.", + help="Full path to the yaml file with the generated pipeline.", ) return parser.parse_args() @@ -91,7 +184,9 @@ def get_args(): path_to_httomobackends = os.path.dirname(httomo_backends.__file__) args = get_args() path_to_pipelines = args.input - output_folder = args.output - return_val = yaml_pipelines_generator(path_to_pipelines, path_to_httomobackends, output_folder) + path_to_output_file = args.output + return_val = yaml_pipelines_generator( + path_to_pipelines, path_to_httomobackends, path_to_output_file + ) if return_val == 0: - print("YAML pipelines have been successfully generated!") + print("YAML pipeline has been successfully generated!") From 229a50adb9c674742f5630f19c4e937113952d6c Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 27 Jan 2025 12:02:02 +0000 Subject: [PATCH 4/8] working pipelines generator --- .../scripts/yaml_pipelines_generator.py | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py index e4aef50f6..49279e055 100644 --- a/docs/source/scripts/yaml_pipelines_generator.py +++ b/docs/source/scripts/yaml_pipelines_generator.py @@ -27,20 +27,14 @@ """ import argparse import os -from typing import Any, List, Dict -import yaml import ruamel.yaml +import httomo_backends CS = ruamel.yaml.comments.CommentedSeq # defaults to block style -def FS(x): # flow style list - res = CS(x) - res.fa.set_flow_style() - return res - - -import httomo_backends +def __represent_none(self, data): + return self.represent_scalar("tag:yaml.org,2002:null", "null") def yaml_pipelines_generator( @@ -57,12 +51,14 @@ def yaml_pipelines_generator( returns zero if the processing is successful """ + yaml = ruamel.yaml.YAML(typ="rt") + # open YAML file to inspect - with open(path_to_pipelines, "r") as stream: + with open(path_to_pipelines, "r") as file: try: - pipeline_file_content = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) + pipeline_file_content = yaml.load(file) + except OSError as e: + print("loading yaml file with methods failed", e) with open(path_to_output_file, "w") as f: # a loop over methods in the high-level pipeline file @@ -86,9 +82,9 @@ def yaml_pipelines_generator( ) with open(full_path_to_yamls, "r") as stream: try: - yaml_template_method = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) + yaml_template_method = yaml.load(stream) + except OSError as e: + print("loading yaml template failed", e) if "loaders" in module_name: # should be the first method in the list @@ -124,20 +120,27 @@ def yaml_pipelines_generator( elif "algorithm" in module_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Reconstruction method. Use reference to the center if the method is used above or set to an ", + "Reconstruction method. Use a reference to the center or set to an integer.", indent=0, ) - # pipeline_full.yaml_add_eol_comment( - # "End-of-line comment", - # "center", - # column=10, - # ) elif "calculate_stats" in method_name: pipeline_full.yaml_set_comment_before_after_key( i, "Calculate global statistics on the reconstructed volume (min/max needed specifically for data rescaling) ", indent=0, ) + elif "rescale_to_int" in method_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Rescaling the data using min/max obtained from calculate_stats", + indent=0, + ) + elif "images" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "Saving data into images", + indent=0, + ) else: pipeline_full.yaml_set_comment_before_after_key( i, @@ -146,11 +149,17 @@ def yaml_pipelines_generator( ) pipeline_full += yaml_template_method + # tried inline comments here, doesn't work for some reason + # pipeline_full[5]["parameters"].yaml_add_eol_comment( + # key="center", + # comment="Some inline comment", + # ) + yaml.representer.add_representer(type(None), __represent_none) ruamel.yaml.dump( pipeline_full, f, Dumper=ruamel.yaml.RoundTripDumper, - default_flow_style=False, + default_flow_style=None, width=50, indent=0, ) From f90eacafb7a2c17d7e20cb0e013b9336db7d4094 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 27 Jan 2025 16:03:03 +0000 Subject: [PATCH 5/8] removes unused CI jobs and adds httomo backend install into docs --- .github/workflows/httomo_docs.yml | 4 + .github/workflows/main_conda_package_test.yml | 50 ----------- .github/workflows/main_weekly_build.yml | 89 ------------------- .github/workflows/run_tests_iris.yml | 2 +- conda/environment.yml | 2 + docs/source/doc-conda-requirements.yml | 2 + 6 files changed, 9 insertions(+), 140 deletions(-) delete mode 100644 .github/workflows/main_conda_package_test.yml delete mode 100644 .github/workflows/main_weekly_build.yml diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index 4bda85575..1b2cb9641 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -31,6 +31,10 @@ jobs: activate-environment: httomo-docs environment-file: ./docs/source/doc-conda-requirements.yml + - name: Install httomo-backends + run: | + pip install --no-deps httomo-backends + - name: Build docs run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ diff --git a/.github/workflows/main_conda_package_test.yml b/.github/workflows/main_conda_package_test.yml deleted file mode 100644 index 84dd2f120..000000000 --- a/.github/workflows/main_conda_package_test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Weekly conda package test - -on: - schedule: - - cron: '55 0 * * 1' # At 00:55 every Monday - -jobs: - download-test-iris: - runs-on: iris-gpu - container: - image: nvidia/cuda:11.6.2-devel-ubi8 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - ref: "main" - fetch-depth: 0 - - - name: Create conda environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: conda/environment.yml - environment-name: httomo - post-cleanup: 'all' - init-shell: bash - - - name: Install httomo - run: | - micromamba activate httomo - micromamba install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -y - micromamba list - - - name: Generate yaml templates - run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/httomo_modules.yaml -o ./httomo/yaml_templates/httomo - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./httomo/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./httomo/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./httomo/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./httomo/yaml_templates/httomolibgpu - - - name: Run tests - run: | - pytest tests/ diff --git a/.github/workflows/main_weekly_build.yml b/.github/workflows/main_weekly_build.yml deleted file mode 100644 index af792fa0a..000000000 --- a/.github/workflows/main_weekly_build.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: HTTomo main build (weekly) - -on: - schedule: - - cron: '0 0 * * 1' # At 00:00 on Monday - -jobs: - - install-httomo-main-test-iris: - runs-on: iris-gpu - container: - image: nvidia/cuda:11.6.2-devel-ubi8 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - - - name: Create conda environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: conda/environment.yml - environment-name: httomo - post-cleanup: 'all' - init-shell: bash - - - name: Install httomo-main - run: | - micromamba activate httomo - pip install .[dev] - micromamba list - - - name: Generate yaml templates - run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/httomo_modules.yaml -o ./httomo/yaml_templates/httomo - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./httomo/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./httomo/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./httomo/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./httomo/yaml_templates/httomolibgpu - - - name: Run tests - run: | - pytest tests/ - - conda-upload: - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - ref: "main" - fetch-depth: 0 - - # setup Python 3.10 - - name: Setup Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies with Conda - run: | - $CONDA/bin/conda install -c conda-forge conda-build - $CONDA/bin/conda install -c conda-forge anaconda-client - $CONDA/bin/conda update conda - $CONDA/bin/conda update conda-build - $CONDA/bin/conda list - - - name: Decrypt a secret - run: ./.scripts/decrypt_secret.sh - env: - LARGE_SECRET_PASSPHRASE: ${{ secrets.LARGE_SECRET_PASSPHRASE }} - - - name: Build and upload the package to httomo conda cloud - env: - LABEL: main - run: | - chmod +x ./.scripts/conda_upload.sh - ./.scripts/conda_upload.sh - diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_iris.yml index d17e8f187..c107a8513 100644 --- a/.github/workflows/run_tests_iris.yml +++ b/.github/workflows/run_tests_iris.yml @@ -36,7 +36,7 @@ jobs: run: | micromamba activate httomo pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar ccpi-regularisation-cupy + pip install httomolibgpu tomobar pip install --no-deps httomo-backends pip install . micromamba list diff --git a/conda/environment.yml b/conda/environment.yml index 4c2471188..52c5f7ddf 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,6 +1,7 @@ name: httomo channels: - conda-forge + - httomo dependencies: - astra-toolbox - click @@ -23,4 +24,5 @@ dependencies: - pytest-cov - pytest-mock - tomopy=1.15 + - ccpi-regulariser diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index e8a15fd8a..053bbc44d 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -17,3 +17,5 @@ dependencies: - loguru - graypy - tqdm + - ruamel.yaml + - pip From 04bac643915d3e9fd418e283efdab41486958501 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 27 Jan 2025 16:23:40 +0000 Subject: [PATCH 6/8] auto pipelines generation in CI --- .github/workflows/httomo_docs.yml | 4 ++++ ...{pipelines_full_list.yaml => gpu_pipeline1_directive.yaml} | 0 2 files changed, 4 insertions(+) rename docs/source/pipelines_full/{pipelines_full_list.yaml => gpu_pipeline1_directive.yaml} (100%) diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index 1b2cb9641..62e0f0c46 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -35,6 +35,10 @@ jobs: run: | pip install --no-deps httomo-backends + - name: Generate full yaml pipelines using directives + run: | + python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml + - name: Build docs run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ diff --git a/docs/source/pipelines_full/pipelines_full_list.yaml b/docs/source/pipelines_full/gpu_pipeline1_directive.yaml similarity index 100% rename from docs/source/pipelines_full/pipelines_full_list.yaml rename to docs/source/pipelines_full/gpu_pipeline1_directive.yaml From 68275ab4106d6a04a643dcd2299b98778736ae0e Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 27 Jan 2025 17:20:03 +0000 Subject: [PATCH 7/8] updates to the generator, version update ruamel --- docs/source/doc-conda-requirements.yml | 2 +- .../scripts/yaml_pipelines_generator.py | 83 +++++++++++++------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index 053bbc44d..dfeaf21eb 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -17,5 +17,5 @@ dependencies: - loguru - graypy - tqdm - - ruamel.yaml + - ruamel.yaml>=0.18 - pip diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py index 49279e055..0892463d4 100644 --- a/docs/source/scripts/yaml_pipelines_generator.py +++ b/docs/source/scripts/yaml_pipelines_generator.py @@ -51,7 +51,7 @@ def yaml_pipelines_generator( returns zero if the processing is successful """ - yaml = ruamel.yaml.YAML(typ="rt") + yaml = ruamel.yaml.YAML(typ="rt", pure=True) # open YAML file to inspect with open(path_to_pipelines, "r") as file: @@ -61,7 +61,7 @@ def yaml_pipelines_generator( print("loading yaml file with methods failed", e) with open(path_to_output_file, "w") as f: - # a loop over methods in the high-level pipeline file + # a loop over methods in the high-level pipeline file (directive) methods_no = len(pipeline_file_content) pipeline_full = CS() for i in range(methods_no): @@ -90,79 +90,112 @@ def yaml_pipelines_generator( # should be the first method in the list pipeline_full.yaml_set_comment_before_after_key( i, - "Standard tomography loader for NeXus files", + "--- Standard tomography loader for NeXus files. ---", indent=0, ) + pipeline_full += yaml_template_method elif "rotation" in module_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Center of Rotation method for automatic center finding. Required for reconstruction", + "--- Center of Rotation auto-finding. Required for reconstruction bellow. ---", indent=0, ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="ind", + comment="A vertical slice (sinogram) index to calculate CoR, `mid` can be used for middle", + ) + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="cor_initialisation_value", + comment="Use if an approximate CoR is known", + ) + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="average_radius", + comment="Average several sinograms to improve SNR, one can try 3-5 range", + ) + pipeline_full[i]["side_outputs"].yaml_add_eol_comment( + key="cor", + comment="A side output of the method, here a CoR scalar value", + ) elif "corr" in module_name and "remove_outlier" in method_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Removing dead pixels in the data, aka zingers. Only required if there are sharp streaks in the reconstruction.", + "--- Removing dead pixels in the data, aka zingers. Use if sharp streaks are present in reconstruction. ---", indent=0, ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="dif", + comment="A difference between the outlier value and the median value of neighbouring pixels.", + ) elif "normalize" in module_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Normalisation of projection data using collected flats/darks and taking negative log (not needed with Paganin). ", + "--- Normalisation of projection data using collected flats/darks images. --- ", indent=0, ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="minus_log", + comment="If Paganin method is used bellow, set it to false.", + ) elif "stripe" in module_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Method to remove stripe artefacts in the data that lead to ring artefacts in the reconstruction. ", + "--- Method to remove stripe artefacts in the data that lead to ring artefacts in the reconstruction. --- ", indent=0, ) + pipeline_full += yaml_template_method elif "algorithm" in module_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Reconstruction method. Use a reference to the center or set to an integer.", + "--- Reconstruction method. ---", indent=0, ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="center", + comment="Reference to center of rotation side output OR an integer.", + ) + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="recon_mask_radius", + comment="Zero pixels outside the mask-circle radius.", + ) elif "calculate_stats" in method_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Calculate global statistics on the reconstructed volume (min/max needed specifically for data rescaling) ", + "--- Calculate global statistics on the reconstructed volume, required for data rescaling. ---", indent=0, ) + pipeline_full += yaml_template_method elif "rescale_to_int" in method_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Rescaling the data using min/max obtained from calculate_stats", + "--- Rescaling the data using min/max obtained from `calculate_stats`. ---", indent=0, ) + pipeline_full += yaml_template_method elif "images" in module_name: pipeline_full.yaml_set_comment_before_after_key( i, - "Saving data into images", + "--- Saving data into images. ---", indent=0, ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="file_format", + comment="`tif` or `jpeg` can be used.", + ) else: pipeline_full.yaml_set_comment_before_after_key( i, "--------------------------------------------------------#", indent=0, ) - pipeline_full += yaml_template_method + pipeline_full += yaml_template_method - # tried inline comments here, doesn't work for some reason - # pipeline_full[5]["parameters"].yaml_add_eol_comment( - # key="center", - # comment="Some inline comment", - # ) yaml.representer.add_representer(type(None), __represent_none) - ruamel.yaml.dump( - pipeline_full, - f, - Dumper=ruamel.yaml.RoundTripDumper, - default_flow_style=None, - width=50, - indent=0, - ) + yaml.dump(pipeline_full, f) return 0 From ddd12403b24ab8a24035abd74856cfa9f552f4bc Mon Sep 17 00:00:00 2001 From: dkazanc Date: Mon, 27 Jan 2025 17:24:46 +0000 Subject: [PATCH 8/8] redirect to auto generated pipeline --- docs/source/pipelines/yaml.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/pipelines/yaml.rst b/docs/source/pipelines/yaml.rst index 88862f0c2..62fb8f1f1 100644 --- a/docs/source/pipelines/yaml.rst +++ b/docs/source/pipelines/yaml.rst @@ -37,7 +37,7 @@ It is recommended to use GPU-based pipelines and methods from the httomolib and .. dropdown:: Basic GPU pipeline which uses functions from the httomolibgpu library. - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_gpu1.yaml + .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml :language: yaml