diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index 4bda85575..62e0f0c46 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -31,6 +31,14 @@ jobs: activate-environment: httomo-docs environment-file: ./docs/source/doc-conda-requirements.yml + - name: Install httomo-backends + run: | + pip install --no-deps httomo-backends + + - name: Generate full yaml pipelines using directives + run: | + python ./docs/source/scripts/yaml_pipelines_generator.py -i ./docs/source/pipelines_full/gpu_pipeline1_directive.yaml -o ./docs/source/pipelines_full/gpu_pipeline1.yaml + - name: Build docs run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ diff --git a/.github/workflows/main_conda_package_test.yml b/.github/workflows/main_conda_package_test.yml deleted file mode 100644 index 84dd2f120..000000000 --- a/.github/workflows/main_conda_package_test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Weekly conda package test - -on: - schedule: - - cron: '55 0 * * 1' # At 00:55 every Monday - -jobs: - download-test-iris: - runs-on: iris-gpu - container: - image: nvidia/cuda:11.6.2-devel-ubi8 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - ref: "main" - fetch-depth: 0 - - - name: Create conda environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: conda/environment.yml - environment-name: httomo - post-cleanup: 'all' - init-shell: bash - - - name: Install httomo - run: | - micromamba activate httomo - micromamba install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -y - micromamba list - - - name: Generate yaml templates - run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/httomo_modules.yaml -o ./httomo/yaml_templates/httomo - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./httomo/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./httomo/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./httomo/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./httomo/yaml_templates/httomolibgpu - - - name: Run tests - run: | - pytest tests/ diff --git a/.github/workflows/main_weekly_build.yml b/.github/workflows/main_weekly_build.yml deleted file mode 100644 index af792fa0a..000000000 --- a/.github/workflows/main_weekly_build.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: HTTomo main build (weekly) - -on: - schedule: - - cron: '0 0 * * 1' # At 00:00 on Monday - -jobs: - - install-httomo-main-test-iris: - runs-on: iris-gpu - container: - image: nvidia/cuda:11.6.2-devel-ubi8 - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - - - name: Create conda environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: conda/environment.yml - environment-name: httomo - post-cleanup: 'all' - init-shell: bash - - - name: Install httomo-main - run: | - micromamba activate httomo - pip install .[dev] - micromamba list - - - name: Generate yaml templates - run: | - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/httomo_modules.yaml -o ./httomo/yaml_templates/httomo - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./httomo/yaml_templates/tomopy - python ./scripts/yaml_unsupported_tomopy_remove.py -t ./httomo/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./httomo/yaml_templates/httomolib - python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./httomo/yaml_templates/httomolibgpu - - - name: Run tests - run: | - pytest tests/ - - conda-upload: - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - ref: "main" - fetch-depth: 0 - - # setup Python 3.10 - - name: Setup Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies with Conda - run: | - $CONDA/bin/conda install -c conda-forge conda-build - $CONDA/bin/conda install -c conda-forge anaconda-client - $CONDA/bin/conda update conda - $CONDA/bin/conda update conda-build - $CONDA/bin/conda list - - - name: Decrypt a secret - run: ./.scripts/decrypt_secret.sh - env: - LARGE_SECRET_PASSPHRASE: ${{ secrets.LARGE_SECRET_PASSPHRASE }} - - - name: Build and upload the package to httomo conda cloud - env: - LABEL: main - run: | - chmod +x ./.scripts/conda_upload.sh - ./.scripts/conda_upload.sh - diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_iris.yml index d17e8f187..c107a8513 100644 --- a/.github/workflows/run_tests_iris.yml +++ b/.github/workflows/run_tests_iris.yml @@ -36,7 +36,7 @@ jobs: run: | micromamba activate httomo pip install --upgrade --force-reinstall pillow - pip install httomolibgpu tomobar ccpi-regularisation-cupy + pip install httomolibgpu tomobar pip install --no-deps httomo-backends pip install . micromamba list diff --git a/conda/environment.yml b/conda/environment.yml index 4c2471188..52c5f7ddf 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,6 +1,7 @@ name: httomo channels: - conda-forge + - httomo dependencies: - astra-toolbox - click @@ -23,4 +24,5 @@ dependencies: - pytest-cov - pytest-mock - tomopy=1.15 + - ccpi-regulariser diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml index e8a15fd8a..dfeaf21eb 100644 --- a/docs/source/doc-conda-requirements.yml +++ b/docs/source/doc-conda-requirements.yml @@ -17,3 +17,5 @@ dependencies: - loguru - graypy - tqdm + - ruamel.yaml>=0.18 + - pip diff --git a/docs/source/pipelines/yaml.rst b/docs/source/pipelines/yaml.rst index 88862f0c2..62fb8f1f1 100644 --- a/docs/source/pipelines/yaml.rst +++ b/docs/source/pipelines/yaml.rst @@ -37,7 +37,7 @@ It is recommended to use GPU-based pipelines and methods from the httomolib and .. dropdown:: Basic GPU pipeline which uses functions from the httomolibgpu library. - .. literalinclude:: ../../../tests/samples/pipeline_template_examples/pipeline_gpu1.yaml + .. literalinclude:: ../pipelines_full/gpu_pipeline1.yaml :language: yaml diff --git a/docs/source/pipelines_full/gpu_pipeline1_directive.yaml b/docs/source/pipelines_full/gpu_pipeline1_directive.yaml new file mode 100644 index 000000000..9b6d2533a --- /dev/null +++ b/docs/source/pipelines_full/gpu_pipeline1_directive.yaml @@ -0,0 +1,18 @@ +- method: standard_tomo + module_path: httomo.data.hdf.loaders +- method: find_center_vo + module_path: httomolibgpu.recon.rotation +- method: remove_outlier + module_path: httomolibgpu.misc.corr +- method: normalize + module_path: httomolibgpu.prep.normalize +- method: remove_all_stripe + module_path: httomolibgpu.prep.stripe +- method: FBP + module_path: httomolibgpu.recon.algorithm +- method: calculate_stats + module_path: httomo.methods +- method: rescale_to_int + module_path: httomolibgpu.misc.rescale +- method: save_to_images + module_path: httomolib.misc.images diff --git a/docs/source/scripts/yaml_pipelines_generator.py b/docs/source/scripts/yaml_pipelines_generator.py new file mode 100644 index 000000000..0892463d4 --- /dev/null +++ b/docs/source/scripts/yaml_pipelines_generator.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# --------------------------------------------------------------------------- +# Copyright 2022 Diamond Light Source Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +# Created By : Tomography Team +# Created Date: 22/January/2025 +# version ='0.1' +# --------------------------------------------------------------------------- +"""Script that generates YAML pipeline for HTTomo using YAML templates from httomo-backends +(should be installed in environment). + +Please run the generator as: + python -m yaml_pipelines_generator -i /path/to/pipelines.yml -o /path/to/output/ +""" +import argparse +import os +import ruamel.yaml +import httomo_backends + +CS = ruamel.yaml.comments.CommentedSeq # defaults to block style + + +def __represent_none(self, data): + return self.represent_scalar("tag:yaml.org,2002:null", "null") + + +def yaml_pipelines_generator( + path_to_pipelines: str, path_to_httomobackends: str, path_to_output_file: str +) -> int: + """function that builds YAML pipeline using YAML templates from httomo-backends + + Args: + path_to_pipelines: path to the YAML file which contains a high-level description of the required pipeline to be built. + path_to_httomobackends: path to httomo-backends on the system, where YAML templates stored. + path_to_output_file: path to output file with the generated pipeline + + Returns: + returns zero if the processing is successful + """ + + yaml = ruamel.yaml.YAML(typ="rt", pure=True) + + # open YAML file to inspect + with open(path_to_pipelines, "r") as file: + try: + pipeline_file_content = yaml.load(file) + except OSError as e: + print("loading yaml file with methods failed", e) + + with open(path_to_output_file, "w") as f: + # a loop over methods in the high-level pipeline file (directive) + methods_no = len(pipeline_file_content) + pipeline_full = CS() + for i in range(methods_no): + method_content = pipeline_file_content[i] + method_name = method_content["method"] + module_name = method_content["module_path"] + # get the corresponding yaml template from httomo-backends + backend_name = module_name[0 : module_name.find(".")] + full_path_to_yamls = ( + path_to_httomobackends + + "/yaml_templates/" + + backend_name + + "/" + + module_name + + "/" + + method_name + + ".yaml" + ) + with open(full_path_to_yamls, "r") as stream: + try: + yaml_template_method = yaml.load(stream) + except OSError as e: + print("loading yaml template failed", e) + + if "loaders" in module_name: + # should be the first method in the list + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Standard tomography loader for NeXus files. ---", + indent=0, + ) + pipeline_full += yaml_template_method + elif "rotation" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Center of Rotation auto-finding. Required for reconstruction bellow. ---", + indent=0, + ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="ind", + comment="A vertical slice (sinogram) index to calculate CoR, `mid` can be used for middle", + ) + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="cor_initialisation_value", + comment="Use if an approximate CoR is known", + ) + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="average_radius", + comment="Average several sinograms to improve SNR, one can try 3-5 range", + ) + pipeline_full[i]["side_outputs"].yaml_add_eol_comment( + key="cor", + comment="A side output of the method, here a CoR scalar value", + ) + elif "corr" in module_name and "remove_outlier" in method_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Removing dead pixels in the data, aka zingers. Use if sharp streaks are present in reconstruction. ---", + indent=0, + ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="dif", + comment="A difference between the outlier value and the median value of neighbouring pixels.", + ) + elif "normalize" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Normalisation of projection data using collected flats/darks images. --- ", + indent=0, + ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="minus_log", + comment="If Paganin method is used bellow, set it to false.", + ) + elif "stripe" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Method to remove stripe artefacts in the data that lead to ring artefacts in the reconstruction. --- ", + indent=0, + ) + pipeline_full += yaml_template_method + elif "algorithm" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Reconstruction method. ---", + indent=0, + ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="center", + comment="Reference to center of rotation side output OR an integer.", + ) + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="recon_mask_radius", + comment="Zero pixels outside the mask-circle radius.", + ) + elif "calculate_stats" in method_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Calculate global statistics on the reconstructed volume, required for data rescaling. ---", + indent=0, + ) + pipeline_full += yaml_template_method + elif "rescale_to_int" in method_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Rescaling the data using min/max obtained from `calculate_stats`. ---", + indent=0, + ) + pipeline_full += yaml_template_method + elif "images" in module_name: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--- Saving data into images. ---", + indent=0, + ) + pipeline_full += yaml_template_method + pipeline_full[i]["parameters"].yaml_add_eol_comment( + key="file_format", + comment="`tif` or `jpeg` can be used.", + ) + else: + pipeline_full.yaml_set_comment_before_after_key( + i, + "--------------------------------------------------------#", + indent=0, + ) + pipeline_full += yaml_template_method + + yaml.representer.add_representer(type(None), __represent_none) + yaml.dump(pipeline_full, f) + + return 0 + + +def get_args(): + parser = argparse.ArgumentParser( + description="Script that generates YAML pipelines for HTTomo " + "using YAML templates from httomo-backends." + ) + parser.add_argument( + "-i", + "--input", + type=str, + default=None, + help="A path to the list of pipelines needed to be built within a yaml file", + ) + parser.add_argument( + "-o", + "--output", + type=str, + default="./", + help="Full path to the yaml file with the generated pipeline.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + path_to_httomobackends = os.path.dirname(httomo_backends.__file__) + args = get_args() + path_to_pipelines = args.input + path_to_output_file = args.output + return_val = yaml_pipelines_generator( + path_to_pipelines, path_to_httomobackends, path_to_output_file + ) + if return_val == 0: + print("YAML pipeline has been successfully generated!")