From 79a2cc2c9cc7a2b06bbeb294bc810a55f3a0c13f Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 25 May 2021 11:25:32 +0200 Subject: [PATCH] meantime update (#56) * move CI docs CircleCI > GHA * add makefile * update docs conf * fix pillow ver * CI: triggers & skip drafts * CI: drop appveyor --- .github/workflows/ci_experiment.yml | 6 +- .github/workflows/ci_pkg-install.yml | 9 +-- .github/workflows/ci_testing.yml | 9 ++- .github/workflows/docker-builds.yml | 3 + .github/workflows/docs-check.yml | 83 +++++++++++++++++++++ MANIFEST.in | 1 + Makefile | 21 ++++++ appveyor.yml | 71 ------------------ bm_experiments/bm_DROP.py | 6 +- bm_experiments/bm_RVSS.py | 20 ++--- bm_experiments/bm_bUnwarpJ.py | 22 +++--- circle.yml | 91 +---------------------- docs/requirements.txt | 4 +- docs/source/conf.py | 19 +++-- requirements-py27.txt => require-py27.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 17 files changed, 164 insertions(+), 207 deletions(-) create mode 100644 .github/workflows/docs-check.yml create mode 100644 Makefile delete mode 100644 appveyor.yml rename requirements-py27.txt => require-py27.txt (83%) diff --git a/.github/workflows/ci_experiment.yml b/.github/workflows/ci_experiment.yml index 2507779b..f1d8acdd 100644 --- a/.github/workflows/ci_experiment.yml +++ b/.github/workflows/ci_experiment.yml @@ -6,10 +6,12 @@ on: # Trigger the workflow on push or pull request, but only for the master bra branches: [master] pull_request: branches: [master] + types: [opened, reopened, ready_for_review, synchronize] jobs: - pytest: + bash-expt: + if: github.event.pull_request.draft == false runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -34,7 +36,7 @@ jobs: - name: Set py2.7 dependencies if: matrix.python-version == 2.7 run: | - cp -r requirements-py27.txt requirements.txt + cp -r require-py27.txt requirements.txt # Note: This uses an internal pip API and may not always work # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml index c13ba746..a8aa79f2 100644 --- a/.github/workflows/ci_pkg-install.yml +++ b/.github/workflows/ci_pkg-install.yml @@ -1,11 +1,10 @@ name: Install package # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: - # Trigger the workflow on push or pull request, - # but only for the master branch - - push - - pull_request +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: [master] + pull_request: {} jobs: pkg-check: diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml index 350055f3..bdc510e6 100644 --- a/.github/workflows/ci_testing.yml +++ b/.github/workflows/ci_testing.yml @@ -1,9 +1,10 @@ name: CI testing # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: # Trigger the workflow on push or pull request, but only for the master branch - - push - - pull_request +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: [master] + pull_request: {} jobs: pytest: @@ -38,7 +39,7 @@ jobs: - name: Set py2.7 dependencies if: matrix.python-version == 2.7 run: | - cp -r requirements-py27.txt requirements.txt + cp -r require-py27.txt requirements.txt # required for matplotlib @py2 pip install -U backports.functools_lru_cache diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 02150082..c5306ad5 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -6,9 +6,12 @@ on: # Trigger the workflow on push or pull request, but only for the master bra branches: [master] pull_request: branches: [master] + types: [opened, reopened, ready_for_review, synchronize] jobs: + build-push: + if: github.event.pull_request.draft == false runs-on: ubuntu-20.04 strategy: matrix: diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml new file mode 100644 index 00000000..478cdbdc --- /dev/null +++ b/.github/workflows/docs-check.yml @@ -0,0 +1,83 @@ +name: "Docs check" +# https://github.com/marketplace/actions/sphinx-build + +on: +- pull_request + +jobs: + + test-docs: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + # Note: This uses an internal pip API and may not always work + # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + sudo apt-get update && sudo apt-get install -y cmake pandoc + pip --version + pip install -r requirements.txt + pip install -r docs/requirements.txt + pip list + shell: bash + + - name: Test Documentation + run: | + # First run the same pipeline as Read-The-Docs + cd docs + make doctest + + make-docs: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + # Note: This uses an internal pip API and may not always work + # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake pandoc latexmk + sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures + pip --version + pip install -r requirements.txt + pip install -r docs/requirements.txt + pip list + shell: bash + + - name: Make Documentation + run: | + cd docs + make clean + make html --debug --jobs 2 SPHINXOPTS="-W" + make latexpdf + + - name: Upload built docs + uses: actions/upload-artifact@v2 + with: + name: docs-results-${{ runner.os }}-${{ matrix.python-version }} + path: docs/build/html/ + if: success() diff --git a/MANIFEST.in b/MANIFEST.in index 4643244c..424b0afd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,6 +6,7 @@ recursive-exclude __pycache__ *.pyc *.pyo *.orig # Include the README include *.md +exclude Makefile # Include the license file include LICENSE diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..cd187277 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +.PHONY: test clean docs + +clean: + # clean all temp runs + rm -rf .pytest_cache + rm -rf ./docs/build + rm -rf ./docs/source/*.md + rm -rf ./docs/source/api + rm -rf ./docs/source/notebooks + +test: clean + pip install --quiet -r requirements.txt + pip install --quiet -r tests/requirements.txt + + # run tests with coverage + python -m coverage run --source birl -m pytest birl tests -v + python -m coverage report + +docs: clean + pip install --quiet -r docs/requirements.txt + python -m sphinx -b html -W --keep-going docs/source docs/build diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 343985f5..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,71 +0,0 @@ -# https://www.appveyor.com/docs/appveyor-yml/ -environment: - - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script interpreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd" - - matrix: - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. - # See: http://www.appveyor.com/docs/installed-software#python - - #- PYTHON: "C:\\Python27" - # PYTHON_VERSION: "2.7.x" - # PYTHON_ARCH: "32" - - #- PYTHON: "C:\\Python27-x64" - # PYTHON_VERSION: "2.7.x" - # PYTHON_ARCH: "64" - - #- PYTHON: "C:\\Python36-x64" - # PYTHON_VERSION: "3.6.x" - # PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - -build: off - -# https://www.appveyor.com/docs/build-cache/ -cache: - - C:\ProgramData\chocolatey\bin -> appveyor.yml - - C:\ProgramData\chocolatey\lib -> appveyor.yml - - '%LOCALAPPDATA%\pip\Cache -> appveyor.yml' - -# scripts that run after cloning repository -install: - # If there is a newer build queued for the same PR, cancel this one. - # The AppVeyor 'rollout builds' option is supposed to serve the same - # purpose but it is problematic because it tends to cancel builds pushed - # directly to master instead of just PR builds (or the converse). - #- choco upgrade chocolatey - #- choco install -y opencv - # Prepend newly installed Python to the PATH of this build (this cannot be - # done from inside the powershell script as it would require to restart - # the parent CMD process). - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - python -m pip install --upgrade pip - - pip install "setuptools<46" -U # v46 crashes openslide-python install - - pip install -r requirements.txt - - pip install -r ./tests/requirements.txt - -# scripts to run before tests (working directory and environment changes are persisted from the previous steps such as "before_build") -before_test: - - python --version - - pip --version - - pip list - - dir - -# to run your custom scripts instead of automatic tests -test_script: - - coverage run --source birl -m pytest birl tests bm_dataset bm_experiments bm_ANHIR -v - - mkdir results && touch configs/sample_config.yaml - - python bm_experiments/bm_comp_perform.py -o ./results -n 1 - - python birl/bm_template.py -n anhir -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results --preprocessing matching-rgb gray --unique --visual -cfg configs/sample_config.yaml - -on_success: - - coverage report - - codecov diff --git a/bm_experiments/bm_DROP.py b/bm_experiments/bm_DROP.py index e0617b52..a7468d3c 100644 --- a/bm_experiments/bm_DROP.py +++ b/bm_experiments/bm_DROP.py @@ -132,9 +132,9 @@ def _prepare_img_registration(self, item): t_start = time.time() for path_img, col in [(path_im_ref, self.COL_IMAGE_REF), (path_im_move, self.COL_IMAGE_MOVE)]: - item[col + self.COL_IMAGE_EXT_TEMP] = \ - convert_image_to_mhd(path_img, path_out_dir=path_reg_dir, overwrite=False, - to_gray=True, scaling=item.get('scaling', 1.)) + item[col + self.COL_IMAGE_EXT_TEMP] = convert_image_to_mhd( + path_img, path_out_dir=path_reg_dir, overwrite=False, to_gray=True, scaling=item.get('scaling', 1.) + ) item[self.COL_TIME_CONVERT] = time.time() - t_start return item diff --git a/bm_experiments/bm_RVSS.py b/bm_experiments/bm_RVSS.py index c6969fb8..05dc1fa4 100644 --- a/bm_experiments/bm_RVSS.py +++ b/bm_experiments/bm_RVSS.py @@ -86,18 +86,20 @@ class BmRVSS(ImRegBenchmark): DIR_OUTPUTS = 'output' # PATH_SCRIPT_HIST_MATCH_IJM = os.path.join(PATH_IJ_SCRIPTS, 'histogram-matching-for-macro.bsh') #: command for executing the image registration - COMMAND_REGISTRATION = \ - '%(exec_Fiji)s --headless %(path_bsh)s' \ - ' %(dir_input)s/ %(dir_output)s/ %(dir_output)s/' \ + COMMAND_REGISTRATION = ( + '%(exec_Fiji)s --headless %(path_bsh)s' + ' %(dir_input)s/ %(dir_output)s/ %(dir_output)s/' ' %(ref_name)s %(params)s' + ) #: command for executing the warping image and landmarks - COMMAND_WARP_LANDMARKS = \ - '%(exec_Fiji)s --headless %(path_bsh)s' \ - ' %(source)s %(target)s' \ - ' %(output)s/' + BmUnwarpJ.NAME_LANDMARKS + \ - ' %(output)s/' + BmUnwarpJ.NAME_LANDMARKS_WARPED + \ - ' %(transf)s' \ + COMMAND_WARP_LANDMARKS = ( + '%(exec_Fiji)s --headless %(path_bsh)s' + ' %(source)s %(target)s' + ' %(output)s/' + BmUnwarpJ.NAME_LANDMARKS + ' ' + ' %(output)s/' + BmUnwarpJ.NAME_LANDMARKS_WARPED + ' ' + ' %(transf)s' ' %(warp)s' + ) #: required parameters in the configuration file for RVSS REQUIRED_PARAMS_RVSS = ('shrinkingConstraint', 'featuresModelIndex', 'registrationModelIndex') #: default RVSS parameters diff --git a/bm_experiments/bm_bUnwarpJ.py b/bm_experiments/bm_bUnwarpJ.py index d5acca67..cbc45aed 100644 --- a/bm_experiments/bm_bUnwarpJ.py +++ b/bm_experiments/bm_bUnwarpJ.py @@ -100,11 +100,12 @@ class BmUnwarpJ(ImRegBenchmark): # PATH_SCRIPT_HIST_MATCH_IJM = os.path.join(PATH_IJ_SCRIPTS, # 'histogram-matching-for-macro.bsh') #: command for executing the image registration - COMMAND_REGISTRATION = \ - '%(exec_Fiji)s --headless %(path_bsh)s' \ - ' %(source)s %(target)s %(params)s' \ - ' %(output)s/transform-direct.txt' \ + COMMAND_REGISTRATION = ( + '%(exec_Fiji)s --headless %(path_bsh)s' + ' %(source)s %(target)s %(params)s' + ' %(output)s/transform-direct.txt' ' %(output)s/transform-inverse.txt' + ) #: internal name of converted landmarks for tranf. script NAME_LANDMARKS = 'source_landmarks.pts' #: name of warped moving landmarks by tranf. script @@ -114,14 +115,13 @@ class BmUnwarpJ(ImRegBenchmark): #: resulting direct transformation NAME_TRANSF_DIRECT = 'transform-direct.txt' #: command for executing the warping image and landmarks - COMMAND_WARP_LANDMARKS = \ - '%(exec_Fiji)s --headless %(path_bsh)s' \ - ' %(source)s %(target)s' \ - ' %(output)s/' + NAME_LANDMARKS + \ - ' %(output)s/' + NAME_LANDMARKS_WARPED + \ - ' %(transf-inv)s' \ - ' %(transf-dir)s' \ + COMMAND_WARP_LANDMARKS = ( + '%(exec_Fiji)s --headless %(path_bsh)s' + ' %(source)s %(target)s' + ' %(output)s/' + NAME_LANDMARKS + ' %(output)s/' + NAME_LANDMARKS_WARPED + ' %(transf-inv)s' + ' %(transf-dir)s' ' %(warp)s' + ) #: required parameters in the configuration file for bUnwarpJ REQUIRED_PARAMS_BUNWARPJ = ( 'mode', 'subsampleFactor', 'minScale', 'maxScale', 'divWeight', 'curlWeight', 'landmarkWeight', 'imageWeight', diff --git a/circle.yml b/circle.yml index 4b8eaef8..5d1e61ab 100644 --- a/circle.yml +++ b/circle.yml @@ -17,7 +17,7 @@ references: command: | py_ver=$(python -c"import sys; print(sys.version_info.major)") if [ $py_ver -eq 2 ]; then - cp -r requirements-py27.txt requirements.txt ; + cp -r require-py27.txt requirements.txt ; fi pip install -U backports.functools_lru_cache # required for matplotlib @py2 pip install "setuptools<46" -U # v46 crashes openslide-python install @@ -27,13 +27,6 @@ references: pip --version pip list - formatting: &formatting - run: - name: Formatting - command: | - pip install flake8 -q - flake8 . - test_coverage: &test_coverage run: name: Testing and coverage @@ -59,66 +52,9 @@ references: make html --debug --jobs 2 SPHINXOPTS="-W" make latexpdf - run_dataset: &run_dataset - run: - name: Dataset - command: | - mkdir output - python bm_dataset/rescale_tissue_images.py -i "./data-images/rat-kidney_/scale-5pc/*.jpg" --scales 5 -ext .png --nb_workers 2 - python bm_dataset/rescale_tissue_landmarks.py -a ./data-images -d ./output --nb_selected 0.5 --nb_total 200 - python bm_dataset/generate_regist_pairs.py -i "./data-images/images/artificial_*.jpg" -l "./data-images/landmarks/artificial_*.csv" -csv ./data-images/cover_artificial.csv --mode each2all - # python bm_experiments/create_real_synth_dataset.py -i ./data-images/images/Rat_Kidney_HE.jpg -l ./data-images/landmarks/Rat_Kidney_HE.csv -o ./data-images/synth_dataset -nb 3 --nb_workers 1 - # python bm_experiments/generate_regist_pairs.py -i "./data-images/synth_dataset/*.jpg" -l "./data-images/synth_dataset/*.csv" -csv ./data-images/cover_synth-dataset.csv --mode each2all - - run_experiment: &run_experiment - run: - name: General experiments - command: | - mkdir applications && mkdir results && touch configs/sample_config.yaml - # python bm_experiments/bm_comp_perform.py -o ./results -n 1 # reach time-out while running in 36 threads - python birl/bm_template.py -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results --visual --unique -cfg configs/sample_config.yaml - rm ./data-images/*_/*/*_HE.csv # remove target landmarks from histol. tissue - python birl/bm_template.py -n anhir -t ./data-images/pairs-imgs-lnds_histol.csv -d ./data-images -o ./results --preprocessing matching-rgb gray -cfg configs/sample_config.yaml - python bm_experiments/evaluate_experiment.py -d ./data-images -e ./results/BmTemplate_anhir --visual - tree ./results/BmTemplate - - run_bUnwarpJ: &run_bUnwarpJ - run: - name: Experiment - bUnwarpJ - command: | - wget https://downloads.imagej.net/fiji/archive/20200708-1553/fiji-linux64.zip -O applications/fiji-linux64.zip --progress=bar:force:noscroll - unzip -q applications/fiji-linux64.zip -d applications/ - rm applications/fiji-linux64.zip - rm -rf results && mkdir results - python bm_experiments/bm_bUnwarpJ.py -n mix -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_bUnwarpJ_histol.yaml --preprocessing matching-rgb --visual - python bm_experiments/bm_bUnwarpJ.py -n anhir -t ./data-images/pairs-imgs-lnds_histol.csv -d ./data-images -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_bUnwarpJ-SIFT_histol.yaml --unique - tree -L 3 ./results - rm -rf ./applications/Fiji.app - no_output_timeout: 10m - - run_RVSS: &run_RVSS - run: - name: Experiment - RVSS - command: | - wget https://downloads.imagej.net/fiji/archive/20200708-1553/fiji-linux64.zip -O applications/fiji-linux64.zip --progress=bar:force:noscroll - unzip -q applications/fiji-linux64.zip -d applications/ - rm applications/fiji-linux64.zip - rm -rf results && mkdir results - python bm_experiments/bm_RVSS.py -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_RVSS_histol.yaml --visual - tree -L 3 ./results - rm -rf ./applications/Fiji.app - no_output_timeout: 10m - jobs: - Formatting: - docker: - - image: circleci/python:3.6 - steps: - - checkout - - *formatting - Py3-Tests: docker: - image: circleci/python:3.6 @@ -143,31 +79,11 @@ jobs: - store_artifacts: path: test-reports - Py3-Experiments: - docker: - - image: circleci/python:3.6 - steps: &steps_expts - - checkout - # INSTALLATION - - *install_deps - #- run: git lfs pull - - *install_pips - # EXPERIMENTS - - *run_dataset - - *run_experiment - - *run_bUnwarpJ - - *run_RVSS - Py2-Tests: docker: - image: circleci/python:2.7 steps: *steps_tests - Py2-Experiments: - docker: - - image: circleci/python:2.7 - steps: *steps_expts - Build-Docs: docker: - image: readthedocs/build:latest @@ -187,9 +103,6 @@ workflows: version: 2 build: jobs: - - Formatting - Py2-Tests - Py3-Tests - #- Py2-Experiments - #- Py3-Experiments - - Build-Docs + #- Build-Docs diff --git a/docs/requirements.txt b/docs/requirements.txt index 8fe16f2c..228dd1d7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ sphinx>=2.0, <3.0 -recommonmark # fails with badges m2r # fails with multi-line text -nbsphinx +myst-parser +nbsphinx>=0.8.5 pandoc docutils<0.15 # higher version breaks py2 imgconverter diff --git a/docs/source/conf.py b/docs/source/conf.py index b42202b6..ed0fb706 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -90,7 +90,7 @@ 'sphinx.ext.linkcode', 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', - 'recommonmark', + 'myst_parser', # 'm2r', 'nbsphinx', # https://github.com/sphinx-doc/sphinx/issues/4720 @@ -107,6 +107,8 @@ nbsphinx_execute = 'never' nbsphinx_allow_errors = True +myst_update_mathjax = False + # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # @@ -347,10 +349,11 @@ def find_source(): autodoc_member_order = 'groupwise' autoclass_content = 'both' -autodoc_default_flags = [ - 'members', - 'undoc-members', - 'show-inheritance', - 'private-members', - # 'special-members', 'inherited-members' -] +autodoc_default_options = { + 'members': True, + 'undoc-members': True, + 'private-members': True, + 'methods': True, + 'exclude-members': '_abc_impl', + 'show-inheritance': True, +} diff --git a/requirements-py27.txt b/require-py27.txt similarity index 83% rename from requirements-py27.txt rename to require-py27.txt index 0adbf6ec..24ea7408 100644 --- a/requirements-py27.txt +++ b/require-py27.txt @@ -2,7 +2,7 @@ numpy>=1.9 scipy>=0.10.0 pandas>=0.17.1 six>=1.7.3 -pillow>=4.0, <7 # 7.x fails to load PNG via skimage +pillow matplotlib>=2.0.2, <3.0.0 # new version does not support py2 tqdm>=4.7.4, <=4.30 # higher fails ascii for py2 pyyaml diff --git a/requirements.txt b/requirements.txt index 4d688edd..95957587 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy>=1.13.3 scipy>=1.0 pandas>=0.21 six>=1.10 -pillow>=4.0, !=7 # 7.x fails to load PNG via skimage +pillow>=8.1.2 matplotlib>=2.0.2 tqdm>=4.7.4 pyyaml>=3.12 diff --git a/setup.py b/setup.py index 23867371..e5c10e3d 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def load_requirements(fname='requirements.txt'): if sys.version_info.major == 2: - requirements = load_requirements('requirements-py27.txt') + requirements = load_requirements('require-py27.txt') else: requirements = load_requirements('requirements.txt')