From 31671c2e9ed469c1c29f61aef5ade74c0286bbd6 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 29 Apr 2024 12:34:31 -0400 Subject: [PATCH 01/37] downgrade mac-os to 13 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f3f2c071..d7851dd3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: python-version: [3.8, 3.9, "3.10", 3.11] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-13] steps: - uses: s-weigand/setup-conda@v1 - uses: actions/checkout@v2 From 9a0afd2022ab7d8759b0c74391dc8aa85f7fa56e Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 29 Apr 2024 13:14:18 -0400 Subject: [PATCH 02/37] downgrade to macos-12 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index d7851dd3..3034b47d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: python-version: [3.8, 3.9, "3.10", 3.11] - os: [ubuntu-latest, windows-latest, macos-13] + os: [ubuntu-latest, windows-latest, macos-12] steps: - uses: s-weigand/setup-conda@v1 - uses: actions/checkout@v2 From edb9e859768d04b31087f57ef2748d10591ef4d8 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 10:21:15 -0700 Subject: [PATCH 03/37] removed setup-conda --- .github/workflows/run-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 3034b47d..a1191c08 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -38,9 +38,8 @@ jobs: fail-fast: false matrix: python-version: [3.8, 3.9, "3.10", 3.11] - os: [ubuntu-latest, windows-latest, macos-12] + os: [ubuntu-latest, windows-latest, macos-13] steps: - - uses: s-weigand/setup-conda@v1 - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags - name: Setup Python ${{ matrix.python-version }} From d229fc6915db06a2c54217a6a4043698526ef184 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 10:28:44 -0700 Subject: [PATCH 04/37] added macos-13 to update testing data --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index d012a203..c7c0d81e 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - uses: s-weigand/setup-conda@v1 - uses: actions/checkout@v2 From d4a833058e14f88a0e6fac47bfc5c3347700fa15 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 10:30:08 -0700 Subject: [PATCH 05/37] removed conda --- .github/workflows/update-testing-data.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index c7c0d81e..a7f59dfd 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -13,7 +13,6 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - - uses: s-weigand/setup-conda@v1 - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags - name: Setup Python 3.11 From 0886d36df80c4fd4990213b014ef1311b6d38419 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 10:35:50 -0700 Subject: [PATCH 06/37] added 3.12 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a1191c08..0feddef0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, "3.10", 3.11] + python-version: [3.8, 3.9, "3.10", 3.11, 3.12] os: [ubuntu-latest, windows-latest, macos-13] steps: - uses: actions/checkout@v2 From 031d80cd6ca28451b2522064d3f032395a347329 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 12:00:43 -0700 Subject: [PATCH 07/37] added quotes to version numbers --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0feddef0..5fe8c8ac 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.9, "3.10", 3.11, 3.12] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: [ubuntu-latest, windows-latest, macos-13] steps: - uses: actions/checkout@v2 From c135135acf3a1ca9895f8cc7a95accb720da87d6 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 13:19:44 -0700 Subject: [PATCH 08/37] added macos-latest --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 5fe8c8ac..80095510 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, windows-latest, macos-13] + os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags From c644ebadc87deda8b8e62debd98c89fcd2a18e04 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 29 Apr 2024 16:00:48 -0700 Subject: [PATCH 09/37] removed macos-latest --- .github/workflows/run-tests.yml | 2 +- .github/workflows/update-testing-data.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 80095510..07b511f4 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip + os: [ubuntu-latest, windows-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index a7f59dfd..b52fdc6e 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip + os: [ubuntu-latest, windows-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags From de710b19d06bc4acc9ee9c5e3f9f4e4a0877e9f8 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 10:03:13 -0700 Subject: [PATCH 10/37] added dev testing --- .github/workflows/dev-testing.yaml | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/dev-testing.yaml diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml new file mode 100644 index 00000000..e65a190f --- /dev/null +++ b/.github/workflows/dev-testing.yaml @@ -0,0 +1,53 @@ +name: Dev Testing +on: + pull_request: + workflow_dispatch: + +jobs: + run: + name: Ubuntu tests with Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - run: git fetch --prune --unshallow --tags + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Global Setup + run: | + pip install -U pip + pip install pytest-xdist<=3.5.0 + git config --global user.email "CI@example.com" + git config --global user.name "CI Almighty" + pip install wheel==0.41.2 # needed for scanimage + + - name: Install full requirements + run: pip install --no-cache-dir .[full,test] + + - name: Clone and Install NeuroConv + run: | + git clone https://github.com/catalystneuro/neuroconv.git + cd neuroconv + pip install --no-cache-dir .[full,test] + cd ../ + - name: Get ophys_testing_data current head hash + id: ophys + run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + uses: actions/cache@v2 + id: cache-ophys-datasets + with: + path: ./ophys_testing_data + key: ophys-datasets-042023-ubuntu-latest-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + + - name: Run NeuroConv tests + run: | + pytest -n auto --dist loadscope neuroconv/tests/test_ophys + pytest -n auto --dist loadscope neuroconv/tests/test_on_data/test_imaging_interfaces.py + pytest -n auto --dist loadscope neuroconv/tests/test_on_data/test_segmentation_interfaces.py From 6c7250a7d0cf4103004b9e8ac8b57d5164268b36 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 10:09:25 -0700 Subject: [PATCH 11/37] unpin xdist --- .github/workflows/dev-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml index e65a190f..d9e9a141 100644 --- a/.github/workflows/dev-testing.yaml +++ b/.github/workflows/dev-testing.yaml @@ -22,7 +22,7 @@ jobs: - name: Global Setup run: | pip install -U pip - pip install pytest-xdist<=3.5.0 + pip install pytest-xdist git config --global user.email "CI@example.com" git config --global user.name "CI Almighty" pip install wheel==0.41.2 # needed for scanimage From 842922553b5876c3ad01f685eac104ca062c451c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 10:22:08 -0700 Subject: [PATCH 12/37] condense tests into 1 line for better output --- .github/workflows/dev-testing.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml index d9e9a141..f7302fe8 100644 --- a/.github/workflows/dev-testing.yaml +++ b/.github/workflows/dev-testing.yaml @@ -48,6 +48,4 @@ jobs: - name: Run NeuroConv tests run: | - pytest -n auto --dist loadscope neuroconv/tests/test_ophys - pytest -n auto --dist loadscope neuroconv/tests/test_on_data/test_imaging_interfaces.py - pytest -n auto --dist loadscope neuroconv/tests/test_on_data/test_segmentation_interfaces.py + pytest -n auto --dist loadscope neuroconv/tests/test_ophys neuroconv/tests/test_on_data/test_imaging_interfaces.py neuroconv/tests/test_on_data/test_segmentation_interfaces.py From c46a6cbac703774558665cf52ddbc072ca432af8 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 10:28:38 -0700 Subject: [PATCH 13/37] downgrade python setup for wheel support --- .github/workflows/dev-testing.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml index f7302fe8..b59bd5cd 100644 --- a/.github/workflows/dev-testing.yaml +++ b/.github/workflows/dev-testing.yaml @@ -12,10 +12,10 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags - - name: Setup Python - uses: actions/setup-python@v5 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 9cf698c836244a894303dcabe17f59269bb2a599 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 10:44:33 -0700 Subject: [PATCH 14/37] added macos-latest with scanimage skips --- .github/workflows/run-tests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 07b511f4..bdae1764 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, windows-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip + os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags @@ -74,7 +74,11 @@ jobs: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Run full pytest with coverage + - name: Run full pytest with coverage (macos-latest) + if: ${{ matrix.os == 'macos-latest' }} + run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,test_scanimage_utils.py,test_scanimagetiffimagingextractor.py + - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) + if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml - if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'}} name: Upload full coverage to Codecov From 7c9e8a29c9f04594bfed3e5c2f26e932b02a4ca2 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 10:51:19 -0700 Subject: [PATCH 15/37] continue on collection errors --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index bdae1764..26240565 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,7 +76,7 @@ jobs: - name: Run full pytest with coverage (macos-latest) if: ${{ matrix.os == 'macos-latest' }} - run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,test_scanimage_utils.py,test_scanimagetiffimagingextractor.py + run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,test_scanimage_utils.py,test_scanimagetiffimagingextractor.py --continue-on-collection-errors - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From 1c01b400956885c45b6c2dd27027f02510d35977 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:00:50 -0700 Subject: [PATCH 16/37] added platform check for scanimage import --- tests/test_scan_image_tiff.py | 7 ++++++- tests/test_scanimagetiffimagingextractor.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_scan_image_tiff.py b/tests/test_scan_image_tiff.py index a0d7c25d..88d58287 100644 --- a/tests/test_scan_image_tiff.py +++ b/tests/test_scan_image_tiff.py @@ -5,7 +5,12 @@ from hdmf.testing import TestCase from numpy.testing import assert_array_equal -from ScanImageTiffReader import ScanImageTiffReader +import platform + +if ( + platform.processor() != "arm" +): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 + from ScanImageTiffReader import ScanImageTiffReader from roiextractors import TiffImagingExtractor, ScanImageTiffImagingExtractor diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 92e943e1..5eefe42e 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -1,6 +1,11 @@ import pytest from numpy.testing import assert_array_equal -from ScanImageTiffReader import ScanImageTiffReader +import platform + +if ( + platform.processor() != "arm" +): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 + from ScanImageTiffReader import ScanImageTiffReader from roiextractors import ( ScanImageTiffSinglePlaneImagingExtractor, ScanImageTiffMultiPlaneImagingExtractor, From 0fdc21d67e0e10a88249b827094a5980626dff75 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:01:12 -0700 Subject: [PATCH 17/37] added platform check for scanimage import --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 26240565..bdae1764 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,7 +76,7 @@ jobs: - name: Run full pytest with coverage (macos-latest) if: ${{ matrix.os == 'macos-latest' }} - run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,test_scanimage_utils.py,test_scanimagetiffimagingextractor.py --continue-on-collection-errors + run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,test_scanimage_utils.py,test_scanimagetiffimagingextractor.py - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From ef308b1d4e4243fbe0ecd95354307c808d294f6a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:25:28 -0700 Subject: [PATCH 18/37] switched to try-except --- tests/test_scan_image_tiff.py | 10 +++++++--- tests/test_scanimagetiffimagingextractor.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_scan_image_tiff.py b/tests/test_scan_image_tiff.py index 88d58287..c70466f0 100644 --- a/tests/test_scan_image_tiff.py +++ b/tests/test_scan_image_tiff.py @@ -7,10 +7,14 @@ from numpy.testing import assert_array_equal import platform -if ( - platform.processor() != "arm" -): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 +# if ( +# platform.processor() != "arm" +# ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 +# from ScanImageTiffReader import ScanImageTiffReader +try: from ScanImageTiffReader import ScanImageTiffReader +except OSError: + pass from roiextractors import TiffImagingExtractor, ScanImageTiffImagingExtractor diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 5eefe42e..325e8b84 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -2,10 +2,14 @@ from numpy.testing import assert_array_equal import platform -if ( - platform.processor() != "arm" -): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 +# if ( +# platform.processor() != "arm" +# ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 +# from ScanImageTiffReader import ScanImageTiffReader +try: from ScanImageTiffReader import ScanImageTiffReader +except OSError: + pass from roiextractors import ( ScanImageTiffSinglePlaneImagingExtractor, ScanImageTiffMultiPlaneImagingExtractor, From 06fb8c06f16bebcafffe8cbe684ac4ed0e998631 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:35:33 -0700 Subject: [PATCH 19/37] fixed ignores --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index bdae1764..7c2281d1 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,7 +76,7 @@ jobs: - name: Run full pytest with coverage (macos-latest) if: ${{ matrix.os == 'macos-latest' }} - run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,test_scanimage_utils.py,test_scanimagetiffimagingextractor.py + run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,tests/test_scanimage_utils.py,tests/test_scanimagetiffimagingextractor.py - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From fa3280a33ab8747e78e24da11a6a974e8598a35d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:45:24 -0700 Subject: [PATCH 20/37] fixed ignores --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7c2281d1..9ac873df 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,7 +76,7 @@ jobs: - name: Run full pytest with coverage (macos-latest) if: ${{ matrix.os == 'macos-latest' }} - run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py,tests/test_scanimage_utils.py,tests/test_scanimagetiffimagingextractor.py + run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py --ignore=tests/test_scanimage_utils.py --ignore=tests/test_scanimagetiffimagingextractor.py - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From 041cde7b40338ab84cdcbdb3ce6dc1d118792e0c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:48:05 -0700 Subject: [PATCH 21/37] updated dev install --- .github/workflows/dev-testing.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml index b59bd5cd..e7589117 100644 --- a/.github/workflows/dev-testing.yaml +++ b/.github/workflows/dev-testing.yaml @@ -28,7 +28,9 @@ jobs: pip install wheel==0.41.2 # needed for scanimage - name: Install full requirements - run: pip install --no-cache-dir .[full,test] + run: | + pip install .[test] + pip install .[full] - name: Clone and Install NeuroConv run: | From ba6050f82ffdb54c4ba0f4376f0bf350c307f659 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:51:59 -0700 Subject: [PATCH 22/37] fixed ignores --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9ac873df..da88da4d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -76,7 +76,7 @@ jobs: - name: Run full pytest with coverage (macos-latest) if: ${{ matrix.os == 'macos-latest' }} - run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/tests_scan_image_tiff.py --ignore=tests/test_scanimage_utils.py --ignore=tests/test_scanimagetiffimagingextractor.py + run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/test_scan_image_tiff.py --ignore=tests/test_scanimage_utils.py --ignore=tests/test_scanimagetiffimagingextractor.py --ignore=tests/test_io.py - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From 69e86fe863a6063998ac8d5a70a62909225de859 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 12:57:35 -0700 Subject: [PATCH 23/37] added formatting --- .github/workflows/run-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index da88da4d..92a8a4f9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -74,9 +74,14 @@ jobs: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Run full pytest with coverage (macos-latest) + - name: Run full pytest (macos-latest) if: ${{ matrix.os == 'macos-latest' }} - run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml --ignore=tests/test_scan_image_tiff.py --ignore=tests/test_scanimage_utils.py --ignore=tests/test_scanimagetiffimagingextractor.py --ignore=tests/test_io.py + run: | + pytest -n auto --dist loadscope \ + --ignore=tests/test_scan_image_tiff.py \ + --ignore=tests/test_scanimage_utils.py \ + --ignore=tests/test_scanimagetiffimagingextractor.py \ + --ignore=tests/test_io.py - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) if: ${{ matrix.os != 'macos-latest' }} run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From 062c0432fc55e65711e67ddaa984b8312ecad5f7 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 13:08:28 -0700 Subject: [PATCH 24/37] added macos-latest to update-testing-data --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index b52fdc6e..a7f59dfd 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip + os: [ubuntu-latest, windows-latest, macos-latest, macos-13] # macos-13 is the latest version of macOS with Intel chip steps: - uses: actions/checkout@v2 - run: git fetch --prune --unshallow --tags From 4c35b055cd494b53b2f32fff63534f8f06d75ec6 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 13:10:44 -0700 Subject: [PATCH 25/37] switched back to if-else import --- tests/test_scan_image_tiff.py | 10 +++------- tests/test_scanimagetiffimagingextractor.py | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/test_scan_image_tiff.py b/tests/test_scan_image_tiff.py index c70466f0..88d58287 100644 --- a/tests/test_scan_image_tiff.py +++ b/tests/test_scan_image_tiff.py @@ -7,14 +7,10 @@ from numpy.testing import assert_array_equal import platform -# if ( -# platform.processor() != "arm" -# ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 -# from ScanImageTiffReader import ScanImageTiffReader -try: +if ( + platform.processor() != "arm" +): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 from ScanImageTiffReader import ScanImageTiffReader -except OSError: - pass from roiextractors import TiffImagingExtractor, ScanImageTiffImagingExtractor diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 325e8b84..5eefe42e 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -2,14 +2,10 @@ from numpy.testing import assert_array_equal import platform -# if ( -# platform.processor() != "arm" -# ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 -# from ScanImageTiffReader import ScanImageTiffReader -try: +if ( + platform.processor() != "arm" +): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 from ScanImageTiffReader import ScanImageTiffReader -except OSError: - pass from roiextractors import ( ScanImageTiffSinglePlaneImagingExtractor, ScanImageTiffMultiPlaneImagingExtractor, From 4f441792864ca910ff0a6547b84b01d3c2c58a08 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 30 Apr 2024 16:19:27 -0700 Subject: [PATCH 26/37] updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88fe901c..a38f716d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Features +* Updated testing workflows to include python 3.12, m1/intel macos, and dev tests to check neuroconv: [PR #317](https://github.com/catalystneuro/roiextractors/pull/317) + ### Fixes * Remove unecessary scipy import error handling [#315]((https://github.com/catalystneuro/roiextractors/pull/315) From c95957d43cb58beda5a1342072ec4160b49858b3 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 1 May 2024 17:01:18 -0400 Subject: [PATCH 27/37] try pytestmark --- tests/test_scanimagetiffimagingextractor.py | 27 ++++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 5eefe42e..46b2614a 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -1,11 +1,9 @@ +import sys +import platform + import pytest from numpy.testing import assert_array_equal -import platform -if ( - platform.processor() != "arm" -): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 - from ScanImageTiffReader import ScanImageTiffReader from roiextractors import ( ScanImageTiffSinglePlaneImagingExtractor, ScanImageTiffMultiPlaneImagingExtractor, @@ -16,6 +14,8 @@ from .setup_paths import OPHYS_DATA_PATH +IS_ARM64_MAC = sys.platform == "darwin" and platform.processor() == "arm" +pytestmark = pytest.mark.skipif(IS_ARM64_MAC) @pytest.fixture(scope="module") def file_path(): @@ -49,13 +49,11 @@ def expected_properties(): def scan_image_tiff_single_plane_imaging_extractor(request, file_path): return ScanImageTiffSinglePlaneImagingExtractor(file_path=file_path, **request.param) - @pytest.mark.parametrize("channel_name, plane_name", [("Invalid Channel", "0"), ("Channel 1", "Invalid Plane")]) def test_ScanImageTiffSinglePlaneImagingExtractor__init__invalid(file_path, channel_name, plane_name): with pytest.raises(ValueError): ScanImageTiffSinglePlaneImagingExtractor(file_path=file_path, channel_name=channel_name, plane_name=plane_name) - def test_ScanImageTiffSinglePlaneImagingExtractor__init__metadata_provided(file_path): metadata = extract_extra_metadata(file_path) parsed_metadata = parse_metadata(metadata) @@ -69,7 +67,6 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__metadata_provided(file_ assert extractor.metadata == metadata assert extractor.parsed_metadata == parsed_metadata - def test_ScanImageTiffSinglePlaneImagingExtractor__init__invalid_metadata_provided(file_path): metadata = {"invalid_key": "invalid_value"} parsed_metadata = {"invalid_key": "invalid_value"} @@ -82,7 +79,6 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__invalid_metadata_provid parsed_metadata=parsed_metadata, ) - def test_ScanImageTiffSinglePlaneImagingExtractor__init__parsed_metadata_not_provided(file_path): metadata = extract_extra_metadata(file_path) with pytest.raises(AssertionError): @@ -93,9 +89,10 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__parsed_metadata_not_pro metadata=metadata, ) - @pytest.mark.parametrize("frame_idxs", (0, [0, 1, 2], [0, 2, 5])) def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, expected_properties): + from ScanImageTiffReader import ScanImageTiffReader + frames = scan_image_tiff_single_plane_imaging_extractor.get_frames(frame_idxs=frame_idxs) file_path = str(scan_image_tiff_single_plane_imaging_extractor.file_path) plane = scan_image_tiff_single_plane_imaging_extractor.plane @@ -105,6 +102,7 @@ def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, frames_per_slice = expected_properties["frames_per_slice"] if isinstance(frame_idxs, int): frame_idxs = [frame_idxs] + raw_idxs = [] for idx in frame_idxs: cycle = idx // frames_per_slice @@ -116,18 +114,19 @@ def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, + channel ) raw_idxs.append(raw_idx) + with ScanImageTiffReader(file_path) as io: assert_array_equal(frames, io.data()[raw_idxs]) - @pytest.mark.parametrize("frame_idxs", ([-1], [50])) def test_get_frames_invalid(scan_image_tiff_single_plane_imaging_extractor, frame_idxs): with pytest.raises(ValueError): scan_image_tiff_single_plane_imaging_extractor.get_frames(frame_idxs=frame_idxs) - @pytest.mark.parametrize("frame_idx", (1, 3, 5)) def test_get_single_frame(scan_image_tiff_single_plane_imaging_extractor, expected_properties, frame_idx): + from ScanImageTiffReader import ScanImageTiffReader + frame = scan_image_tiff_single_plane_imaging_extractor._get_single_frame(frame=frame_idx) file_path = str(scan_image_tiff_single_plane_imaging_extractor.file_path) plane = scan_image_tiff_single_plane_imaging_extractor.plane @@ -160,6 +159,8 @@ def test_get_video( start_frame, end_frame, ): + from ScanImageTiffReader import ScanImageTiffReader + video = scan_image_tiff_single_plane_imaging_extractor.get_video(start_frame=start_frame, end_frame=end_frame) if start_frame is None: start_frame = 0 @@ -171,6 +172,7 @@ def test_get_video( num_planes = expected_properties["num_planes"] num_channels = expected_properties["num_channels"] frames_per_slice = expected_properties["frames_per_slice"] + raw_idxs = [] for idx in range(start_frame, end_frame): cycle = idx // frames_per_slice @@ -182,6 +184,7 @@ def test_get_video( + channel ) raw_idxs.append(raw_idx) + with ScanImageTiffReader(file_path) as io: assert_array_equal(video, io.data()[raw_idxs]) From 2821f04ca5d5f0d99c4de23563688f8ea1f2d20a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 21:01:25 +0000 Subject: [PATCH 28/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_scanimagetiffimagingextractor.py | 22 ++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 46b2614a..7a912d5d 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -17,6 +17,7 @@ IS_ARM64_MAC = sys.platform == "darwin" and platform.processor() == "arm" pytestmark = pytest.mark.skipif(IS_ARM64_MAC) + @pytest.fixture(scope="module") def file_path(): return OPHYS_DATA_PATH / "imaging_datasets" / "ScanImage" / "scanimage_20220923_roi.tif" @@ -49,11 +50,13 @@ def expected_properties(): def scan_image_tiff_single_plane_imaging_extractor(request, file_path): return ScanImageTiffSinglePlaneImagingExtractor(file_path=file_path, **request.param) + @pytest.mark.parametrize("channel_name, plane_name", [("Invalid Channel", "0"), ("Channel 1", "Invalid Plane")]) def test_ScanImageTiffSinglePlaneImagingExtractor__init__invalid(file_path, channel_name, plane_name): with pytest.raises(ValueError): ScanImageTiffSinglePlaneImagingExtractor(file_path=file_path, channel_name=channel_name, plane_name=plane_name) + def test_ScanImageTiffSinglePlaneImagingExtractor__init__metadata_provided(file_path): metadata = extract_extra_metadata(file_path) parsed_metadata = parse_metadata(metadata) @@ -67,6 +70,7 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__metadata_provided(file_ assert extractor.metadata == metadata assert extractor.parsed_metadata == parsed_metadata + def test_ScanImageTiffSinglePlaneImagingExtractor__init__invalid_metadata_provided(file_path): metadata = {"invalid_key": "invalid_value"} parsed_metadata = {"invalid_key": "invalid_value"} @@ -79,6 +83,7 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__invalid_metadata_provid parsed_metadata=parsed_metadata, ) + def test_ScanImageTiffSinglePlaneImagingExtractor__init__parsed_metadata_not_provided(file_path): metadata = extract_extra_metadata(file_path) with pytest.raises(AssertionError): @@ -89,10 +94,11 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__parsed_metadata_not_pro metadata=metadata, ) + @pytest.mark.parametrize("frame_idxs", (0, [0, 1, 2], [0, 2, 5])) def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, expected_properties): from ScanImageTiffReader import ScanImageTiffReader - + frames = scan_image_tiff_single_plane_imaging_extractor.get_frames(frame_idxs=frame_idxs) file_path = str(scan_image_tiff_single_plane_imaging_extractor.file_path) plane = scan_image_tiff_single_plane_imaging_extractor.plane @@ -102,7 +108,7 @@ def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, frames_per_slice = expected_properties["frames_per_slice"] if isinstance(frame_idxs, int): frame_idxs = [frame_idxs] - + raw_idxs = [] for idx in frame_idxs: cycle = idx // frames_per_slice @@ -114,19 +120,21 @@ def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, + channel ) raw_idxs.append(raw_idx) - + with ScanImageTiffReader(file_path) as io: assert_array_equal(frames, io.data()[raw_idxs]) + @pytest.mark.parametrize("frame_idxs", ([-1], [50])) def test_get_frames_invalid(scan_image_tiff_single_plane_imaging_extractor, frame_idxs): with pytest.raises(ValueError): scan_image_tiff_single_plane_imaging_extractor.get_frames(frame_idxs=frame_idxs) + @pytest.mark.parametrize("frame_idx", (1, 3, 5)) def test_get_single_frame(scan_image_tiff_single_plane_imaging_extractor, expected_properties, frame_idx): from ScanImageTiffReader import ScanImageTiffReader - + frame = scan_image_tiff_single_plane_imaging_extractor._get_single_frame(frame=frame_idx) file_path = str(scan_image_tiff_single_plane_imaging_extractor.file_path) plane = scan_image_tiff_single_plane_imaging_extractor.plane @@ -160,7 +168,7 @@ def test_get_video( end_frame, ): from ScanImageTiffReader import ScanImageTiffReader - + video = scan_image_tiff_single_plane_imaging_extractor.get_video(start_frame=start_frame, end_frame=end_frame) if start_frame is None: start_frame = 0 @@ -172,7 +180,7 @@ def test_get_video( num_planes = expected_properties["num_planes"] num_channels = expected_properties["num_channels"] frames_per_slice = expected_properties["frames_per_slice"] - + raw_idxs = [] for idx in range(start_frame, end_frame): cycle = idx // frames_per_slice @@ -184,7 +192,7 @@ def test_get_video( + channel ) raw_idxs.append(raw_idx) - + with ScanImageTiffReader(file_path) as io: assert_array_equal(video, io.data()[raw_idxs]) From f90d4bfb199634f10c0162e3929625675c875882 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 1 May 2024 17:05:55 -0400 Subject: [PATCH 29/37] remove CI split --- .github/workflows/run-tests.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 92a8a4f9..80095510 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -74,16 +74,7 @@ jobs: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - name: Run full pytest (macos-latest) - if: ${{ matrix.os == 'macos-latest' }} - run: | - pytest -n auto --dist loadscope \ - --ignore=tests/test_scan_image_tiff.py \ - --ignore=tests/test_scanimage_utils.py \ - --ignore=tests/test_scanimagetiffimagingextractor.py \ - --ignore=tests/test_io.py - - name: Run full pytest with coverage (ubuntu-latest, windows-latest, macos-13) - if: ${{ matrix.os != 'macos-latest' }} + - name: Run full pytest with coverage run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml - if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'}} name: Upload full coverage to Codecov From e40d6c41e41e4a91f9b4f6cc0089c9df3c97a40c Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 1 May 2024 17:07:30 -0400 Subject: [PATCH 30/37] add reason --- tests/test_scanimagetiffimagingextractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 7a912d5d..59ba86a1 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -14,8 +14,8 @@ from .setup_paths import OPHYS_DATA_PATH -IS_ARM64_MAC = sys.platform == "darwin" and platform.processor() == "arm" -pytestmark = pytest.mark.skipif(IS_ARM64_MAC) +IS_ARM64_MAC = platform.processor() == "arm" and sys.platform == "darwin" +pytestmark = pytest.mark.skipif(IS_ARM64_MAC, reason="ScanImage does not yet support ARM64 Mac.") @pytest.fixture(scope="module") From ef563ad8ff42ee02615c2d1a2feb5d2ba82367fb Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:10:42 -0700 Subject: [PATCH 31/37] merge --- tests/test_scanimagetiffimagingextractor.py | 25 ++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 59ba86a1..55d9370c 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -1,6 +1,3 @@ -import sys -import platform - import pytest from numpy.testing import assert_array_equal @@ -10,12 +7,20 @@ ScanImageTiffSinglePlaneMultiFileImagingExtractor, ScanImageTiffMultiPlaneMultiFileImagingExtractor, ) -from roiextractors.extractors.tiffimagingextractors.scanimagetiff_utils import extract_extra_metadata, parse_metadata - +from roiextractors.extractors.tiffimagingextractors.scanimagetiff_utils import ( + extract_extra_metadata, + parse_metadata, + _get_scanimage_reader, +) from .setup_paths import OPHYS_DATA_PATH -IS_ARM64_MAC = platform.processor() == "arm" and sys.platform == "darwin" -pytestmark = pytest.mark.skipif(IS_ARM64_MAC, reason="ScanImage does not yet support ARM64 Mac.") +import platform + +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" +if ( + is_m_series_mac +): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 + pytest.skip("ScanImageTiffReader does not support M-series Macs", allow_module_level=True) @pytest.fixture(scope="module") @@ -97,7 +102,7 @@ def test_ScanImageTiffSinglePlaneImagingExtractor__init__parsed_metadata_not_pro @pytest.mark.parametrize("frame_idxs", (0, [0, 1, 2], [0, 2, 5])) def test_get_frames(scan_image_tiff_single_plane_imaging_extractor, frame_idxs, expected_properties): - from ScanImageTiffReader import ScanImageTiffReader + ScanImageTiffReader = _get_scanimage_reader() frames = scan_image_tiff_single_plane_imaging_extractor.get_frames(frame_idxs=frame_idxs) file_path = str(scan_image_tiff_single_plane_imaging_extractor.file_path) @@ -133,7 +138,7 @@ def test_get_frames_invalid(scan_image_tiff_single_plane_imaging_extractor, fram @pytest.mark.parametrize("frame_idx", (1, 3, 5)) def test_get_single_frame(scan_image_tiff_single_plane_imaging_extractor, expected_properties, frame_idx): - from ScanImageTiffReader import ScanImageTiffReader + ScanImageTiffReader = _get_scanimage_reader() frame = scan_image_tiff_single_plane_imaging_extractor._get_single_frame(frame=frame_idx) file_path = str(scan_image_tiff_single_plane_imaging_extractor.file_path) @@ -167,7 +172,7 @@ def test_get_video( start_frame, end_frame, ): - from ScanImageTiffReader import ScanImageTiffReader + ScanImageTiffReader = _get_scanimage_reader() video = scan_image_tiff_single_plane_imaging_extractor.get_video(start_frame=start_frame, end_frame=end_frame) if start_frame is None: From ed5452a6357771cfc7eea0a2c430a893de792660 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:02:49 -0700 Subject: [PATCH 32/37] moved skips to inside pytest files so that they get skipped when running tests locally --- tests/test_scan_image_tiff.py | 17 ++++++++++------- tests/test_scanimage_utils.py | 8 ++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/test_scan_image_tiff.py b/tests/test_scan_image_tiff.py index 88d58287..589f1811 100644 --- a/tests/test_scan_image_tiff.py +++ b/tests/test_scan_image_tiff.py @@ -2,19 +2,21 @@ from pathlib import Path from tempfile import mkdtemp from shutil import rmtree, copy - from hdmf.testing import TestCase from numpy.testing import assert_array_equal +import pytest + +from roiextractors import TiffImagingExtractor, ScanImageTiffImagingExtractor +from roiextractors.extractors.tiffimagingextractors.scanimagetiff_utils import _get_scanimage_reader +from .setup_paths import OPHYS_DATA_PATH + import platform +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" if ( - platform.processor() != "arm" + is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 - from ScanImageTiffReader import ScanImageTiffReader - -from roiextractors import TiffImagingExtractor, ScanImageTiffImagingExtractor - -from .setup_paths import OPHYS_DATA_PATH + pytest.skip("ScanImageTiffReader does not support M-series Macs", allow_module_level=True) class TestScanImageTiffExtractor(TestCase): @@ -23,6 +25,7 @@ def setUpClass(cls): cls.file_path = OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "sample_scanimage.tiff" cls.tmpdir = Path(mkdtemp()) cls.imaging_extractor = ScanImageTiffImagingExtractor(file_path=cls.file_path, sampling_frequency=30.0) + ScanImageTiffReader = _get_scanimage_reader() with ScanImageTiffReader(filename=str(cls.imaging_extractor.file_path)) as io: cls.data = io.data() diff --git a/tests/test_scanimage_utils.py b/tests/test_scanimage_utils.py index 619a2327..8429eba0 100644 --- a/tests/test_scanimage_utils.py +++ b/tests/test_scanimage_utils.py @@ -11,6 +11,14 @@ from .setup_paths import OPHYS_DATA_PATH +import platform + +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" +if ( + is_m_series_mac +): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 + pytest.skip("ScanImageTiffReader does not support M-series Macs", allow_module_level=True) + def test_get_scanimage_reader(): ScanImageTiffReader = _get_scanimage_reader() From 56e9f346604b48348c2600264c50867a4975b8bd Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:15:46 -0700 Subject: [PATCH 33/37] updated codecov to v4 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 80095510..7519be86 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -78,7 +78,7 @@ jobs: run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml - if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'}} name: Upload full coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./codecov.xml From a7c63278cf7688110724e352e528e8d4620fa38a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:21:28 -0700 Subject: [PATCH 34/37] lowercase darwin --- tests/test_scan_image_tiff.py | 2 +- tests/test_scanimage_utils.py | 2 +- tests/test_scanimagetiffimagingextractor.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_scan_image_tiff.py b/tests/test_scan_image_tiff.py index 589f1811..61979319 100644 --- a/tests/test_scan_image_tiff.py +++ b/tests/test_scan_image_tiff.py @@ -12,7 +12,7 @@ import platform -is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" +is_m_series_mac = platform.system() == "darwin" and platform.machine() == "arm64" if ( is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 diff --git a/tests/test_scanimage_utils.py b/tests/test_scanimage_utils.py index 8429eba0..ca9b8cfe 100644 --- a/tests/test_scanimage_utils.py +++ b/tests/test_scanimage_utils.py @@ -13,7 +13,7 @@ import platform -is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" +is_m_series_mac = platform.system() == "darwin" and platform.machine() == "arm64" if ( is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 55d9370c..16a80d92 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -16,7 +16,7 @@ import platform -is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" +is_m_series_mac = platform.system() == "darwin" and platform.machine() == "arm64" if ( is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 From d22967412b552b6c12b7fb10fa72b09a23e162c9 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:30:17 -0700 Subject: [PATCH 35/37] captialized Darwin --- tests/test_scan_image_tiff.py | 2 +- tests/test_scanimage_utils.py | 2 +- tests/test_scanimagetiffimagingextractor.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_scan_image_tiff.py b/tests/test_scan_image_tiff.py index 61979319..589f1811 100644 --- a/tests/test_scan_image_tiff.py +++ b/tests/test_scan_image_tiff.py @@ -12,7 +12,7 @@ import platform -is_m_series_mac = platform.system() == "darwin" and platform.machine() == "arm64" +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" if ( is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 diff --git a/tests/test_scanimage_utils.py b/tests/test_scanimage_utils.py index ca9b8cfe..8429eba0 100644 --- a/tests/test_scanimage_utils.py +++ b/tests/test_scanimage_utils.py @@ -13,7 +13,7 @@ import platform -is_m_series_mac = platform.system() == "darwin" and platform.machine() == "arm64" +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" if ( is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 diff --git a/tests/test_scanimagetiffimagingextractor.py b/tests/test_scanimagetiffimagingextractor.py index 16a80d92..55d9370c 100644 --- a/tests/test_scanimagetiffimagingextractor.py +++ b/tests/test_scanimagetiffimagingextractor.py @@ -16,7 +16,7 @@ import platform -is_m_series_mac = platform.system() == "darwin" and platform.machine() == "arm64" +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" if ( is_m_series_mac ): # Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 From 2602fe802fb70a65b59ac62d748aa495183b66dd Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:33:13 -0700 Subject: [PATCH 36/37] add skips to test_io --- tests/test_io.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_io.py b/tests/test_io.py index 95e9937f..2f9186fb 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -25,6 +25,11 @@ from .setup_paths import OPHYS_DATA_PATH, OUTPUT_PATH +import platform + +# Remove this check once scanimage tiff reader is available on ARM -- see https://gitlab.com/vidriotech/scanimagetiffreader-python/-/issues/31 +is_m_series_mac = platform.system() == "Darwin" and platform.machine() == "arm64" + def custom_name_func(testcase_func, param_num, param): return ( @@ -68,6 +73,9 @@ class TestExtractors(TestCase): @parameterized.expand(imaging_extractor_list, name_func=custom_name_func) def test_imaging_extractors(self, extractor_class, extractor_kwargs): + if extractor_class == ScanImageTiffImagingExtractor and is_m_series_mac: + return + extractor = extractor_class(**extractor_kwargs) check_imaging_return_types(extractor) @@ -88,6 +96,9 @@ def test_imaging_extractors(self, extractor_class, extractor_kwargs): @parameterized.expand(imaging_extractor_list, name_func=custom_name_func) def test_imaging_extractors_canonical_shape(self, extractor_class, extractor_kwargs): """Test that get_video and get_frame methods for their shapes and types under different indexing scenarios""" + if extractor_class == ScanImageTiffImagingExtractor and is_m_series_mac: + return + extractor = extractor_class(**extractor_kwargs) image_size = extractor.get_image_size() num_channels = extractor.get_num_channels() From f4d0e68a9f1c1fc6ad9f1a13279c94d5e8a2d9ab Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 1 May 2024 14:40:05 -0700 Subject: [PATCH 37/37] added concurrency to testing workflows --- .github/workflows/dev-testing.yaml | 4 ++++ .github/workflows/run-tests.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml index e7589117..fbe55627 100644 --- a/.github/workflows/dev-testing.yaml +++ b/.github/workflows/dev-testing.yaml @@ -3,6 +3,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: run: name: Ubuntu tests with Python ${{ matrix.python-version }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7519be86..ac903b3f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -6,6 +6,10 @@ on: workflows: [update-testing-data] types: [completed] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: on-failure: name: Notify on failure