diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 2686d394..c13decad 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -45,21 +45,20 @@ jobs: # Indicates the location of the vcpkg as a Git submodule of the project repository. VCPKG_ROOT: ${{ github.workspace }}/external/LibAPR/vcpkg CIBW_ENVIRONMENT_WINDOWS: EXTRA_CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=D:\\a\\pyapr\\pyapr\\external\\LibAPR\\vcpkg\\scripts\\buildsystems\\vcpkg.cmake -DVCPKG_MANIFEST_DIR=D:\\a\\pyapr\\pyapr\\external\\LibAPR\\" - CIBW_BUILD: "cp36-* cp37-* cp38-* cp39-* cp310-*" + CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-*" CIBW_SKIP: "*musllinux*" CIBW_ARCHS: "auto64" - CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_24" CIBW_BUILD_VERBOSITY: 1 - CIBW_REPAIR_WHEEL_COMMAND_MACOS: "pip uninstall -y delocate && pip install git+https://github.com/Chia-Network/delocate.git && delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} -v {wheel}" + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "pip install -U delocate && delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} -v {wheel}" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "pip install -U wheel delvewheel && python fix_windows_wheel.py {wheel} {dest_dir}" CIBW_TEST_REQUIRES: "pytest pytest-qt pytest-xvfb" CIBW_TEST_COMMAND: "python3 -m pytest -vv {project}/pyapr/tests" CIBW_TEST_SKIP: "*-win_amd64" # windows tests are run separately - CIBW_BEFORE_BUILD_LINUX: "apt update && apt install -y libtiff5-dev libhdf5-dev" + CIBW_BEFORE_BUILD_LINUX: "yum makecache && yum install -y libtiff-devel hdf5-devel" CIBW_ENVIRONMENT_MACOS: CPPFLAGS="-I/usr/local/opt/llvm/include" LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib" CXX="/usr/local/opt/llvm/bin/clang++" CC="/usr/local/opt/llvm/bin/clang" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: true @@ -109,7 +108,7 @@ jobs: - name: Install cibuildwheel run: | - python3 -m pip install cibuildwheel==2.5.0 + python3 -m pip install cibuildwheel - name: Install OpenMP dependencies with brew for OSX if: contains(matrix.os,'macos') @@ -139,11 +138,11 @@ jobs: fail-fast: false matrix: os: [ windows-latest ] - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: false diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index b95df0f2..dc8b2b6a 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -32,7 +32,7 @@ jobs: VCPKG_ROOT: ${{ github.workspace }}/external/LibAPR/vcpkg EXTRA_CMAKE_ARGS: "-DCMAKE_TOOLCHAIN_FILE='${{ github.workspace }}/external/LibAPR/vcpkg/scripts/buildsystems/vcpkg.cmake'" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: true @@ -75,7 +75,7 @@ jobs: - name: Check file existence id: check_files - uses: andstor/file-existence-action@v1 + uses: andstor/file-existence-action@v2 with: files: "${{ env.VCPKG_ROOT }}/vcpkg" diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index e5e1ed02..602dc2d4 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -37,7 +37,7 @@ jobs: VCPKG_ROOT: ${{ github.workspace }}/external/LibAPR/vcpkg EXTRA_CMAKE_ARGS: "-DCMAKE_TOOLCHAIN_FILE='${{ github.workspace }}/external/LibAPR/vcpkg/scripts/buildsystems/vcpkg.cmake'" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: true @@ -80,7 +80,7 @@ jobs: - name: Check file existence id: check_files - uses: andstor/file-existence-action@v1 + uses: andstor/file-existence-action@v2 with: files: "${{ env.VCPKG_ROOT }}/vcpkg" diff --git a/CMakeLists.txt b/CMakeLists.txt index a15afc37..d73f9e2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,25 +101,25 @@ if(WIN32) message(STATUS "Compiling on windows with CLANG!") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fcxx-exceptions") set(CMAKE_CXX_FLAGS_DEBUG "/MD /Z7") - set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 -Xclang -O3 /nologo /fp:fast") #-flto=thin -march=native /O2 /Ob2 + set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 -Xclang -O3 /nologo") #-flto=thin -march=native /O2 /Ob2 endif() if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") message(STATUS "Compiling on windows with MSVC!") - set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 /O2 /Ob2 /nologo /fp:fast") + set(CMAKE_CXX_FLAGS_RELEASE "/MD /EHsc /std:c++17 /arch:AVX2 /O2 /Ob2 /nologo") set(CMAKE_CXX_FLAGS_DEBUG "/MD /Z7") endif() else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -Wall -pedantic ") if(CMAKE_COMPILER_IS_GNUCC) - set(CMAKE_CXX_FLAGS_RELEASE "-O4 -ffast-math") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Bdynamic") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ldl -lz") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(CMAKE_CXX_FLAGS_RELEASE "-O3 -ffast-math") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lz") endif() diff --git a/README.md b/README.md index 60540550..2a040888 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,23 @@ [![License](https://img.shields.io/pypi/l/pyapr.svg?color=green)](https://raw.githubusercontent.com/AdaptiveParticles/pyapr/master/LICENSE) [![Python Version](https://img.shields.io/pypi/pyversions/pyapr.svg?color=blue)]((https://python.org)) [![PyPI](https://img.shields.io/pypi/v/pyapr.svg?color=green)](https://pypi.org/project/pyapr/) -![PowerShell Gallery](https://img.shields.io/powershellgallery/p/DNS.1.1.1.1) -[![DOI](https://zenodo.org/badge/184399854.svg)](https://zenodo.org/badge/latestdoi/184399854) +[![Downloads](https://static.pepy.tech/badge/pyapr)](https://pepy.tech/project/pyapr) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7304045.svg)](https://doi.org/10.5281/zenodo.7304045) Documentation can be found [here](https://adaptiveparticles.github.io/pyapr/index.html). -Content-adaptive storage and processing of large volumetric microscopy data using -the Adaptive Particle Representation (APR). +Content-adaptive storage and processing of large volumetric microscopy data using the Adaptive Particle Representation (APR). + +The APR is an adaptive image representation designed primarily for large 3D fluorescence microscopy datasets. By replacing pixels with particles positioned according to the image content, it enables orders-of-magnitude compression of sparse image data while maintaining image quality. However, unlike most compression formats, the APR can be used directly in a wide range of processing tasks - even on the GPU! + +| Pixels | APR | +| :--: | :--: | +| ![img.png](./docs/images/pix_joined.png) | ![img.png](./docs/images/apr_joined.png) | +| Uniform sampling | Adaptive sampling | + +*[image source](https://bbbc.broadinstitute.org/bbbc/BBBC032), +[illustration source](https://ieeexplore.ieee.org/abstract/document/9796006)* -The APR is an adaptive image representation designed primarily for large 3D fluorescence -microscopy datasets. By replacing pixels with particles positioned according to the -image content, it enables orders-of-magnitude compression of sparse image data -while maintaining image quality. However, unlike most compression formats, the APR -can be used directly in a wide range of processing tasks - even on the GPU! For more detailed information about the APR and its use, see: - [Adaptive particle representation of fluorescence microscopy images](https://www.nature.com/articles/s41467-018-07390-9) (nature communications) @@ -25,29 +29,65 @@ For more detailed information about the APR and its use, see: **pyapr** is built on top of the C++ library [LibAPR] using [pybind11]. -## Installation -For Windows 10, OSX, and Linux and Python versions 3.7-3.9 direct installation with OpenMP support should work via [pip]: +## Quick start guide + +Convert images to APR using minimal amounts of code (*see [get_apr_demo](demo/get_apr_demo.py) and [get_apr_interactive_demo](demo/get_apr_interactive_demo.py) for additional options*). + +```python +import pyapr +from skimage import io + +# read image into numpy array +img = io.imread('my_image.tif') + +# convert to APR using default settings +apr, parts = pyapr.converter.get_apr(img) + +# write APR to file +pyapr.io.write('my_image.apr', apr, parts) ``` -pip install pyapr + +![img.png](./docs/images/apr_file.png) + +To return to the pixel representation: +```python +# reconstruct pixel image +img = pyapr.reconstruction.reconstruct_constant(apr, parts) ``` -Note: Due to the use of OpenMP, it is encouraged to install as part of a virtualenv. -See [INSTALL] for manual build instructions. -## Exclusive features +Inspect APRs using our makeshift image viewers (*see [napari-apr-viewer] for less experimental visualization options*). + +```python +# read APR from file +apr, parts = pyapr.io.read('my_image.apr') -In addition to providing wrappers for most of the functionality of LibAPR, we provide a number of -new features that simplify the generation and handling of the APR. For example: +# launch viewer +pyapr.viewer.parts_viewer(apr, parts) +``` +![img.png](./docs/images/view_apr.png) -* Interactive APR conversion (see [get_apr_interactive_demo](demo/get_apr_interactive_demo.py) and - [get_apr_by_block_interactive_demo](demo/get_apr_by_block_interactive_demo.py)) -* Interactive APR z-slice viewer (see [viewer_demo](demo/viewer_demo.py)) -* Interactive APR raycast (maximum intensity projection) viewer (see [raycast_demo](demo/raycast_demo.py)) -* Interactive lossy compression of particle intensities (see [compress_particles_demo](demo/compress_particles_demo.py)) +The `View Level` toggle allows you to see the adaptation (brighter = higher resolution). -For further examples see the [demo scripts]. +![img.png](./docs/images/view_level.png) -Also be sure to check out our (experimental) [napari] plugin: [napari-apr-viewer]. +Or view the result in 3D using APR-native maximum intensity projection raycast (cpu). +```python +# launch raycast viewer +pyapr.viewer.raycast_viewer(apr, parts) +``` +![img.png](./docs/images/raycast.png) + +See the [demo scripts] for more examples. + +## Installation +For Windows 10, OSX, and Linux direct installation with OpenMP support should work via [pip]: +``` +pip install pyapr +``` +Note: Due to the use of OpenMP, it is encouraged to install as part of a virtualenv. + +See [INSTALL] for manual build instructions. ## License diff --git a/docs/images/apr_file.png b/docs/images/apr_file.png new file mode 100644 index 00000000..4c245ffb Binary files /dev/null and b/docs/images/apr_file.png differ diff --git a/docs/images/apr_joined.png b/docs/images/apr_joined.png new file mode 100644 index 00000000..e81ae985 Binary files /dev/null and b/docs/images/apr_joined.png differ diff --git a/docs/images/pix_joined.png b/docs/images/pix_joined.png new file mode 100644 index 00000000..8b6af883 Binary files /dev/null and b/docs/images/pix_joined.png differ diff --git a/docs/images/raycast.png b/docs/images/raycast.png new file mode 100644 index 00000000..4de92bff Binary files /dev/null and b/docs/images/raycast.png differ diff --git a/docs/images/view_apr.png b/docs/images/view_apr.png new file mode 100644 index 00000000..dbf46f2e Binary files /dev/null and b/docs/images/view_apr.png differ diff --git a/docs/images/view_level.png b/docs/images/view_level.png new file mode 100644 index 00000000..a76cc322 Binary files /dev/null and b/docs/images/view_level.png differ diff --git a/external/LibAPR b/external/LibAPR index 9e55e7e5..2e95a626 160000 --- a/external/LibAPR +++ b/external/LibAPR @@ -1 +1 @@ -Subproject commit 9e55e7e5be7dbbd21e00dfce41df64fb7542d092 +Subproject commit 2e95a6264a4ed8e86d2140e0e34a10c01419e657 diff --git a/pyapr/data_containers/__init__.py b/pyapr/data_containers/__init__.py index 1efc60a0..c7e07922 100644 --- a/pyapr/data_containers/__init__.py +++ b/pyapr/data_containers/__init__.py @@ -1,5 +1,5 @@ from _pyaprwrapper.data_containers import APR, APRParameters, FloatParticles, ShortParticles, LongParticles, \ - ByteParticles, ReconPatch, PixelDataByte, PixelDataShort, PixelDataFloat, APRPtrVector, LazyAccess, \ + ByteParticles, IntParticles, ReconPatch, PixelDataByte, PixelDataShort, PixelDataFloat, APRPtrVector, LazyAccess, \ LazyDataByte, LazyDataShort, LazyDataLong, LazyDataFloat, LazyIterator, LinearIterator __all__ = [ @@ -7,6 +7,7 @@ 'APRParameters', 'ByteParticles', 'ShortParticles', + 'IntParticles', 'FloatParticles', 'LongParticles', 'ReconPatch', diff --git a/pyapr/data_containers/src/BindParticleData.hpp b/pyapr/data_containers/src/BindParticleData.hpp index a2be26aa..cfe845a3 100644 --- a/pyapr/data_containers/src/BindParticleData.hpp +++ b/pyapr/data_containers/src/BindParticleData.hpp @@ -19,6 +19,7 @@ struct TypeParseTraits{ REGISTER_PARSE_TYPE(uint8); REGISTER_PARSE_TYPE(uint16); +REGISTER_PARSE_TYPE(int32); REGISTER_PARSE_TYPE(uint64); REGISTER_PARSE_TYPE(float); diff --git a/pyapr/reconstruction/APRSlicer.py b/pyapr/reconstruction/APRSlicer.py index 05bd2fe9..79d6d1fd 100644 --- a/pyapr/reconstruction/APRSlicer.py +++ b/pyapr/reconstruction/APRSlicer.py @@ -1,6 +1,6 @@ -from _pyaprwrapper.data_containers import APR, ReconPatch, ByteParticles, ShortParticles, FloatParticles, LongParticles +from _pyaprwrapper.data_containers import APR, ReconPatch, ByteParticles, ShortParticles, FloatParticles, LongParticles, IntParticles from _pyaprwrapper.tree import fill_tree_mean, fill_tree_max -from ..utils import particles_to_type +from ..utils import particles_to_type, type_to_particles from .._common import _check_input from . import reconstruct_constant, reconstruct_level, reconstruct_smooth import numpy as np @@ -23,7 +23,7 @@ class APRSlicer: """ def __init__(self, apr: APR, - parts: Union[ByteParticles, ShortParticles, LongParticles, FloatParticles], + parts: Union[ByteParticles, ShortParticles, LongParticles, FloatParticles, IntParticles], mode: str = 'constant', level_delta: int = 0, tree_mode: str = 'mean'): @@ -66,6 +66,14 @@ def shape(self): @property def ndim(self): return 3 + + def __array__(self): + # allows things like np.max(APRSlicer) + return np.array(self.parts) + + def astype(self, typespec): + parts = type(type_to_particles(typespec))(np.array(self.parts).astype(typespec)) + return APRSlicer(self.apr, parts, mode=self.mode, level_delta=self.patch.level_delta, tree_mode='max' if typespec in [int, np.int32, 'int', 'int32'] else 'mean') def new_empty_slice(self): return np.zeros((self.patch.z_end-self.patch.z_begin, diff --git a/pyapr/reconstruction/src/BindReconstruction.hpp b/pyapr/reconstruction/src/BindReconstruction.hpp index 6de7e7a5..7e18d9a1 100644 --- a/pyapr/reconstruction/src/BindReconstruction.hpp +++ b/pyapr/reconstruction/src/BindReconstruction.hpp @@ -275,11 +275,13 @@ void AddReconstruction(py::module &m) { bindReconstruct(m); bindReconstruct(m); bindReconstruct(m); + bindReconstruct(m); bindReconstructPatch(m); bindReconstructPatch(m); bindReconstructPatch(m); bindReconstructPatch(m); + bindReconstructPatch(m); bindReconstructLazy(m); bindReconstructLazy(m); diff --git a/pyapr/tests/test_reconstruction.py b/pyapr/tests/test_reconstruction.py index b07d08e0..d321d922 100644 --- a/pyapr/tests/test_reconstruction.py +++ b/pyapr/tests/test_reconstruction.py @@ -41,3 +41,14 @@ def test_reconstruction(mode, ndim): assert np.allclose(slicer[0], lazy_slicer[0]) assert np.allclose(slicer[0, float(0), :], lazy_slicer[0, float(0), :]) + + assert np.max(slicer) == parts.max() == np.max(np.array(parts)) + + slicer = slicer.astype(np.float32) + assert isinstance(slicer.parts, pyapr.FloatParticles) + + slicer = slicer.astype(int) + assert isinstance(slicer.parts, pyapr.IntParticles) + + with pytest.raises(ValueError): + slicer = slicer.astype(np.int8) \ No newline at end of file diff --git a/pyapr/tests/test_utils.py b/pyapr/tests/test_utils.py index 064f25bb..09c3fc9b 100644 --- a/pyapr/tests/test_utils.py +++ b/pyapr/tests/test_utils.py @@ -7,6 +7,7 @@ def test_utils(): assert isinstance(pyapr.utils.type_to_particles(np.uint8), pyapr.ByteParticles) assert isinstance(pyapr.utils.type_to_particles(np.uint16), pyapr.ShortParticles) + assert isinstance(pyapr.utils.type_to_particles(np.int32), pyapr.IntParticles) assert isinstance(pyapr.utils.type_to_particles(np.uint64), pyapr.LongParticles) assert isinstance(pyapr.utils.type_to_particles(np.float32), pyapr.FloatParticles) @@ -17,6 +18,7 @@ def test_utils(): assert pyapr.utils.particles_to_type(pyapr.ByteParticles()) is np.uint8 assert pyapr.utils.particles_to_type(pyapr.ShortParticles()) is np.uint16 + assert pyapr.utils.particles_to_type(pyapr.IntParticles()) is np.int32 assert pyapr.utils.particles_to_type(pyapr.LongParticles()) is np.uint64 assert pyapr.utils.particles_to_type(pyapr.FloatParticles()) is np.float32 @@ -29,7 +31,7 @@ def test_utils(): pyapr.utils.particles_to_type(np.zeros(5, dtype=np.uint16)) with pytest.raises(ValueError): - pyapr.utils.type_to_particles(np.int32) + pyapr.utils.type_to_particles(np.int64) with pytest.raises(ValueError): pyapr.utils.type_to_lazy_particles(np.float64) diff --git a/pyapr/tree/src/BindFillTree.hpp b/pyapr/tree/src/BindFillTree.hpp index 32189077..18ecc6e8 100644 --- a/pyapr/tree/src/BindFillTree.hpp +++ b/pyapr/tree/src/BindFillTree.hpp @@ -103,16 +103,19 @@ void AddFillTree(py::module &m) { bindFillTreeMean(m); bindFillTreeMean(m); bindFillTreeMean(m); + bindFillTreeMean(m); bindFillTreeMin(m); bindFillTreeMin(m); bindFillTreeMin(m); bindFillTreeMin(m); + bindFillTreeMin(m); bindFillTreeMax(m); bindFillTreeMax(m); bindFillTreeMax(m); bindFillTreeMax(m); + bindFillTreeMax(m); bindSampleFromTree(m); bindSampleFromTree(m); diff --git a/pyapr/utils/types.py b/pyapr/utils/types.py index 0f9b8829..fb7986b3 100644 --- a/pyapr/utils/types.py +++ b/pyapr/utils/types.py @@ -1,4 +1,4 @@ -from _pyaprwrapper.data_containers import ByteParticles, ShortParticles, FloatParticles, LongParticles, \ +from _pyaprwrapper.data_containers import ByteParticles, ShortParticles, FloatParticles, LongParticles, IntParticles, \ LazyDataByte, LazyDataShort, LazyDataFloat, LazyDataLong import numpy as np from typing import Union @@ -32,8 +32,10 @@ def type_to_particles(typespec: Union[str, type]) -> ParticleData: return ByteParticles() if typespec in ('uint64', np.uint64): return LongParticles() + if typespec in (int, 'int', 'int32', np.int32): + return IntParticles() raise ValueError(f'Type {typespec} is currently not supported. Valid types are \'uint8\', \'uint16\', ' - f'\'uint64\' and \'float\'') + f'\'uint64\', \'int\' and \'float\'') def type_to_lazy_particles(typespec: Union[str, type]) -> LazyData: @@ -87,4 +89,6 @@ def particles_to_type(parts: Union[ParticleData, LazyData]) -> type: return np.uint8 if isinstance(parts, (LongParticles, LazyDataLong)): return np.uint64 + if isinstance(parts, IntParticles): + return np.int32 raise TypeError(f'Input must be of type {ParticleData} or {LazyData} ({type(parts)} was provided)') diff --git a/setup.cfg b/setup.cfg index 26086e61..7a287fb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,11 +12,12 @@ classifiers = Intended Audience :: Developers License :: OSI Approved :: Apache Software License Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Topic :: Scientific/Engineering + Topic :: Scientific/Engineering :: Image Processing [options] zip_safe = False diff --git a/wrappers/pythonBind.cpp b/wrappers/pythonBind.cpp index 205d6cef..e3dd8612 100644 --- a/wrappers/pythonBind.cpp +++ b/wrappers/pythonBind.cpp @@ -70,6 +70,7 @@ PYBIND11_MODULE(APR_PYTHON_MODULE_NAME, m) { AddPyParticleData(data_containers, "Float"); AddPyParticleData(data_containers, "Short"); AddPyParticleData(data_containers, "Long"); + AddPyParticleData(data_containers, "Int"); // wrap lazy classes AddLazyAccess(data_containers, "LazyAccess");