From 2728da800812ae4115ceabab34e6e8a901227306 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 11 Jul 2023 12:47:38 +0200 Subject: [PATCH] fix: use new data loading apprach Proposed in nipreps/niworkflows#816. Resolves: #377. Co-authored-by: Chris Markiewicz --- docs/api.rst | 1 + docs/conf.py | 4 +- docs/requirements.txt | 5 +- sdcflows/__init__.py | 14 +- sdcflows/data/__init__.py | 204 +++++++++++++++++++++++ sdcflows/workflows/apply/registration.py | 6 +- sdcflows/workflows/fit/pepolar.py | 7 +- sdcflows/workflows/fit/syn.py | 15 +- 8 files changed, 234 insertions(+), 22 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index dbb481dc73..79b04032ea 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -7,6 +7,7 @@ Information on specific functions, classes, and methods. :glob: api/sdcflows.cli + api/sdcflows.data api/sdcflows.fieldmaps api/sdcflows.interfaces api/sdcflows.transform diff --git a/docs/conf.py b/docs/conf.py index c40e3b0c52..eb4f61580e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -83,7 +83,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -218,7 +218,7 @@ apidoc_module_dir = "../sdcflows" apidoc_output_dir = "api" -apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*", "data/*"] +apidoc_excluded_paths = ["conftest.py", "*/tests/*", "tests/*"] apidoc_separate_modules = True apidoc_extra_args = ["--module-first", "-d 1", "-T"] diff --git a/docs/requirements.txt b/docs/requirements.txt index 5ed611dfe8..bc1305a8e6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ attrs >= 20.1.0 -furo ~= 2021.10.09 +furo matplotlib >= 2.2.0 nibabel nipype >= 1.5.1 @@ -9,7 +9,6 @@ numpy packaging pydot >= 1.2.3 pydotplus -sphinx ~= 4.2 +sphinx sphinxcontrib-apidoc -sphinxcontrib-napoleon templateflow diff --git a/sdcflows/__init__.py b/sdcflows/__init__.py index fc4b27a728..40b11e5720 100644 --- a/sdcflows/__init__.py +++ b/sdcflows/__init__.py @@ -1,6 +1,9 @@ """SDCflows - :abbr:`SDC (susceptibility distortion correction)` by DUMMIES, for dummies.""" +from sdcflows.data import Loader + __packagename__ = "sdcflows" -__copyright__ = "2022, The NiPreps developers" +__copyright__ = "2023, The NiPreps developers" + try: from ._version import __version__ except ModuleNotFoundError: @@ -12,3 +15,12 @@ __version__ = "unknown" del get_distribution del DistributionNotFound + +__all__ = ( + "__version__", + "__packagename__", + "__copyright__", + "load_resource", +) + +load_resource = Loader(__package__) diff --git a/sdcflows/data/__init__.py b/sdcflows/data/__init__.py index e69de29bb2..f4d711c8de 100644 --- a/sdcflows/data/__init__.py +++ b/sdcflows/data/__init__.py @@ -0,0 +1,204 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +# +# Copyright 2023 The NiPreps Developers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# We support and encourage derived works from this project, please read +# about our expectations at +# +# https://www.nipreps.org/community/licensing/ +# +"""SDCFlows data files + +.. autofunction:: load + +.. automethod:: load.readable + +.. automethod:: load.as_path + +.. automethod:: load.cached + +.. autoclass:: Loader +""" +from __future__ import annotations + +import atexit +import os +from contextlib import AbstractContextManager, ExitStack +from functools import cached_property +from pathlib import Path +from types import ModuleType +from typing import Union + +try: + from functools import cache +except ImportError: # PY38 + from functools import lru_cache as cache + +try: # Prefer backport to leave consistency to dependency spec + from importlib_resources import as_file, files +except ImportError: + from importlib.resources import as_file, files # type: ignore + +try: # Prefer stdlib so Sphinx can link to authoritative documentation + from importlib.resources.abc import Traversable +except ImportError: + from importlib_resources.abc import Traversable + +__all__ = ["load"] + + +class Loader: + """A loader for package files relative to a module + + This class wraps :mod:`importlib.resources` to provide a getter + function with an interpreter-lifetime scope. For typical packages + it simply passes through filesystem paths as :class:`~pathlib.Path` + objects. For zipped distributions, it will unpack the files into + a temporary directory that is cleaned up on interpreter exit. + + This loader accepts a fully-qualified module name or a module + object. + + Expected usage:: + + '''Data package + + .. autofunction:: load_data + + .. automethod:: load_data.readable + + .. automethod:: load_data.as_path + + .. automethod:: load_data.cached + ''' + + from sdcflows.data import Loader + + load_data = Loader(__package__) + + :class:`~Loader` objects implement the :func:`callable` interface + and generate a docstring, and are intended to be treated and documented + as functions. + + For greater flexibility and improved readability over the ``importlib.resources`` + interface, explicit methods are provided to access resources. + + +---------------+----------------+------------------+ + | On-filesystem | Lifetime | Method | + +---------------+----------------+------------------+ + | `True` | Interpreter | :meth:`cached` | + +---------------+----------------+------------------+ + | `True` | `with` context | :meth:`as_path` | + +---------------+----------------+------------------+ + | `False` | n/a | :meth:`readable` | + +---------------+----------------+------------------+ + + It is also possible to use ``Loader`` directly:: + + from sdcflows.data import Loader + + Loader(other_package).readable('data/resource.ext').read_text() + + with Loader(other_package).as_path('data') as pkgdata: + # Call function that requires full Path implementation + func(pkgdata) + + # contrast to + + from importlib_resources import files, as_file + + files(other_package).joinpath('data/resource.ext').read_text() + + with as_file(files(other_package) / 'data') as pkgdata: + func(pkgdata) + + .. automethod:: readable + + .. automethod:: as_path + + .. automethod:: cached + """ + + def __init__(self, anchor: Union[str, ModuleType]): + self._anchor = anchor + self.files = files(anchor) + self.exit_stack = ExitStack() + atexit.register(self.exit_stack.close) + # Allow class to have a different docstring from instances + self.__doc__ = self._doc + + @cached_property + def _doc(self): + """Construct docstring for instances + + Lists the public top-level paths inside the location, where + non-public means has a `.` or `_` prefix or is a 'tests' + directory. + """ + top_level = sorted( + os.path.relpath(p, self.files) + "/"[: p.is_dir()] + for p in self.files.iterdir() + if p.name[0] not in (".", "_") and p.name != "tests" + ) + doclines = [ + f"Load package files relative to ``{self._anchor}``.", + "", + "This package contains the following (top-level) files/directories:", + "", + *(f"* ``{path}``" for path in top_level), + ] + + return "\n".join(doclines) + + def readable(self, *segments) -> Traversable: + """Provide read access to a resource through a Path-like interface. + + This file may or may not exist on the filesystem, and may be + efficiently used for read operations, including directory traversal. + + This result is not cached or copied to the filesystem in cases where + that would be necessary. + """ + return self.files.joinpath(*segments) + + def as_path(self, *segments) -> AbstractContextManager[Path]: + """Ensure data is available as a :class:`~pathlib.Path`. + + This method generates a context manager that yields a Path when + entered. + + This result is not cached, and any temporary files that are created + are deleted when the context is exited. + """ + return as_file(self.files.joinpath(*segments)) + + @cache + def cached(self, *segments) -> Path: + """Ensure data is available as a :class:`~pathlib.Path`. + + Any temporary files that are created remain available throughout + the duration of the program, and are deleted when Python exits. + + Results are cached so that multiple calls do not unpack the same + data multiple times, but the cache is sensitive to the specific + argument(s) passed. + """ + return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments))) + + __call__ = cached + + +load = Loader(__package__) diff --git a/sdcflows/workflows/apply/registration.py b/sdcflows/workflows/apply/registration.py index 0e90efdf84..1ef0fdd9b1 100644 --- a/sdcflows/workflows/apply/registration.py +++ b/sdcflows/workflows/apply/registration.py @@ -29,10 +29,10 @@ The target EPI is the distorted dataset (or a reference thereof). """ -from pkg_resources import resource_filename as pkgrf from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from niworkflows.engine.workflows import LiterateWorkflow as Workflow +from sdcflows.data import load as load_data def init_coeff2epi_wf( @@ -111,9 +111,7 @@ def init_coeff2epi_wf( # Register the reference of the fieldmap to the reference # of the target image (the one that shall be corrected) - ants_settings = pkgrf( - "sdcflows", f"data/fmap-any_registration{'_testing' * sloppy}.json" - ) + ants_settings = load_data(f"fmap-any_registration{'_testing' * sloppy}.json") coregister = pe.Node( Registration( diff --git a/sdcflows/workflows/fit/pepolar.py b/sdcflows/workflows/fit/pepolar.py index a50b9d1f51..1ea07a156a 100644 --- a/sdcflows/workflows/fit/pepolar.py +++ b/sdcflows/workflows/fit/pepolar.py @@ -21,11 +21,11 @@ # https://www.nipreps.org/community/licensing/ # """Datasets with multiple phase encoded directions.""" -from pkg_resources import resource_filename as _pkg_fname from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from niworkflows.engine.workflows import LiterateWorkflow as Workflow +from sdcflows.data import load as load_data INPUT_FIELDS = ("metadata", "in_data") _PEPOLAR_DESC = """\ @@ -146,9 +146,10 @@ def init_topup_wf( # The core of the implementation # Feed the input images in LAS orientation, so FSL does not run funky reorientations to_las = pe.Node(ReorientImageAndMetadata(target_orientation="LAS"), name="to_las") + topup = pe.Node( TOPUP( - config=_pkg_fname("sdcflows", f"data/flirtsch/b02b0{'_quick' * sloppy}.cnf") + config=load_data(f"flirtsch/b02b0{'_quick' * sloppy}.cnf") ), name="topup", ) @@ -332,7 +333,7 @@ def init_3dQwarp_wf(omp_nthreads=1, debug=False, name="pepolar_estimate_wf"): align_pes = pe.Node( Registration( - from_file=_pkg_fname("sdcflows", "data/translation_rigid.json"), + from_file=load_data("translation_rigid.json"), output_warped_image=True, ), name="align_pes", diff --git a/sdcflows/workflows/fit/syn.py b/sdcflows/workflows/fit/syn.py index b463bcc229..9183facdeb 100644 --- a/sdcflows/workflows/fit/syn.py +++ b/sdcflows/workflows/fit/syn.py @@ -34,10 +34,11 @@ """ -from pkg_resources import resource_filename from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu from niworkflows.engine.workflows import LiterateWorkflow as Workflow +from sdcflows.data import load as load_data + DEFAULT_MEMORY_MIN_GB = 0.01 INPUT_FIELDS = ( @@ -120,7 +121,6 @@ def init_syn_sdc_wf( Short description of the estimation method that was run. """ - from pkg_resources import resource_filename as pkgrf from packaging.version import parse as parseversion, Version from nipype.interfaces.ants import ImageMath from niworkflows.interfaces.fixes import ( @@ -234,7 +234,7 @@ def init_syn_sdc_wf( # SyN Registration Core syn = pe.Node( Registration( - from_file=pkgrf("sdcflows", f"data/sd_syn{'_sloppy' * sloppy}.json") + from_file=load_data(f"sd_syn{'_sloppy' * sloppy}.json") ), name="syn", n_procs=omp_nthreads, @@ -401,7 +401,6 @@ def init_syn_preprocessing_wf( the cost function of SyN. """ - from pkg_resources import resource_filename as pkgrf from niworkflows.interfaces.nibabel import ( IntensityClip, ApplyMask, @@ -448,13 +447,11 @@ def init_syn_preprocessing_wf( mem_gb=DEFAULT_MEMORY_MIN_GB, run_without_submitting=True, ) - transform_list.inputs.in3 = pkgrf( - "sdcflows", "data/fmap_atlas_2_MNI152NLin2009cAsym_affine.mat" - ) + transform_list.inputs.in3 = load_data("fmap_atlas_2_MNI152NLin2009cAsym_affine.mat") prior2epi = pe.Node( ApplyTransforms( invert_transform_flags=[True, False, False], - input_image=pkgrf("sdcflows", "data/fmap_atlas.nii.gz"), + input_image=load_data("fmap_atlas.nii.gz"), ), name="prior2epi", n_procs=omp_nthreads, @@ -496,7 +493,7 @@ def init_syn_preprocessing_wf( ) epi2anat = pe.Node( - Registration(from_file=resource_filename("sdcflows", "data/affine.json")), + Registration(from_file=load_data("affine.json")), name="epi2anat", n_procs=omp_nthreads, )