From e2f96ae32bde756b327c2a3e41efb368f065c979 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Fri, 28 Jun 2024 15:48:41 +0200 Subject: [PATCH 1/3] structure to distribute keras models --- src/freesas/resources/__init__.py | 149 ++++++++++++++++++ .../resources/keras_models/Rg+Dmax.keras | 0 .../resources/keras_models/meson.build | 7 + src/freesas/resources/meson.build | 9 ++ 4 files changed, 165 insertions(+) create mode 100644 src/freesas/resources/__init__.py create mode 100644 src/freesas/resources/keras_models/Rg+Dmax.keras create mode 100644 src/freesas/resources/keras_models/meson.build create mode 100644 src/freesas/resources/meson.build diff --git a/src/freesas/resources/__init__.py b/src/freesas/resources/__init__.py new file mode 100644 index 0000000..35afbad --- /dev/null +++ b/src/freesas/resources/__init__.py @@ -0,0 +1,149 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (C) 2016-2024 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Access project's data and documentation files. + +All access to data and documentation files MUST be made through the functions +of this modules to ensure access across different distribution schemes: + +- Installing from source or from wheel +- Installing package as a zip (through the use of pkg_resources) +- Linux packaging willing to install data files (and doc files) in + alternative folders. In this case, this file must be patched. +- Frozen fat binary application using pyFAI (frozen with cx_Freeze or py2app). + This needs special care for the resource files in the setup: + +- With cx_Freeze, add pyFAI/resources to include_files: + +.. code-block:: python + + import pyFAI.resources + pyFAI_include_files = (os.path.dirname(pyFAI.resources.__file__), + os.path.join('pyFAI', 'resources')) + setup(..., + options={'build_exe': {'include_files': [pyFAI_include_files]}} + ) + +- With py2app, add pyFAI in the packages list of the py2app options: + +.. code-block:: python + + setup(..., + options={'py2app': {'packages': ['pyFAI']}} + ) +""" + +__authors__ = ["V.A. Sole", "Thomas Vincent"] +__license__ = "MIT" +__date__ = "07/03/2024" + + +import os +import sys +import logging + +logger = logging.getLogger(__name__) + +# importlib_resources is useful when this package is stored in a zip +# When importlib.resources is not available, the resources dir defaults to the +# directory containing this module. +if sys.version_info >= (3,9): + import importlib.resources as importlib_resources +else: + try: + import importlib_resources + except ImportError: + logger.info("Unable to import importlib_resources") + logger.debug("Backtrace", exc_info=True) + importlib_resources = None + +if importlib_resources is not None: + import atexit + from contextlib import ExitStack + file_manager = ExitStack() + atexit.register(file_manager.close) + + +# For packaging purpose, patch this variable to use an alternative directory +# E.g., replace with _RESOURCES_DIR = '/usr/share/pyFAI/data' +_RESOURCES_DIR = None + +# For packaging purpose, patch this variable to use an alternative directory +# E.g., replace with _RESOURCES_DIR = '/usr/share/pyFAI/doc' +# Not in use, uncomment when functionnality is needed +# _RESOURCES_DOC_DIR = None + +# cx_Freeze forzen support +# See http://cx-freeze.readthedocs.io/en/latest/faq.html#using-data-files +if getattr(sys, 'frozen', False): + # Running in a frozen application: + # We expect resources to be located either in a pyFAI/resources/ dir + # relative to the executable or within this package. + _dir = os.path.join(os.path.dirname(sys.executable), 'pyFAI', 'resources') + if os.path.isdir(_dir): + _RESOURCES_DIR = _dir + + +def resource_filename(resource): + """Return filename corresponding to resource. + + resource can be the name of either a file or a directory. + The existence of the resource is not checked. + + :param str resource: Resource path relative to resource directory + using '/' path separator. + :return: Absolute resource path in the file system + """ + # Not in use, uncomment when functionnality is needed + # If _RESOURCES_DOC_DIR is set, use it to get resources in doc/ subflodler + # from an alternative directory. + # if _RESOURCES_DOC_DIR is not None and (resource is 'doc' or + # resource.startswith('doc/')): + # # Remove doc folder from resource relative path + # return os.path.join(_RESOURCES_DOC_DIR, *resource.split('/')[1:]) + + if _RESOURCES_DIR is not None: # if set, use this directory + return os.path.join(_RESOURCES_DIR, *resource.split('/')) + elif importlib_resources is None: # Fallback if pkg_resources is not available + return os.path.join(os.path.abspath(os.path.dirname(__file__)), + *resource.split('/')) + else: # Preferred way to get resources as it supports zipfile package + ref = importlib_resources.files(__name__) / resource + path = file_manager.enter_context(importlib_resources.as_file(ref)) + return str(path) + + +_integrated = False + + +def silx_integration(): + """Provide pyFAI resources accessible throug silx using a prefix.""" + global _integrated + if _integrated: + return + import silx.resources + silx.resources.register_resource_directory("pyfai", + __name__, + _RESOURCES_DIR) + _integrated = True diff --git a/src/freesas/resources/keras_models/Rg+Dmax.keras b/src/freesas/resources/keras_models/Rg+Dmax.keras new file mode 100644 index 0000000..e69de29 diff --git a/src/freesas/resources/keras_models/meson.build b/src/freesas/resources/keras_models/meson.build new file mode 100644 index 0000000..b327524 --- /dev/null +++ b/src/freesas/resources/keras_models/meson.build @@ -0,0 +1,7 @@ +py.install_sources( + [ + 'Rg+Dmax.keras', + ], + pure: false, # Will be installed next to binaries + subdir: 'freesas/resources/keras_models' # Folder relative to site-packages to install to +) diff --git a/src/freesas/resources/meson.build b/src/freesas/resources/meson.build new file mode 100644 index 0000000..2b915d7 --- /dev/null +++ b/src/freesas/resources/meson.build @@ -0,0 +1,9 @@ +subdir('keras_models') + +py.install_sources( + [ + '__init__.py', + ], + pure: false, # Will be installed next to binaries + subdir: 'freesas/resources' # Folder relative to site-packages to install to +) From 6947c6f1f2f00c32f0041921f469e69b27e46635 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Fri, 28 Jun 2024 16:07:24 +0200 Subject: [PATCH 2/3] non regression test for access to keras models --- src/freesas/meson.build | 1 + src/freesas/test/meson.build | 1 + src/freesas/test/test_all.py | 2 ++ src/freesas/test/test_resources.py | 54 ++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 src/freesas/test/test_resources.py diff --git a/src/freesas/meson.build b/src/freesas/meson.build index c2841e5..fe49ebe 100644 --- a/src/freesas/meson.build +++ b/src/freesas/meson.build @@ -1,5 +1,6 @@ subdir('app') subdir('ext') +subdir('resources') subdir('test') diff --git a/src/freesas/test/meson.build b/src/freesas/test/meson.build index 9273818..981fa4c 100644 --- a/src/freesas/test/meson.build +++ b/src/freesas/test/meson.build @@ -12,6 +12,7 @@ py.install_sources([ 'test_model.py', 'test_sas_argparser.py', 'test_sasio.py', +'test_resources.py', 'utilstest.py', ], pure: false, # Will be installed next to binaries diff --git a/src/freesas/test/test_all.py b/src/freesas/test/test_all.py index 1377672..2831e99 100644 --- a/src/freesas/test/test_all.py +++ b/src/freesas/test/test_all.py @@ -17,6 +17,7 @@ from . import test_sasio from . import test_sas_argparser from . import test_fitting +from . import test_resources def suite(): @@ -30,6 +31,7 @@ def suite(): testSuite.addTest(test_sasio.suite()) testSuite.addTest(test_sas_argparser.suite()) testSuite.addTest(test_fitting.suite()) + testSuite.addTest(test_resources.suite()) return testSuite diff --git a/src/freesas/test/test_resources.py b/src/freesas/test/test_resources.py new file mode 100644 index 0000000..c660a8f --- /dev/null +++ b/src/freesas/test/test_resources.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Project: freesas +# https://github.com/kif/freesas +# +# Copyright (C) 2024-2024 European Synchrotron Radiation Facility, Grenoble, France +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +__authors__ = ["Jérôme Kieffer"] +__license__ = "MIT" +__date__ = "28/06/2024" + +import unittest +import logging +import os +from ..resources import resource_filename +logger = logging.getLogger(__name__) + +class TestResources(unittest.TestCase): + + def test_filename(self): + """ + Test for returning the actual path of an existing model + """ + self.assertTrue(os.path.exists(resource_filename("keras_models/Rg+Dmax.keras")), + msg="file exists") + + +def suite(): + test_suite = unittest.TestSuite() + test_suite.addTest(TestResources("test_filename")) + return test_suite + + +if __name__ == '__main__': + runner = unittest.TextTestRunner() + runner.run(suite()) From 5d8978ef3914734727b07c9daa1ef751012c1e8f Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Fri, 28 Jun 2024 16:11:54 +0200 Subject: [PATCH 3/3] replace pyFAI with freesas in comments --- src/freesas/resources/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/freesas/resources/__init__.py b/src/freesas/resources/__init__.py index 35afbad..5ddcc71 100644 --- a/src/freesas/resources/__init__.py +++ b/src/freesas/resources/__init__.py @@ -100,7 +100,7 @@ # Running in a frozen application: # We expect resources to be located either in a pyFAI/resources/ dir # relative to the executable or within this package. - _dir = os.path.join(os.path.dirname(sys.executable), 'pyFAI', 'resources') + _dir = os.path.join(os.path.dirname(sys.executable), 'freesas', 'resources') if os.path.isdir(_dir): _RESOURCES_DIR = _dir @@ -138,12 +138,12 @@ def resource_filename(resource): def silx_integration(): - """Provide pyFAI resources accessible throug silx using a prefix.""" + """Provide freesas resources accessible throug silx using a prefix.""" global _integrated if _integrated: return import silx.resources - silx.resources.register_resource_directory("pyfai", + silx.resources.register_resource_directory("freesas", __name__, _RESOURCES_DIR) _integrated = True