From 82ec2d02736ee0bfe881f4cd88119c5da05a45a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kripner?= Date: Tue, 14 Jan 2025 19:11:20 +0100 Subject: [PATCH 1/6] Add mongo and s3 optional dependencies. Move xmltodict and tqdm only to groups where libraries are actually used. --- requirements.txt | 10 ++++++---- setup.py | 10 ++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 74c52da0..faf37e30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,16 +7,12 @@ numpy<2;python_version=='3.9' # required to use pint uncertainties # required pint # required netCDF4 # required -boto3 # required matplotlib # required scipy # required h5py # required -pymongo # required dnspython # required -xmltodict # required xarray # required setuptools>=41.2 # required -tqdm # required Cython # required importlib_metadata <5;python_version=='3.7' # required @@ -28,10 +24,16 @@ importlib_metadata <5;python_version=='3.7' # required # pyhdc # hdc # imas # imas +# tqdm # imas # pyuda # uda +# pymongo # mongo + +# boto3 # aws + # bs4 # build_structures +# xmltodict # build_structures # Sphinx # build_documentation # sphinx-bootstrap-theme # build_documentation diff --git a/setup.py b/setup.py index 0b7c5c4d..3c171434 100644 --- a/setup.py +++ b/setup.py @@ -7,16 +7,12 @@ 'uncertainties', 'pint', 'netCDF4', - 'boto3', 'matplotlib', 'scipy', 'h5py', - 'pymongo', 'dnspython', - 'xmltodict', 'xarray', 'setuptools>=41.2', - 'tqdm', 'Cython', # latest xarray version that works with 3.7 doesn't work # with importlib_metadata >5 since they deprecated the `get()` method on Entrypoints @@ -31,9 +27,11 @@ 'pygacode', ], 'hdc': ['pyhdc'], - 'imas': ['imas'], + 'imas': ['imas', 'tqdm'], 'uda': ['pyuda'], - 'build_structures': ['bs4'], + 'mongo': ['pymongo'], + 's3': ['boto3'], + 'build_structures': ['bs4', 'xmltodict'], 'build_documentation': ['Sphinx', 'sphinx-bootstrap-theme', 'sphinx-gallery', 'Pillow'], } From a060bab5bf4d2f8bbfda42974eff2fbe6b7ca603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kripner?= Date: Tue, 14 Jan 2025 19:13:31 +0100 Subject: [PATCH 2/6] Remove dnspython dependency as it is most likely not used anymore. --- imports_check.py | 2 +- omas/tests/warning_setup.py | 1 - requirements.txt | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/imports_check.py b/imports_check.py index 9b2cc6e5..484b1630 100644 --- a/imports_check.py +++ b/imports_check.py @@ -10,7 +10,7 @@ warnings.simplefilter('ignore', category=DeprecationWarning) warnings.simplefilter('ignore', category=FutureWarning) -mapper = {'dnspython': None, 'sphinx-bootstrap-theme': None, 'sphinx-gallery': None, 'Sphinx': 'sphinx', 'Pillow': 'PIL'} +mapper = {'sphinx-bootstrap-theme': None, 'sphinx-gallery': None, 'Sphinx': 'sphinx', 'Pillow': 'PIL'} filename = os.path.split(os.path.abspath(__file__))[0] + os.sep + 'requirements.txt' with open(filename, 'r') as f: diff --git a/omas/tests/warning_setup.py b/omas/tests/warning_setup.py index d7a2f15e..f8443247 100644 --- a/omas/tests/warning_setup.py +++ b/omas/tests/warning_setup.py @@ -23,7 +23,6 @@ def set_omas_warnings(): warnings.filterwarnings('always', category=UserWarning, message='No contour levels were found.*') warnings.filterwarnings('always', category=matplotlib.MatplotlibDeprecationWarning) warnings.filterwarnings('always', category=RuntimeWarning, message='invalid value encountered.*') - warnings.filterwarnings('always', category=DeprecationWarning, message='please use dns.resolver.resolve.*') warnings.filterwarnings( 'always', category=UserWarning, message='Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure' ) diff --git a/requirements.txt b/requirements.txt index faf37e30..1d2706fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ netCDF4 # required matplotlib # required scipy # required h5py # required -dnspython # required xarray # required setuptools>=41.2 # required Cython # required From b0b0832374c1660b5f5d6ee906f656c9c729f634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kripner?= Date: Tue, 14 Jan 2025 19:32:06 +0100 Subject: [PATCH 3/6] Add documentation for optional dependencies. --- sphinx/source/install.rst | 46 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/sphinx/source/install.rst b/sphinx/source/install.rst index ac17ae44..b0ed8b7b 100644 --- a/sphinx/source/install.rst +++ b/sphinx/source/install.rst @@ -5,7 +5,7 @@ Installation OMAS runs with *Python3.6+* -**Pypi**: To install `OMAS with pip `_ (for users): +**PyPI**: To install `OMAS with pip `_ (for users): .. code-block:: none @@ -38,6 +38,50 @@ The development version of omas can also be installed with pip: List of `Python package requirements <_static/requirements.txt>`_. +Optional dependencies +--------------------- + +Some dependencies are not required for the core functionality of OMAS, +but are needed for some of the additional features. + +Here is a list of optional dependencies: + +.. Table of optional dependencies: mongo, s3, build_structures, build_documentation + ++----------------------+----------------------+ +| **Package** | **Description** | ++======================+======================+ +| `mongo` | Support for MongoDB | ++----------------------+----------------------+ +| `s3` | Support for AWS S3 | ++----------------------+----------------------+ +| `build_structures` | Support for building IMAS structures | ++----------------------+----------------------+ +| `build_documentation`| Support for building Sphinx documentation | ++----------------------+----------------------+ + +Optional dependencies can be installed with the following command: + +**PyPI**: + +.. code-block:: none + + pip install --upgrade omas[optional,optional2,...] + +**Conda**: All optional dependencies are installed by default. To suppress the installation of optional dependencies, use the following command: + +.. code-block:: none + + conda install -c conda-forge omas --no-deps + +**GitHub**: + +.. code-block:: none + + pip install --upgrade -e '.[optional,optional2,...]' + +**Note**: Skip the `--upgrade` option if you are installing the package for the first time. + Installation with IMAS ====================== From 869348c48f289db8cd9356525e996c6f21766382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kripner?= Date: Tue, 14 Jan 2025 19:44:26 +0100 Subject: [PATCH 4/6] Mark S3 and mongo as optional in how.rst Add installation to github workflow. --- .github/workflows/regression_no_munittest.yml | 2 +- sphinx/source/how.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regression_no_munittest.yml b/.github/workflows/regression_no_munittest.yml index 31aef0db..122a518c 100644 --- a/.github/workflows/regression_no_munittest.yml +++ b/.github/workflows/regression_no_munittest.yml @@ -29,7 +29,7 @@ jobs: - name: Install OMAS run: | - python3 -m pip install .[machine] + python3 -m pip install .[machine, s3, mongo] - name: Move to root and try importing OMAS run: | diff --git a/sphinx/source/how.rst b/sphinx/source/how.rst index cf9293df..09e06649 100644 --- a/sphinx/source/how.rst +++ b/sphinx/source/how.rst @@ -145,9 +145,9 @@ The `ODS` class extends native Python dictionary and list classes with: +---------------+--------------------------------------------------------------------------------------+-------------------------------+----------------+-----------------------+ | **HDF5** | Files using binary HDF5 format (hierarchical data structure) | Binary file | no | h5py | +---------------+--------------------------------------------------------------------------------------+-------------------------------+----------------+-----------------------+ -| **S3** | Cloud storage using Amazon Simple Storage Service | Object Store | yes | boto3 | +| **S3** | Cloud storage using Amazon Simple Storage Service | Object Store | yes | boto3 (optional) | +---------------+--------------------------------------------------------------------------------------+-------------------------------+----------------+-----------------------+ -| **MongoDB** | Document-based distributed database | MongoDB Database | yes | pymongo | +| **MongoDB** | Document-based distributed database | MongoDB Database | yes | pymongo (optional) | +---------------+--------------------------------------------------------------------------------------+-------------------------------+----------------+-----------------------+ | **HDC** | Hierarchical Dynamic Containers | Memory | no | pyhdc (optional) | +---------------+--------------------------------------------------------------------------------------+-------------------------------+----------------+-----------------------+ From 5dd6fad060521eb5df99eb94358b45bb6753a8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kripner?= Date: Tue, 14 Jan 2025 19:49:34 +0100 Subject: [PATCH 5/6] Further (blind) attempts to fix deployment workflow. --- .github/workflows/regression.yml | 1 + .github/workflows/regression_no_munittest.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 41e1740f..3cf4b245 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -42,6 +42,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install -r requirements.txt + python3 -m pip install boto3 pymongo # numpy is a build time dependency of pygacode # but before pygacode 1.0 it did not specify its build dependencies correctly diff --git a/.github/workflows/regression_no_munittest.yml b/.github/workflows/regression_no_munittest.yml index 122a518c..5a5b7454 100644 --- a/.github/workflows/regression_no_munittest.yml +++ b/.github/workflows/regression_no_munittest.yml @@ -26,10 +26,11 @@ jobs: - name: Install dependencies run: | python3 -m pip install -r requirements.txt + python3 -m pip install boto3 pymongo - name: Install OMAS run: | - python3 -m pip install .[machine, s3, mongo] + python3 -m pip install .[machine] - name: Move to root and try importing OMAS run: | From 3ad7138f145790fdcc1b36a9cc81bb9949c7cc41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Kripner?= Date: Tue, 14 Jan 2025 19:53:15 +0100 Subject: [PATCH 6/6] add xmltodict dep --- .github/workflows/regression.yml | 2 +- .github/workflows/regression_no_munittest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 3cf4b245..5d81937c 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -42,7 +42,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install -r requirements.txt - python3 -m pip install boto3 pymongo + python3 -m pip install boto3 pymongo xmltodict # numpy is a build time dependency of pygacode # but before pygacode 1.0 it did not specify its build dependencies correctly diff --git a/.github/workflows/regression_no_munittest.yml b/.github/workflows/regression_no_munittest.yml index 5a5b7454..5be5c37a 100644 --- a/.github/workflows/regression_no_munittest.yml +++ b/.github/workflows/regression_no_munittest.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install -r requirements.txt - python3 -m pip install boto3 pymongo + python3 -m pip install boto3 pymongo xmltodict - name: Install OMAS run: |