From cb9181ba5c2c9bf986e596c48e3071afa9d80722 Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe <13301940+andersy005@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:44:25 -0800 Subject: [PATCH] Update dependencies and improve import handling for xarray version compatibility (#696) --- ci/environment-docs.yml | 10 +- ci/environment-upstream-dev.yml | 6 +- ci/environment.yml | 10 +- intake_esm/core.py | 15 ++- pyproject.toml | 207 +++++++++++++++----------------- requirements.txt | 6 +- tests/test_core.py | 7 +- 7 files changed, 128 insertions(+), 133 deletions(-) diff --git a/ci/environment-docs.yml b/ci/environment-docs.yml index 4072fd47..ac9f8b9b 100644 --- a/ci/environment-docs.yml +++ b/ci/environment-docs.yml @@ -4,11 +4,11 @@ channels: - nodefaults dependencies: - cftime - - dask[complete]>=2022.02.0 + - dask>=2024.12 - distributed - ecgtools >=2022.10.7 - - fsspec>=2023.05 - - gcsfs >=2023.05 + - fsspec>=2024.12 + - gcsfs >=2024.12 - intake>=2.0 - jupyterlab - matplotlib @@ -19,13 +19,13 @@ dependencies: - pydantic>=2.0 - python-graphviz - python=3.11 - - s3fs >=2023.05 + - s3fs >=2024.12 - sphinx<6.0 - sphinx-copybutton - sphinx-design - watermark - xarray-datatree>=0.0.9 - - xarray>=2023.05 + - xarray>=2024.10 - zarr>=2.12 - furo>=2022.09.15 - pip: diff --git a/ci/environment-upstream-dev.yml b/ci/environment-upstream-dev.yml index 43d676ec..335b5a16 100644 --- a/ci/environment-upstream-dev.yml +++ b/ci/environment-upstream-dev.yml @@ -6,9 +6,9 @@ dependencies: - cartopy - cftime - codecov - - dask>=2022.02.0 + - dask>=2024.12 - fastprogress>=1.0.0 - - gcsfs >=2022.11.0 + - gcsfs >=2024.12 - h5netcdf>=0.8.1 - ipython - matplotlib @@ -28,7 +28,7 @@ dependencies: - pytest-sugar - pytest-xdist - regionmask - - s3fs >=2022.11.0 + - s3fs >=2024.12 - scipy - xarray-datatree - xgcm diff --git a/ci/environment.yml b/ci/environment.yml index 95808f00..4373d969 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -5,10 +5,10 @@ channels: dependencies: - cftime - codecov - - dask[complete]>=2022.02.0 + - dask>=2024.12 - fastprogress>=1.0.0 - - fsspec>=2022.11.0 - - gcsfs >=2022.11.0 + - fsspec>=2024.12 + - gcsfs >=2024.12 - h5netcdf>=0.8.1 - intake>=2.0 - ipython @@ -25,9 +25,9 @@ dependencies: - pytest-sugar - pytest-xdist - pytest-mock - - s3fs >=2022.11.0 + - s3fs >=2024.12 - scipy - - xarray>=2022.06, !=2024.10 + - xarray>=2024.10 - xarray-datatree - zarr>=2.12 # - pytest-icdiff diff --git a/intake_esm/core.py b/intake_esm/core.py index 0c9c8c16..a9de2e91 100644 --- a/intake_esm/core.py +++ b/intake_esm/core.py @@ -5,16 +5,19 @@ from copy import deepcopy import dask +import packaging.version +import xarray as xr try: - from datatree import DataTree - + if packaging.version.Version(xr.__version__) < packaging.version.Version('2024.10'): + from datatree import DataTree + else: + from xarray import DataTree _DATATREE_AVAILABLE = True except ImportError: _DATATREE_AVAILABLE = False import pandas as pd import pydantic -import xarray as xr from fastprogress.fastprogress import progress_bar from intake.catalog import Catalog @@ -81,9 +84,9 @@ def __init__( progressbar: bool = True, sep: str = '.', registry: DerivedVariableRegistry | None = None, - read_csv_kwargs: dict[str, typing.Any] = None, - columns_with_iterables: list[str] = None, - storage_options: dict[str, typing.Any] = None, + read_csv_kwargs: dict[str, typing.Any] | None = None, + columns_with_iterables: list[str] | None = None, + storage_options: dict[str, typing.Any] | None = None, **intake_kwargs: dict[str, typing.Any], ): """Intake Catalog representing an ESM Collection.""" diff --git a/pyproject.toml b/pyproject.toml index e2dd146c..f533455c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,139 +1,126 @@ [build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"] + requires = ["setuptools>=45", "setuptools_scm>=6.2", "wheel"] [project] -name = "intake-esm" -description = "An intake plugin for parsing an Earth System Model (ESM) catalog and loading netCDF files and/or Zarr stores into Xarray datasets." -readme = "README.md" -license = {text="Apache Software License 2.0"} -requires-python = ">=3.10" -maintainers = [ - { name = "NCAR XDev Team", email = "xdev@ucar.edu" }, -] -keywords = [ - "catalog", - "intake", - "xarray", -] -classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering", -] - -dynamic = ["version", "dependencies"] + classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python", + "Topic :: Scientific/Engineering", + ] + description = "An intake plugin for parsing an Earth System Model (ESM) catalog and loading netCDF files and/or Zarr stores into Xarray datasets." + keywords = ["catalog", "intake", "xarray"] + license = { text = "Apache Software License 2.0" } + maintainers = [{ name = "NCAR XDev Team", email = "xdev@ucar.edu" }] + name = "intake-esm" + readme = "README.md" + requires-python = ">=3.10" + + dynamic = ["dependencies", "version"] [tool.setuptools.dynamic] -dependencies = { file = ["requirements.txt"] } -optional-dependencies = { dev = { file = ["requirements-dev.txt"] } } + dependencies = { file = ["requirements.txt"] } + optional-dependencies = { dev = { file = ["requirements-dev.txt"] } } [project.entry-points."intake.drivers"] -esm_datasource = "intake_esm.source:ESMDataSource" -esm_datastore = "intake_esm.core:esm_datastore" + esm_datasource = "intake_esm.source:ESMDataSource" + esm_datastore = "intake_esm.core:esm_datastore" [project.urls] -Documentation = "https://intake-esm.readthedocs.io" -Homepage = "https://intake-esm.readthedocs.io" -Source = "https://github.com/intake/intake-esm" -Tracker = "https://github.com/intake/intake-esm/issues" - + Documentation = "https://intake-esm.readthedocs.io" + Homepage = "https://intake-esm.readthedocs.io" + Source = "https://github.com/intake/intake-esm" + Tracker = "https://github.com/intake/intake-esm/issues" [tool.setuptools.packages.find] -include = ["intake_esm*"] + include = ["intake_esm*"] [tool.setuptools.package-data] -intake_esm = ["py.typed"] - - + intake_esm = ["py.typed"] [tool.setuptools_scm] -version_scheme = "post-release" -local_scheme = "node-and-date" -fallback_version = "999" -write_to = "intake_esm/_version.py" -write_to_template = '__version__ = "{version}"' - - + fallback_version = "999" + local_scheme = "node-and-date" + version_scheme = "post-release" + write_to = "intake_esm/_version.py" + write_to_template = '__version__ = "{version}"' [tool.ruff] -line-length = 100 -target-version = "py310" -extend-include = ["*.ipynb"] - - -builtins = ["ellipsis"] -# Exclude a variety of commonly ignored directories. -exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".ipynb_checkpoints", - ".mypy_cache", - ".nox", - ".pants.d", - ".pyenv", - ".pytest_cache", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - ".vscode", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "site-packages", - "venv", -] + extend-include = ["*.ipynb"] + line-length = 100 + target-version = "py310" + + builtins = ["ellipsis"] + # Exclude a variety of commonly ignored directories. + exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", + ] [tool.ruff.lint] -per-file-ignores = {} -ignore = [ - "E721", # Comparing types instead of isinstance - "E741", # Ambiguous variable names - "E501", # Conflicts with ruff format -] -select = [ - # Pyflakes - "F", - # Pycodestyle - "E", - "W", - # isort - "I", - # Pyupgrade - "UP", -] - + ignore = [ + "E501", # Conflicts with ruff format + "E721", # Comparing types instead of isinstance + "E741", # Ambiguous variable names + ] + per-file-ignores = {} + select = [ + # Pyflakes + "F", + # Pycodestyle + "E", + "W", + # isort + "I", + # Pyupgrade + "UP", + ] [tool.ruff.lint.mccabe] -max-complexity = 18 + max-complexity = 18 [tool.ruff.lint.isort] -known-first-party = ["intake_esm"] -combine-as-imports = true + combine-as-imports = true + known-first-party = ["intake_esm"] [tool.ruff.format] -quote-style = "single" -docstring-code-format = true + docstring-code-format = true + quote-style = "single" [tool.ruff.lint.pydocstyle] -convention = "numpy" + convention = "numpy" [tool.pytest.ini_options] -console_output_style = "count" -addopts = "--cov=./ --cov-report=xml --verbose" -markers = "network: tests requiring a network connection" + addopts = "--cov=./ --cov-report=xml --verbose" + console_output_style = "count" + markers = "network: tests requiring a network connection" diff --git a/requirements.txt b/requirements.txt index 7de0800d..0bc5e820 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -dask[complete]>=2022.02.0 +dask[complete]>=2024.12 fastprogress>=1.0.0 -fsspec>=2022.11.0 +fsspec>=2024.12 intake>=2.0.0 netCDF4>=1.5.5 pandas>=2.1.0 requests>=2.24.0 -xarray>=2022.06 +xarray>=2024.10 zarr>=2.12 pydantic>=2.0 diff --git a/tests/test_core.py b/tests/test_core.py index 38055ed2..7c8e5a48 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,11 +2,16 @@ import os import intake +import packaging.version import pandas as pd import pydantic import pytest import xarray as xr -from datatree import DataTree + +if packaging.version.Version(xr.__version__) < packaging.version.Version('2024.10'): + from datatree import DataTree +else: + from xarray import DataTree import intake_esm