diff --git a/datashader/core.py b/datashader/core.py index b8511e225..44ed2fefc 100644 --- a/datashader/core.py +++ b/datashader/core.py @@ -1287,7 +1287,11 @@ def _source_from_geopandas(self, source): with contextlib.suppress(ImportError): import dask_geopandas - if Version(dask_geopandas.__version__) >= Version("0.4.0"): + + DGP_VERSION = Version(dask_geopandas.__version__).release + # Version 0.4.0 added support for dask_expr and 0.4.3 removed + # support for classic DataFrame + if (0, 4, 0) <= DGP_VERSION < (0, 4, 3): from dask_geopandas.core import GeoDataFrame as gdf1 dfs.append(gdf1) diff --git a/datashader/data_libraries/dask.py b/datashader/data_libraries/dask.py index 12c4f39c0..6399bad5c 100644 --- a/datashader/data_libraries/dask.py +++ b/datashader/data_libraries/dask.py @@ -1,6 +1,5 @@ from __future__ import annotations -from contextlib import suppress import numpy as np import pandas as pd @@ -51,13 +50,7 @@ def dask_pipeline(df, schema, canvas, glyph, summary, *, antialias=False, cuda=F return scheduler(dsk, name) -# Classic Dask.DataFrame -bypixel.pipeline.register(dd.core.DataFrame)(dask_pipeline) - -with suppress(ImportError): - import dask_expr - - bypixel.pipeline.register(dask_expr.DataFrame)(dask_pipeline) +bypixel.pipeline.register(dd.DataFrame)(dask_pipeline) def shape_bounds_st_and_axis(df, canvas, glyph): diff --git a/datashader/tests/utils.py b/datashader/tests/utils.py index 17100611c..fc65e389c 100644 --- a/datashader/tests/utils.py +++ b/datashader/tests/utils.py @@ -2,16 +2,50 @@ from contextlib import contextmanager from importlib import reload from importlib.util import find_spec +from contextlib import suppress +from functools import lru_cache import pytest +from packaging.version import Version -__all__ = ("dask_switcher", "DASK_UNAVAILABLE", "EXPR_UNAVAILABLE", "dask_skip") +__all__ = ("dask_switcher", "DASK_UNAVAILABLE", "dask_skip") DASK_UNAVAILABLE = find_spec("dask") is None -EXPR_UNAVAILABLE = find_spec("dask_expr") is None dask_skip = pytest.mark.skipif(DASK_UNAVAILABLE, reason="dask is not available") + +@lru_cache +def _dask_setup(): + """ + Set-up both dask dataframes, using lru_cahce to only do it once + + """ + import dask + from datashader.data_libraries.dask import bypixel, dask_pipeline + + classic, expr = False, False + + # Removed in Dask 2025.1, and will raise AttributeError + if Version(dask.__version__).release < (2025, 1, 0): + import dask.dataframe as dd + + bypixel.pipeline.register(dd.core.DataFrame)(dask_pipeline) + classic = True + else: + # dask_expr import below will now fail with: + # cannot import name '_Frame' from 'dask.dataframe.core' + expr = True + + with suppress(ImportError): + import dask_expr + + bypixel.pipeline.register(dask_expr.DataFrame)(dask_pipeline) + expr = True + + return classic, expr + + @contextmanager def dask_switcher(*, query=False, extras=None): """ @@ -22,7 +56,12 @@ def dask_switcher(*, query=False, extras=None): """ if DASK_UNAVAILABLE: pytest.skip("dask is not available") - if query and EXPR_UNAVAILABLE: + + classic, expr = _dask_setup() + + if not query and not classic: + pytest.skip("Classic DataFrame no longer supported by dask") + if query and not expr: pytest.skip("dask-expr is not available") import dask diff --git a/examples/conftest.py b/examples/conftest.py index 4964ac611..a67ea6aab 100644 --- a/examples/conftest.py +++ b/examples/conftest.py @@ -1,4 +1,5 @@ from importlib.util import find_spec +from packaging.version import Version collect_ignore_glob = [ "tiling.ipynb", @@ -14,3 +15,22 @@ "user_guide/7_Networks.ipynb", "user_guide/8_Polygons.ipynb", ] + +if find_spec("dask") is not None: + import dask + + # Spatialpandas does not support dask-expr, which is + # only available from this version. + if Version(dask.__version__).release >= (2025, 1, 0): + collect_ignore_glob += [ + "user_guide/8_Polygons.ipynb", + ] + + +def pytest_sessionfinish(session, exitstatus): + # Can be removed when spatialpandas work with dask-expr + + from pytest import ExitCode + + if exitstatus == ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = ExitCode.OK diff --git a/pixi.toml b/pixi.toml index 6c41eaf4d..83a0c0ec4 100644 --- a/pixi.toml +++ b/pixi.toml @@ -22,7 +22,8 @@ build = ["py311", "build"] lint = ["py311", "lint"] [dependencies] -numba = "*" # At top as this dictates Python version +python = "<3.13" +numba = "*" # At top as this dictates Python version colorcet = "*" multipledispatch = "*" numpy = "*" @@ -67,10 +68,10 @@ holoviews = "*" matplotlib-base = ">=3.3" networkx = "*" panel = ">1.1" -pyogrio = "*" +pyogrio = "*" python-graphviz = "*" python-snappy = "*" -rasterio = "!=1.4.0" # 2024-11: Errors and will not solve to latest on Linux + Python 3.9 +rasterio = "!=1.4.0" # 2024-11: Errors and will not solve to latest on Linux + Python 3.9 scikit-image = "*" shapely = ">=2.0.0" spatialpandas = "*" @@ -100,8 +101,8 @@ geodatasets = "*" geopandas-base = "*" netcdf4 = "*" pyarrow = "*" -pyogrio = "*" -rasterio = "!=1.4.0" # 2024-11: Errors and will not solve to latest on Linux + Python 3.9 +pyogrio = "*" +rasterio = "!=1.4.0" # 2024-11: Errors and will not solve to latest on Linux + Python 3.9 rioxarray = "*" scikit-image = "*" shapely = ">=2.0.0" @@ -111,7 +112,9 @@ spatialpandas = "*" nbval = "*" [feature.test-example.tasks] -test-example = { cmd = 'pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip', env = { DASK_DATAFRAME__QUERY_PLANNING = "False" } } +_test-example-1 = 'pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip -k "not Polygon"' +_test-example-2 = { cmd = 'pytest -n logical --dist loadscope --nbval-lax examples --benchmark-skip -k "Polygon"', env = { DASK_DATAFRAME__QUERY_PLANNING = "False" } } +test-example = { depends-on = ["_test-example-1", "_test-example-2"] } [feature.test-gpu] channels = ["rapidsai"] @@ -141,6 +144,7 @@ nbsite = ">=0.8.4,<0.9.0" numpydoc = "*" sphinxcontrib-mermaid = "*" sphinx-reredirects = "*" +dask-core = "<2025.1" # Spatialpandas does not currently support dask-expr [feature.doc.activation.env] DASK_DATAFRAME__QUERY_PLANNING = "False" @@ -165,7 +169,6 @@ build-pip = 'python -m build .' [feature.build.activation.env] MPLBACKEND = "Agg" -DASK_DATAFRAME__QUERY_PLANNING = "False" # ============================================= # =================== LINT ==================== diff --git a/examples/filetimes.py b/scripts/filetimes/filetimes.py similarity index 100% rename from examples/filetimes.py rename to scripts/filetimes/filetimes.py diff --git a/examples/filetimes.sh b/scripts/filetimes/filetimes.sh similarity index 100% rename from examples/filetimes.sh rename to scripts/filetimes/filetimes.sh diff --git a/examples/filetimes.yml b/scripts/filetimes/filetimes.yml similarity index 100% rename from examples/filetimes.yml rename to scripts/filetimes/filetimes.yml