Compatibility with geopandas 1.0 and dask-geopandas 0.4.0 (#1347)

Co-authored-by: Demetris Roumis <[email protected]>

.github/workflows/test.yaml

@@ -66,20 +66,9 @@ jobs:
           MATRIX=$(jq -nsc '{
               "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
               "python-version": ["3.9", "3.10", "3.11", "3.12"],
-              "numpy-version": ["1"],
               "exclude": [{
                   "os": "windows-latest",
                   "python-version": "3.9"
-              }],
-              "include": [{
-                  "os": "ubuntu-latest",
-                  "python-version": "3.12",
-                  "numpy-version": "2"
-              },
-              {
-                  "os": "macos-latest",
-                  "python-version": "3.12",
-                  "numpy-version": "2"
               }]
           }')
           # Won't solve on Windows + Python 3.9
@@ -90,20 +79,9 @@ jobs:
           MATRIX=$(jq -nsc '{
               "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
               "python-version": ["3.9", "3.10", "3.11", "3.12"],
-              "numpy-version": ["1"],
               "exclude": [{
                   "os": "windows-latest",
                   "python-version": "3.9"
-              }],
-              "include": [{
-                  "os": "ubuntu-latest",
-                  "python-version": "3.12",
-                  "numpy-version": "2"
-              },
-              {
-                  "os": "macos-latest",
-                  "python-version": "3.12",
-                  "numpy-version": "2"
               }]
           }')
           # Won't solve on Windows + Python 3.9
@@ -113,13 +91,12 @@ jobs:
         run: |
           MATRIX=$(jq -nsc '{
               "os": ["ubuntu-latest"],
-              "numpy-version": ["1"],
               "python-version": ["3.11"]
           }')
           echo "MATRIX=$MATRIX" >> $GITHUB_ENV
 
   test_suite:
-    name: Tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}, numpy ${{ matrix.numpy-version }}
+    name: Tests on ${{ matrix.os }} with Python ${{ matrix.python-version }}
     needs: [pre_commit, setup]
     runs-on: ${{ matrix.os }}
     strategy:
@@ -132,32 +109,16 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - name: Set channels and envs
-        run: |
-          if [[ ${{ matrix.numpy-version }} == "2" ]]; then
-            channels="pyviz/label/dev,conda-forge/label/numpy_rc,numba/label/dev,conda-forge,nodefaults"
-            envs="-o numpy2"
-          else
-            channels="pyviz/label/dev,numba,conda-forge,nodefaults"
-            envs="-o tests -o examples"
-          fi
-          echo "CHANNELS=$channels" >> $GITHUB_ENV
-          echo "ENVS=$envs" >> $GITHUB_ENV
       - uses: holoviz-dev/holoviz_tasks/install@v0
         with:
-          name: unit_test_suite_np${{ matrix.numpy-version }}
+          name: unit_test_suite
           python-version: ${{ matrix.python-version }}
           channel-priority: flexible
-          channels: ${{ env.CHANNELS }}
-          envs: ${{ env.ENVS }}
+          channels: "pyviz/label/dev,numba,conda-forge,nodefaults"
+          envs: "-o tests -o examples"
           cache: ${{ github.event.inputs.cache || github.event.inputs.cache == '' }}
           conda-update: true
         id: install
-      - name: check version
-        run: |
-          conda activate test-environment
-          python -c "import numba; print('Numba', numba.__version__)"
-          python -c "import numpy; print('Numpy', numpy.__version__)"
       - name: download data
         run: |
           conda activate test-environment
@@ -174,7 +135,7 @@ jobs:
           NUMBA_DISABLE_JIT: 1
       - name: doit test_examples
         env:
-          DASK_DATAFRAME__QUERY_PLANNING: false
+          DASK_DATAFRAME__QUERY_PLANNING: False
         run: |
           conda activate test-environment
           doit test_examples

.gitignore

@@ -82,6 +82,7 @@ examples/export
 examples/tiles_output_directory
 examples/user_guide/export
 examples/user_guide/df_world.parq
+examples/user_guide/sgeodf.parq
 
 **.org
 .doit*

datashader/core.py

@@ -3,11 +3,13 @@
 from numbers import Number
 from math import log10
 import warnings
+import contextlib
 
 import numpy as np
 import pandas as pd
 import dask.dataframe as dd
 import dask.array as da
+from packaging.version import Version
 from xarray import DataArray, Dataset
 
 from .utils import Dispatcher, ngjit, calc_res, calc_bbox, orient_array, \
@@ -1274,20 +1276,23 @@ def _source_from_geopandas(self, source):
         If so, spatially filter the source and return it.
         If not, return None.
         """
-        try:
+        dfs = []
+        with contextlib.suppress(ImportError):
             import geopandas
-        except ImportError:
-            geopandas = None
+            dfs.append(geopandas.GeoDataFrame)
 
-        try:
+        with contextlib.suppress(ImportError):
             import dask_geopandas
-        except ImportError:
-            dask_geopandas = None
+            if Version(dask_geopandas.__version__) >= Version("0.4.0"):
+                from dask_geopandas.core import GeoDataFrame as gdf1
+                from dask_geopandas.expr import GeoDataFrame as gdf2
 
-        if ((geopandas and isinstance(source, geopandas.GeoDataFrame)) or
-              (dask_geopandas and isinstance(source, dask_geopandas.GeoDataFrame))):
+                dfs.extend([gdf1, gdf2])
+            else:
+                dfs.append(dask_geopandas.GeoDataFrame)
+
+        if isinstance(source, tuple(dfs)):
             # Explicit shapely version check as cannot continue unless shapely >= 2
-            from packaging.version import Version
             from shapely import __version__ as shapely_version
             if Version(shapely_version) < Version('2.0.0'):
                 raise ImportError("Use of GeoPandas in Datashader requires Shapely >= 2.0.0")

datashader/tests/test_geopandas.py

@@ -1,16 +1,34 @@
 # Testing GeoPandas and SpatialPandas
+import contextlib
+
 import dask.dataframe as dd
 import datashader as ds
 from datashader.tests.test_pandas import assert_eq_ndarray
 import numpy as np
 from numpy import nan
 import pytest
 from datashader.tests.utils import dask_switcher
+from packaging.version import Version
+
+_backends = [
+    pytest.param(False, id="dask"),
+]
+
+with contextlib.suppress(ImportError):
+    import dask_geopandas
+
+    if Version(dask_geopandas.__version__) >= Version("0.4.0"):
+        _backends.append(pytest.param(True, id="dask-expr"))
 
 
-@pytest.fixture(autouse=True)
-def _classic_dd():
-    with dask_switcher(query=False, extras=["spatialpandas.dask", "dask_geopandas"]): ...
+@pytest.fixture(params=_backends)
+def dask_both(request):
+    with dask_switcher(query=request.param, extras=["spatialpandas.dask", "dask_geopandas.backends", "dask_geopandas"]): ...
+    return request.param
+
+@pytest.fixture
+def dask_classic(request):
+    with dask_switcher(query=False, extras=["spatialpandas.dask", "dask_geopandas.backends", "dask_geopandas"]): ...
 
 try:
     import dask_geopandas
@@ -105,6 +123,15 @@ def _classic_dd():
 ])
 
 
+@pytest.mark.skipif(not dask_geopandas, reason="dask_geopandas not installed")
+def test_dask_geopandas_switcher(dask_both):
+    import dask_geopandas
+    if dask_both:
+        assert dask_geopandas.expr.GeoDataFrame == dask_geopandas.GeoDataFrame
+    else:
+        assert dask_geopandas.core.GeoDataFrame == dask_geopandas.GeoDataFrame
+
+
 @pytest.mark.skipif(not geodatasets, reason="geodatasets not installed")
 @pytest.mark.skipif(not geopandas, reason="geopandas not installed")
 @pytest.mark.parametrize("geom_type, explode, use_boundary",
@@ -144,7 +171,7 @@ def test_lines_geopandas(geom_type, explode, use_boundary):
         ("linestring", True, True),
     ],
 )
-def test_lines_dask_geopandas(geom_type, explode, use_boundary, npartitions):
+def test_lines_dask_geopandas(geom_type, explode, use_boundary, npartitions, dask_both):
     df = geopandas.read_file(geodatasets.get_path("nybb"))
     df["col"] = np.arange(len(df))  # Extra column for aggregation.
     geometry = "boundary" if use_boundary else "geometry"
@@ -176,7 +203,7 @@ def test_lines_dask_geopandas(geom_type, explode, use_boundary, npartitions):
         ("linestring", True, True),
     ],
 )
-def test_lines_spatialpandas(geom_type, explode, use_boundary, npartitions):
+def test_lines_spatialpandas(geom_type, explode, use_boundary, npartitions, dask_classic):
     df = geopandas.read_file(geodatasets.get_path("nybb"))
     df["col"] = np.arange(len(df))  # Extra column for aggregation.
     geometry = "boundary" if use_boundary else "geometry"
@@ -219,7 +246,7 @@ def test_points_geopandas(geom_type):
 @pytest.mark.skipif(not geopandas, reason="geopandas not installed")
 @pytest.mark.parametrize('npartitions', [1, 2, 5])
 @pytest.mark.parametrize("geom_type", ["multipoint", "point"])
-def test_points_dask_geopandas(geom_type, npartitions):
+def test_points_dask_geopandas(geom_type, npartitions, dask_both):
     df = geopandas.read_file(geodatasets.get_path("nybb"))
 
     df["geometry"] = df["geometry"].sample_points(100, rng=93814)  # multipoint
@@ -241,7 +268,7 @@ def test_points_dask_geopandas(geom_type, npartitions):
 @pytest.mark.skipif(not spatialpandas, reason="spatialpandas not installed")
 @pytest.mark.parametrize('npartitions', [0, 1, 2, 5])
 @pytest.mark.parametrize("geom_type", ["multipoint", "point"])
-def test_points_spatialpandas(geom_type, npartitions):
+def test_points_spatialpandas(geom_type, npartitions, dask_classic):
     df = geopandas.read_file(geodatasets.get_path("nybb"))
 
     df["geometry"] = df["geometry"].sample_points(100, rng=93814)  # multipoint
@@ -282,7 +309,7 @@ def test_polygons_geopandas(geom_type):
 @pytest.mark.skipif(not geopandas, reason="geopandas not installed")
 @pytest.mark.parametrize('npartitions', [1, 2, 5])
 @pytest.mark.parametrize("geom_type", ["multipolygon", "polygon"])
-def test_polygons_dask_geopandas(geom_type, npartitions):
+def test_polygons_dask_geopandas(geom_type, npartitions, dask_both):
     df = geopandas.read_file(geodatasets.get_path("nybb"))
     df["col"] = np.arange(len(df))
 
@@ -305,7 +332,7 @@ def test_polygons_dask_geopandas(geom_type, npartitions):
 @pytest.mark.skipif(not spatialpandas, reason="spatialpandas not installed")
 @pytest.mark.parametrize('npartitions', [0, 1, 2, 5])
 @pytest.mark.parametrize("geom_type", ["multipolygon", "polygon"])
-def test_polygons_spatialpandas(geom_type, npartitions):
+def test_polygons_spatialpandas(geom_type, npartitions, dask_classic):
     df = geopandas.read_file(geodatasets.get_path("nybb"))
     df["col"] = np.arange(len(df))
 

doc/conf.py

@@ -58,8 +58,11 @@
     'sphinx.ext.autosummary',
     'numpydoc',
     'nbsite.analytics',
+    'sphinxcontrib.mermaid',
 ]
 
+myst_fence_as_directive = ["mermaid"]
+
 nbsite_analytics = {
     'goatcounter_holoviz': True,
 }