From 171003f63b038fedda1b6cd18caa5c110d49df52 Mon Sep 17 00:00:00 2001 From: "S. Andrew Sheppard" Date: Tue, 19 Mar 2024 20:35:19 +0000 Subject: [PATCH] option to defer geojson for large layers --- tests/gis_app/rest.py | 2 +- tests/test_gis.py | 116 ++++++++++++++++++++++++++++++++++++++ tests/test_post.py | 54 +----------------- tests/test_vector_tile.py | 27 --------- wq/db/rest/maps.py | 39 ++++++++++++- wq/db/rest/routers.py | 20 ++++++- wq/db/rest/serializers.py | 14 ++++- 7 files changed, 185 insertions(+), 87 deletions(-) create mode 100644 tests/test_gis.py delete mode 100644 tests/test_vector_tile.py diff --git a/tests/gis_app/rest.py b/tests/gis_app/rest.py index 2d6a6d2..5273ed8 100644 --- a/tests/gis_app/rest.py +++ b/tests/gis_app/rest.py @@ -4,4 +4,4 @@ if settings.WITH_GIS: rest.router.register(GeometryModel, fields="__all__") - rest.router.register(PointModel, fields="__all__") + rest.router.register(PointModel, fields="__all__", defer_geometry=True) diff --git a/tests/test_gis.py b/tests/test_gis.py new file mode 100644 index 0000000..ce8bee8 --- /dev/null +++ b/tests/test_gis.py @@ -0,0 +1,116 @@ +import unittest +from .base import APITestCase +from rest_framework import status +from django.conf import settings +from .gis_app.models import GeometryModel, PointModel +from django.contrib.auth.models import User +import json + + +class GISTestCase(APITestCase): + def setUp(self): + self.user = User.objects.create(username="testuser", is_superuser=True) + self.client.force_authenticate(self.user) + + @unittest.skipUnless(settings.WITH_GIS, "requires GIS") + def test_rest_geometry_post_geojson(self): + """ + Posting GeoJSON to a model with a geometry field should work. + """ + form = { + "name": "Geometry Test 1", + "geometry": json.dumps( + {"type": "Point", "coordinates": [-90, 44]} + ), + } + + # Test for expected response + response = self.client.post("/geometrymodels.json", form) + self.assertEqual( + response.status_code, status.HTTP_201_CREATED, response.data + ) + + # Double-check ORM model & geometry attribute + obj = GeometryModel.objects.get(id=response.data["id"]) + geom = obj.geometry + self.assertEqual(geom.srid, 4326) + self.assertEqual(geom.x, -90) + self.assertEqual(geom.y, 44) + + @unittest.skipUnless(settings.WITH_GIS, "requires GIS") + def test_rest_geometry_post_wkt(self): + """ + Posting WKT to a model with a geometry field should work. + """ + form = { + "name": "Geometry Test 2", + "geometry": "POINT(%s %s)" % (-97, 50), + } + + # Test for expected response + response = self.client.post("/geometrymodels.json", form) + self.assertEqual( + response.status_code, status.HTTP_201_CREATED, response.data + ) + + # Double-check ORM model & geometry attribute + obj = GeometryModel.objects.get(id=response.data["id"]) + geom = obj.geometry + self.assertEqual(geom.srid, 4326) + self.assertEqual(geom.x, -97) + self.assertEqual(geom.y, 50) + + @unittest.skipIf(settings.VARIANT == "postgis", "postgis supports tiles") + def test_no_tiles(self): + tile = self.client.get("/tiles/0/0/0.pbf") + self.assertEqual( + tile.content.decode(), + "Tile server not supported with this database engine.", + ) + + @unittest.skipUnless(settings.VARIANT == "postgis", "requires postgis") + def test_tiles(self): + empty_tile = self.client.get("/tiles/0/0/0.pbf").content + self.assertEqual(b"", empty_tile) + + PointModel.objects.create(pk=1, geometry="POINT(34 -84)") + single_point = self.client.get("/tiles/0/0/0.pbf").content + self.assertEqual( + single_point, + b'\x1a\x1e\n\npointmodel\x12\x0b\x08\x01\x18\x01"\x05\t\x86&\x84>(\x80 x\x02', + ) + + @unittest.skipUnless(settings.WITH_GIS, "requires GIS") + def test_defer_geometry(self): + PointModel.objects.create(pk=1, geometry="POINT(34 -84)") + points_json = self.client.get("/pointmodels.json").data["list"] + self.assertEqual( + points_json, + [ + { + "id": 1, + "name": "", + "label": "", + } + ], + ) + points_geojson = self.client.get("/pointmodels.geojson").data[ + "features" + ] + self.assertEqual( + points_geojson, + [ + { + "id": 1, + "type": "Feature", + "properties": { + "name": "", + "label": "", + }, + "geometry": { + "type": "Point", + "coordinates": [34, -84], + }, + } + ], + ) diff --git a/tests/test_post.py b/tests/test_post.py index 1992d79..1259373 100644 --- a/tests/test_post.py +++ b/tests/test_post.py @@ -1,11 +1,7 @@ -import unittest from .base import APITestCase from rest_framework import status -import json -from tests.rest_app.models import SlugModel, FieldsetModel -from tests.gis_app.models import GeometryModel +from .rest_app.models import SlugModel, FieldsetModel from django.contrib.auth.models import User -from django.conf import settings class RestPostTestCase(APITestCase): @@ -13,54 +9,6 @@ def setUp(self): self.user = User.objects.create(username="testuser", is_superuser=True) self.client.force_authenticate(self.user) - @unittest.skipUnless(settings.WITH_GIS, "requires GIS") - def test_rest_geometry_post_geojson(self): - """ - Posting GeoJSON to a model with a geometry field should work. - """ - form = { - "name": "Geometry Test 1", - "geometry": json.dumps( - {"type": "Point", "coordinates": [-90, 44]} - ), - } - - # Test for expected response - response = self.client.post("/geometrymodels.json", form) - self.assertEqual( - response.status_code, status.HTTP_201_CREATED, response.data - ) - - # Double-check ORM model & geometry attribute - obj = GeometryModel.objects.get(id=response.data["id"]) - geom = obj.geometry - self.assertEqual(geom.srid, 4326) - self.assertEqual(geom.x, -90) - self.assertEqual(geom.y, 44) - - @unittest.skipUnless(settings.WITH_GIS, "requires GIS") - def test_rest_geometry_post_wkt(self): - """ - Posting WKT to a model with a geometry field should work. - """ - form = { - "name": "Geometry Test 2", - "geometry": "POINT(%s %s)" % (-97, 50), - } - - # Test for expected response - response = self.client.post("/geometrymodels.json", form) - self.assertEqual( - response.status_code, status.HTTP_201_CREATED, response.data - ) - - # Double-check ORM model & geometry attribute - obj = GeometryModel.objects.get(id=response.data["id"]) - geom = obj.geometry - self.assertEqual(geom.srid, 4326) - self.assertEqual(geom.x, -97) - self.assertEqual(geom.y, 50) - def test_rest_date_label_post(self): """ Posting to a model with a date should return a label and an ISO date diff --git a/tests/test_vector_tile.py b/tests/test_vector_tile.py deleted file mode 100644 index f17b7de..0000000 --- a/tests/test_vector_tile.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest -from .base import APITestCase -from django.conf import settings - - -class VectorTileTestCase(APITestCase): - @unittest.skipIf(settings.VARIANT == "postgis", "postgis supports tiles") - def test_no_tiles(self): - tile = self.client.get("/tiles/0/0/0.pbf") - self.assertEqual( - tile.content.decode(), - "Tile server not supported with this database engine.", - ) - - @unittest.skipUnless(settings.VARIANT == "postgis", "requires postgis") - def test_tiles(self): - from .gis_app.models import PointModel - - empty_tile = self.client.get("/tiles/0/0/0.pbf").content - self.assertEqual(b"", empty_tile) - - PointModel.objects.create(pk=1, geometry="POINT(34 -84)") - single_point = self.client.get("/tiles/0/0/0.pbf").content - self.assertEqual( - single_point, - b'\x1a\x1e\n\npointmodel\x12\x0b\x08\x01\x18\x01"\x05\t\x86&\x84>(\x80 x\x02', - ) diff --git a/wq/db/rest/maps.py b/wq/db/rest/maps.py index cf92aa0..cbfc510 100644 --- a/wq/db/rest/maps.py +++ b/wq/db/rest/maps.py @@ -131,10 +131,18 @@ def update_map_config(conf, pages): map_conf.pop("auto_layers", None) map_conf.pop("autoLayers", None) layers = map_conf.get("layers") or [] + reference_layers = pages.copy() if mode in ("list", "detail") and not layers: - layers += get_context_layers(conf, mode) + if conf.get("defer_geometry"): + if mode == "list" and supports_vector_tiles(): + layers += get_tile_layers({conf["name"]: conf}, None) + reference_layers.pop(conf["name"]) + else: + layers += get_geojson_url_layers(conf, mode) + else: + layers += get_context_layers(conf, mode) if supports_vector_tiles(): - layers += get_tile_layers(pages, mode) + layers += get_tile_layers(reference_layers, mode) map_conf["layers"] = layers return conf @@ -210,6 +218,33 @@ def get_context_layers(conf, mode): return layers +def get_geojson_url_layers(conf, mode): + if mode == "list": + url = "{{{rt}}}/" + conf["url"] + ".geojson" + else: + url = "{{{rt}}}/" + conf["url"] + "/{{{id}}}.geojson" + + layers = [] + for field in conf.get("geometry_fields") or []: + layer_conf = { + "name": get_geometry_label(conf, field, mode), + "type": "geojson", + "url": url, + "popup": conf["name"], + } + if mode == "list": + layer_conf["cluster"] = True # TODO: implement in @wq/map-gl + if conf.get("map_color"): + layer_conf["color"] = conf["map_color"] + # TODO: layer_conf["legend"] = ... + layers.append(layer_conf) + + # Only last geometry supported in GeoJSONRenderer + layers = layers[-1:] + + return layers + + def get_tile_layers(pages, mode): layers = [] for conf in sorted(pages.values(), key=layer_sort_key): diff --git a/wq/db/rest/routers.py b/wq/db/rest/routers.py index e053bba..e6af2b0 100644 --- a/wq/db/rest/routers.py +++ b/wq/db/rest/routers.py @@ -274,8 +274,24 @@ def get_queryset_for_model(self, model, request=None): qs = self._querysets[model] else: qs = model.objects.all() - if request and model in self._filters: - qs = self._filters[model](qs, request) + if request: + if model in self._filters: + qs = self._filters[model](qs, request) + config = self.get_model_config(model) or {} + renderer = getattr(request, "accepted_renderer", None) + if ( + config.get("defer_geometry") + and config.get("geometry_fields") + and renderer + and renderer.format != "geojson" + ): + qs = qs.defer( + *[ + field["name"] + for field in config["geometry_fields"] + if "." not in field["name"] + ] + ) return qs def get_cache_filter_for_model(self, model): diff --git a/wq/db/rest/serializers.py b/wq/db/rest/serializers.py index 6b6e07e..a7819e2 100644 --- a/wq/db/rest/serializers.py +++ b/wq/db/rest/serializers.py @@ -636,6 +636,7 @@ def get_fields(self, *args, **kwargs): fields = self.update_id_fields(fields) fields.update(self.get_label_fields(fields)) fields.update(self.get_nested_arrays(fields)) + exclude = set() def get_exclude(meta_name): @@ -649,8 +650,17 @@ def get_exclude(meta_name): if self.is_config: exclude |= get_exclude("config_exclude") - for field in list(exclude): - fields.pop(field, None) + if not self.is_config and not self.is_geojson: + defer_geometry = getattr(self.Meta, "wq_config", {}).get( + "defer_geometry", False + ) + if defer_geometry: + for field_name, field in fields.items(): + if isinstance(field, GeometryField): + exclude.add(field_name) + + for field_name in list(exclude): + fields.pop(field_name, None) return fields