From 6d4237a7f6a5ae62f9004807c169966ce88dfb0f Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 9 Jul 2024 01:01:35 -0400 Subject: [PATCH 01/74] use identification.url as about link on landing page (#1716) --- pygeoapi/api/__init__.py | 7 +++++++ pygeoapi/templates/landing_page.html | 9 +++++++++ tests/api/test_api.py | 17 +++++++++++------ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 8b64ba2d3..793b5969d 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -714,6 +714,13 @@ def landing_page(self, LOGGER.debug('Creating links') # TODO: put title text in config or translatable files? fcm['links'] = [{ + 'rel': 'about', + 'type': 'text/html', + 'title': l10n.translate( + self.config['metadata']['identification']['title'], + request.locale), + 'href': self.config['metadata']['identification']['url'] + }, { 'rel': request.get_linkrel(F_JSON), 'type': FORMAT_TYPES[F_JSON], 'title': l10n.translate('This document as JSON', request.locale), diff --git a/pygeoapi/templates/landing_page.html b/pygeoapi/templates/landing_page.html index e104d3dc3..e135ddb86 100644 --- a/pygeoapi/templates/landing_page.html +++ b/pygeoapi/templates/landing_page.html @@ -40,6 +40,15 @@

{{ config['metadata']['identification']['title'] }}

{% endif %} +
+
+ {% trans %}URL{% endtrans %} +
+
+ + {{ config['metadata']['identification']['url'] | truncate( 70 ) }} +
+
diff --git a/tests/api/test_api.py b/tests/api/test_api.py index c11afca28..6eda0295b 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -271,8 +271,9 @@ def test_apirules_active(config_with_rules, rules_api): assert response.status_code == 200 assert response.is_json links = response.json['links'] + assert links[0]['rel'] == 'about' assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + href.startswith(base_url) for href in (rel['href'] for rel in links[1:]) # noqa ) # Test Starlette @@ -303,8 +304,9 @@ def test_apirules_active(config_with_rules, rules_api): response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa assert response.status_code == 200 links = response.json()['links'] + assert links[0]['rel'] == 'about' assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + href.startswith(base_url) for href in (rel['href'] for rel in links[1:]) # noqa ) @@ -510,14 +512,17 @@ def test_root(config, api_): assert isinstance(root, dict) assert 'links' in root - assert root['links'][0]['rel'] == 'self' - assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] - assert root['links'][0]['href'].endswith('?f=json') + assert root['links'][0]['rel'] == 'about' + assert root['links'][0]['type'] == 'text/html' + assert root['links'][0]['href'] == 'http://example.org' + assert root['links'][1]['rel'] == 'self' + assert root['links'][1]['type'] == FORMAT_TYPES[F_JSON] + assert root['links'][1]['href'].endswith('?f=json') assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' for link in root['links']) assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' for link in root['links']) - assert len(root['links']) == 11 + assert len(root['links']) == 12 assert 'title' in root assert root['title'] == 'pygeoapi default instance' assert 'description' in root From 0169e433297f6e5a6c630e3937e3d5b43515d1cc Mon Sep 17 00:00:00 2001 From: francescoingv <9592487+francescoingv@users.noreply.github.com> Date: Fri, 12 Jul 2024 11:45:24 +0200 Subject: [PATCH 02/74] Update test_tinydb_manager_for_parallel_requests.py (#1720) * Update test_tinydb_manager_for_parallel_requests.py Updated the call to execute_process and now checks the expected number of process results are present. * Update test_tinydb_manager_for_parallel_requests.py --------- Co-authored-by: FrancescoIngv Co-authored-by: Tom Kralidis --- tests/test_tinydb_manager_for_parallel_requests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_tinydb_manager_for_parallel_requests.py b/tests/test_tinydb_manager_for_parallel_requests.py index a1da59ce9..393cea7cd 100644 --- a/tests/test_tinydb_manager_for_parallel_requests.py +++ b/tests/test_tinydb_manager_for_parallel_requests.py @@ -39,6 +39,7 @@ from werkzeug.test import create_environ from pygeoapi.api import API, APIRequest +import pygeoapi.api.processes as processes_api from pygeoapi.util import yaml_load from .util import get_test_file_path @@ -61,7 +62,8 @@ def api_(config, openapi): def _execute_process(api, request, process_id, index, processes_out): - headers, http_status, response = api.execute_process(request, process_id) + headers, http_status, response = processes_api.execute_process( + api, request, process_id) processes_out[index] = {"headers": headers, "http_status": http_status, "response": response} @@ -112,6 +114,7 @@ def test_async_hello_world_process_parallel(api_, config): # Test if jobs are registered and run correctly db = TinyDB(index_name) query = Query() + assert len(processes_out.values()) == NUM_PROCS for process_out in processes_out.values(): try: assert process_out['http_status'] == 200 From e8ce02167fd670bb515f8216d699a3e9abca21d4 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 15 Jul 2024 21:17:51 -0400 Subject: [PATCH 03/74] OAProc: fix response type on delete_job (#1721) (#1726) --- pygeoapi/api/processes.py | 7 +++---- tests/api/test_processes.py | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 8ba759970..3eb9ea695 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -512,9 +512,7 @@ def get_job_result(api: API, request: APIRequest, return headers, HTTPStatus.OK, content -def delete_job( - api: API, request: APIRequest, job_id -) -> Tuple[dict, int, str]: +def delete_job(api: API, request: APIRequest, job_id) -> Tuple[dict, int, str]: """ Delete a process job @@ -522,6 +520,7 @@ def delete_job( :returns: tuple of headers, status code, content """ + response_headers = request.get_response_headers( SYSTEM_LOCALE, **api.api_headers) try: @@ -555,7 +554,7 @@ def delete_job( ) LOGGER.info(response) # TODO: this response does not have any headers - return {}, http_status, response + return {}, http_status, to_json(response, api.pretty_print) def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa diff --git a/tests/api/test_processes.py b/tests/api/test_processes.py index a33972889..bfd5ca98f 100644 --- a/tests/api/test_processes.py +++ b/tests/api/test_processes.py @@ -386,7 +386,10 @@ def test_delete_job(api_): job_id = _execute_a_job(api_) rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + data = json.loads(response) + assert code == HTTPStatus.OK + assert data['message'] == 'Job dismissed' rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) assert code == HTTPStatus.NOT_FOUND From d9adbbd817cc8651cc8f9e56ca096c63921d8ff2 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 16 Jul 2024 11:40:50 -0400 Subject: [PATCH 04/74] add support for TinyDB feature provider (#1724) * add support for TinyDB as a feature provider (#1723) * add tests * update CITE setup * add/update docs * update sample CITE config * update copyright year * reorder imports * fix formatting * delineate features/records via overloading * remove now unused test data * move CITE data to tests/data * fix ref * remove HYDAT from GitHub Actions * remove print statement --- .github/workflows/main.yml | 1 - docs/source/data-publishing/ogcapi-edr.rst | 5 + .../data-publishing/ogcapi-features.rst | 160 ++- docs/source/data-publishing/ogcapi-maps.rst | 12 +- pygeoapi/plugin.py | 3 +- pygeoapi/provider/tinydb_.py | 123 +- tests/cite/README.md | 3 +- .../canada-hydat-daily-mean-02HC003.geojson | 1240 ----------------- tests/cite/cite.config.yml | 4 +- tests/data/README.md | 5 + .../canada-hydat-daily-mean-02HC003.tinydb | 1 + tests/test_tinydb_provider.py | 191 +++ 12 files changed, 403 insertions(+), 1345 deletions(-) delete mode 100644 tests/cite/canada-hydat-daily-mean-02HC003.geojson create mode 100644 tests/data/canada-hydat-daily-mean-02HC003.tinydb create mode 100644 tests/test_tinydb_provider.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bfbd6bf9e..224c07247 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,7 +100,6 @@ jobs: - name: setup test data ⚙️ run: | python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid - python3 tests/load_es_data.py tests/cite/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql diff --git a/docs/source/data-publishing/ogcapi-edr.rst b/docs/source/data-publishing/ogcapi-edr.rst index 1014b40d3..2ad82a166 100644 --- a/docs/source/data-publishing/ogcapi-edr.rst +++ b/docs/source/data-publishing/ogcapi-edr.rst @@ -86,14 +86,19 @@ Data access examples -------------------- * list all collections + * http://localhost:5000/collections * overview of dataset + * http://localhost:5000/collections/foo * dataset position query + * http://localhost:5000/collections/foo/position?coords=POINT(-75%2045) * dataset position query for a specific parameter + * http://localhost:5000/collections/foo/position?coords=POINT(-75%2045)¶meter-name=SST * dataset position query for a specific parameter and time step + * http://localhost:5000/collections/foo/position?coords=POINT(-75%2045)¶meter-name=SST&datetime=2000-01-16 diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index a8e2c9a55..32e657de7 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -31,6 +31,7 @@ parameters. `SQLiteGPKG`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,✅ `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ `Socrata`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ + `TinyDB`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,❌ .. note:: @@ -70,20 +71,6 @@ definition. - http://www.opengis.net/def/crs/EPSG/0/4326 storage_crs: http://www.opengis.net/def/crs/EPSG/0/28992 - -GeoJSON -^^^^^^^ - -To publish a GeoJSON file, the file must be a valid GeoJSON FeatureCollection. - -.. code-block:: yaml - - providers: - - type: feature - name: GeoJSON - data: tests/data/file.json - id_field: id - .. _Elasticsearch: Elasticsearch @@ -125,11 +112,35 @@ The ES provider also has the support for the CQL queries as indicated in the tab .. seealso:: :ref:`cql` for more details on how to use Common Query Language (CQL) to filter the collection with specific queries. +.. _ERDDAP Tabledap Service: + +ERDDAP Tabledap Service +^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + Requires Python package `requests`_ + +To publish from an ERDDAP `Tabledap`_ service, the following are required in your index: + +.. code-block:: yaml + + providers: + - type: feature + name: ERDDAPTabledap + data: http://osmc.noaa.gov/erddap/tabledap/OSMC_Points + id_field: PLATFORM_CODE + time_field: time + options: + filters: "¶meter=\"SLP\"&platform!=\"C-MAN%20WEATHER%20STATIONS\"&platform!=\"TIDE GAUGE STATIONS (GENERIC)\"" + max_age_hours: 12 + +.. note:: + If the ``datetime`` parameter is passed by the client, this overrides the ``options.max_age_hours`` setting. ESRI Feature Service ^^^^^^^^^^^^^^^^^^^^ -To publish an `ESRI Feature Service`_ or `ESRI Map Service`_ specify the URL for the service layer in the ``data`` field. +To publish an ESRI `Feature Service`_ or `Map Service`_ specify the URL for the service layer in the ``data`` field. * ``id_field`` will often be ``OBJECTID``, ``objectid``, or ``FID``. * If the map or feature service is not shared publicly, the ``username`` and ``password`` fields can be set in the @@ -147,6 +158,45 @@ To publish an `ESRI Feature Service`_ or `ESRI Map Service`_ specify the URL for username: username # Optional ArcGIS username password: password # Optional ArcGIS password +GeoJSON +^^^^^^^ + +To publish a GeoJSON file, the file must be a valid GeoJSON FeatureCollection. + +.. code-block:: yaml + + providers: + - type: feature + name: GeoJSON + data: tests/data/file.json + id_field: id + +MongoDB +^^^^^^^ + +.. note:: + Requires Python package pymongo + +.. note:: + Mongo 5 or greater is supported. + +MongoDB (`website `_) is a powerful and versatile NoSQL database that provides numerous advantages, making it a preferred choice for many applications. One of the main reasons to use MongoDB is its ability to handle large volumes of unstructured data, making it ideal for managing diverse data types such as text, geospatial, and multimedia data. Additionally, MongoDB's flexible document model allows for easy schema evolution, enabling developers to iterate quickly and adapt to changing requirements. + +`MongoDB GeoJSON `_ support is available, thus a GeoJSON file can be added to MongoDB using following command + +`mongoimport --db test -c points --file "path/to/file.geojson" --jsonArray` + +Here `test` is the name of database , `points` is the target collection name. + +* each document must be a GeoJSON Feature, with a valid geometry. + +.. code-block:: yaml + + providers: + - type: feature + name: MongoDB + data: mongodb://localhost:27017/testdb + collection: testplaces OGR ^^^ @@ -249,34 +299,6 @@ The OGR provider requires a recent (3+) version of GDAL to be installed. The `crs` query parameter is used as follows: e.g. ``http://localhost:5000/collections/foo/items?crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992``. - -MongoDB -^^^^^^^ - -.. note:: - Requires Python package pymongo - -.. note:: - Mongo 5 or greater is supported. - -`MongoDB `_ is a powerful and versatile NoSQL database that provides numerous advantages, making it a preferred choice for many applications. One of the main reasons to use MongoDB is its ability to handle large volumes of unstructured data, making it ideal for managing diverse data types such as text, geospatial, and multimedia data. Additionally, MongoDB's flexible document model allows for easy schema evolution, enabling developers to iterate quickly and adapt to changing requirements. - -`GeoJSON `_ support is available officially by MongoDB , thus a GeoJSON file can be added to MongoDB using following command - -`mongoimport --db test -c points --file "path/to/file.geojson" --jsonArray` - -Here `test` is the name of database , `points` is the target collection name. - -* each document must be a GeoJSON Feature, with a valid geometry. - -.. code-block:: yaml - - providers: - - type: feature - name: MongoDB - data: mongodb://localhost:27017/testdb - collection: testplaces - .. _Oracle: Oracle @@ -365,7 +387,7 @@ configure mandatory properties. When this is activated, the provider throws an e is not in the query uri. Extra properties -"""""""""""""""""""" +"""""""""""""""" .. code-block:: yaml providers: @@ -388,7 +410,7 @@ Extra properties is a list of strings which are added as fields for data retriev can be used to return expressions computed by the database. Session Pooling -"""""""""""""""" +""""""""""""""" Configured using environment variables. @@ -593,31 +615,25 @@ To publish a `Socrata Open Data API (SODA)`_ endpoint, pygeoapi heavily relies o token: my_token # Optional app token -.. _ERDDAP Tabledap Service: - -ERDDAP Tabledap Service -^^^^^^^^^^^^^^^^^^^^^^^ +TinyDB +^^^^^^ .. note:: - Requires Python package `requests`_ + Requires Python package tinydb -To publish from an ERDDAP `Tabledap`_ service, the following are required in your index: +To publish a TinyDB (`see website `_) index, the following are required in your index: + +* indexes must be documents of valid GeoJSON Features .. code-block:: yaml providers: - type: feature - name: ERDDAPTabledap - data: http://osmc.noaa.gov/erddap/tabledap/OSMC_Points - id_field: PLATFORM_CODE - time_field: time - options: - filters: "¶meter=\"SLP\"&platform!=\"C-MAN%20WEATHER%20STATIONS\"&platform!=\"TIDE GAUGE STATIONS (GENERIC)\"" - max_age_hours: 12 - - -.. note:: - If the ``datetime`` parameter is passed by the client, this overrides the ``options.max_age_hours`` setting. + editable: true|false # optional, default is false + name: TinyDB + data: /path/to/file.db + id_field: identifier + time_field: datetimefield Controlling the order of properties ----------------------------------- @@ -636,36 +652,52 @@ Data access examples -------------------- * list all collections + * http://localhost:5000/collections * overview of dataset + * http://localhost:5000/collections/foo * queryables + * http://localhost:5000/collections/foo/queryables * browse features + * http://localhost:5000/collections/foo/items * paging + * http://localhost:5000/collections/foo/items?offset=10&limit=10 * CSV outputs + * http://localhost:5000/collections/foo/items?f=csv * query features (spatial) + * http://localhost:5000/collections/foo/items?bbox=-180,-90,180,90 * query features (spatial with bbox-crs) + * http://localhost:5000/collections/foo/items?bbox=120000,450000,130000,460000&bbox-crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992 * query features (attribute) + * http://localhost:5000/collections/foo/items?propertyname=foo * query features (temporal) + * http://localhost:5000/collections/foo/items?datetime=2020-04-10T14:11:00Z * query features (temporal) and sort ascending by a property (if no +/- indicated, + is assumed) + * http://localhost:5000/collections/foo/items?datetime=2020-04-10T14:11:00Z&sortby=+datetime * query features (temporal) and sort descending by a property + * http://localhost:5000/collections/foo/items?datetime=2020-04-10T14:11:00Z&sortby=-datetime * query features in a given (and supported) CRS + * http://localhost:5000/collections/foo/items?crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F32633 * query features in a given bounding BBOX and return in given CRS + * http://localhost:5000/collections/foo/items?bbox=120000,450000,130000,460000&bbox-crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F32633 * fetch a specific feature + * http://localhost:5000/collections/foo/items/123 * fetch a specific feature in a given (and supported) CRS + * http://localhost:5000/collections/foo/items/123?crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F32633 .. note:: @@ -682,8 +714,8 @@ Data access examples provider `id_field` values support slashes (i.e. ``my/cool/identifier``). The client request would then be responsible for encoding the identifier accordingly (i.e. ``http://localhost:5000/collections/foo/items/my%2Fcool%2Fidentifier``) -.. _`ESRI Feature Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-feature-service-.htm -.. _`ESRI Map Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-map-service.htm +.. _`Feature Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-feature-service-.htm +.. _`Map Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-map-service.htm .. _`Google Cloud SQL`: https://cloud.google.com/sql .. _`OGC API - Features`: https://www.ogc.org/standards/ogcapi-features .. _`Socrata Open Data API (SODA)`: https://dev.socrata.com diff --git a/docs/source/data-publishing/ogcapi-maps.rst b/docs/source/data-publishing/ogcapi-maps.rst index 8e3dc7c04..30e12b1d1 100644 --- a/docs/source/data-publishing/ogcapi-maps.rst +++ b/docs/source/data-publishing/ogcapi-maps.rst @@ -18,7 +18,7 @@ parameters. :header: Provider, bbox, width/height :align: left - `MapScript`,✅,✅ + `MapScript`_,✅,✅ `WMSFacade`_,✅,✅ @@ -30,7 +30,7 @@ Connection examples MapScript ^^^^^^^^^ -`MapScript`_ is MapServer's scripting interface to map rendering. +MapScript (`see website`_) is MapServer's scripting interface to map rendering. To publish a map via MapScript, the path to data is required, as well as the layer type (`options.type`). To style the data, set `options.style`. If @@ -86,17 +86,23 @@ Data visualization examples --------------------------- * list all collections + * http://localhost:5000/collections * overview of dataset + * http://localhost:5000/collections/foo * map (default format) + * http://localhost:5000/collections/foo/map * map with bbox subset + * http://localhost:5000/collections/foo/map?bbox=-142,42,-52,84 * map with bbox and temporal subset + * http://localhost:5000/collections/foo/map?bbox=-142,42,-52,84&datetime=2020-04-10T14:11:00Z * map with bbox and bbox-crs + * http://localhost:5000/collections/foo/map?bbox-crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F3857&bbox=4.022369384765626%2C50.690447870569436%2C4.681549072265626%2C51.00260125274477&width=800&height=600&transparent .. _`OGC API - Maps`: https://ogcapi.ogc.org/maps -.. _`MapScript`: https://mapserver.org/mapscript/index.html +.. _`see website`: https://mapserver.org/mapscript/index.html diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index 8e922f2a9..7e4225562 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -60,6 +60,7 @@ 'SensorThings': 'pygeoapi.provider.sensorthings.SensorThingsProvider', 'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider', 'Socrata': 'pygeoapi.provider.socrata.SODAServiceProvider', + 'TinyDB': 'pygeoapi.provider.tinydb_.TinyDBProvider', 'TinyDBCatalogue': 'pygeoapi.provider.tinydb_.TinyDBCatalogueProvider', 'WMSFacade': 'pygeoapi.provider.wms_facade.WMSFacadeProvider', 'WMTSFacade': 'pygeoapi.provider.wmts_facade.WMTSFacadeProvider', diff --git a/pygeoapi/provider/tinydb_.py b/pygeoapi/provider/tinydb_.py index 3bac4c059..e46730aba 100644 --- a/pygeoapi/provider/tinydb_.py +++ b/pygeoapi/provider/tinydb_.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -32,17 +32,19 @@ import os import uuid +from dateutil.parser import parse as parse_date from shapely.geometry import shape from tinydb import TinyDB, Query, where from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderItemNotFoundError) +from pygeoapi.util import get_typed_value LOGGER = logging.getLogger(__name__) -class TinyDBCatalogueProvider(BaseProvider): - """TinyDB Catalogue Provider""" +class TinyDBProvider(BaseProvider): + """TinyDB Provider""" def __init__(self, provider_def): """ @@ -50,15 +52,13 @@ def __init__(self, provider_def): :param provider_def: provider definition - :returns: pygeoapi.provider.tinydb_.TinyDBCatalogueProvider + :returns: pygeoapi.provider.tinydb_.TinyDBProvider """ - self.excludes = [ - '_metadata-anytext', - ] - super().__init__(provider_def) + self._excludes = [] + LOGGER.debug(f'Connecting to TinyDB db at {self.data}') if not os.path.exists(self.data): @@ -91,11 +91,28 @@ def get_fields(self): LOGGER.debug(err) return fields - for p in r['properties'].keys(): - if p not in self.excludes: - fields[p] = {'type': 'string'} + for key, value in r['properties'].items(): + if key not in self._excludes: + typed_value = get_typed_value(str(value)) + if isinstance(typed_value, float): + typed_value_type = 'number' + elif isinstance(typed_value, int): + typed_value_type = 'integer' + else: + typed_value_type = 'string' - fields['q'] = {'type': 'string'} + fields[key] = {'type': typed_value_type} + + try: + LOGGER.debug('Attempting to detect date types') + _ = parse_date(value) + if len(value) > 11: + fields[key]['format'] = 'date-time' + else: + fields[key]['format'] = 'date' + except Exception: + LOGGER.debug('No date types detected') + pass return fields @@ -164,11 +181,9 @@ def query(self, offset=0, limit=10, resulttype='results', if properties: LOGGER.debug('processing properties') for prop in properties: - QUERY.append(f"(Q.properties['{prop[0]}']=='{prop[1]}')") + QUERY.append(f"(Q.properties['{prop[0]}']=={prop[1]})") - if q is not None: - for t in q.split(): - QUERY.append(f"(Q.properties['_metadata-anytext'].search('{t}', flags=re.IGNORECASE))") # noqa + QUERY = self._add_search_query(QUERY, q) QUERY_STRING = '&'.join(QUERY) LOGGER.debug(f'QUERY_STRING: {QUERY_STRING}') @@ -188,7 +203,7 @@ def query(self, offset=0, limit=10, resulttype='results', return feature_collection for r in results: - for e in self.excludes: + for e in self._excludes: try: del r['properties'][e] except KeyError: @@ -235,7 +250,7 @@ def get(self, identifier, **kwargs): if record is None: raise ProviderItemNotFoundError('record does not exist') - for e in self.excludes: + for e in self._excludes: try: del record['properties'][e] except KeyError: @@ -259,14 +274,7 @@ def create(self, item): identifier = str(uuid.uuid4()) json_data["id"] = identifier - try: - json_data['properties']['_metadata-anytext'] = ''.join([ - json_data['properties']['title'], - json_data['properties']['description'] - ]) - except KeyError: - LOGGER.debug('Missing title and description') - json_data['properties']['_metadata_anytext'] = '' + json_data = self._add_extra_fields(json_data) LOGGER.debug(f'Inserting data with identifier {identifier}') result = self.db.insert(json_data) @@ -306,17 +314,68 @@ def delete(self, identifier): return True - def _bbox(input_bbox, record_bbox): + def _add_extra_fields(self, json_data: dict) -> dict: """ - Test whether one bbox intersects another + Helper function to add extra fields to an item payload - :param input_bbox: `list` of minx,miny,maxx,maxy - :param record_bbox: `list` of minx,miny,maxx,maxy + :param json_data: `dict` of JSON data - :returns: `bool` of result + :returns: `dict` of updated JSON data """ - return True + return json_data + + def _add_search_query(self, query: list, search_term: str = None) -> str: + """ + Helper function to add extra query predicates + + :param query: `list` of query predicates + :param search_term: `str` of search term + + :returns: `list` of updated query predicates + """ + + return query + + def __repr__(self): + return f' {self.data}' + + +class TinyDBCatalogueProvider(TinyDBProvider): + """TinyDB Catalogue Provider""" + + def __init__(self, provider_def): + super().__init__(provider_def) + + self._excludes = ['_metadata-anytext'] + + def get_fields(self): + fields = super().get_fields() + + fields['q'] = {'type': 'string'} + + return fields + + def _add_extra_fields(self, json_data: dict) -> dict: + LOGGER.debug('Adding catalogue anytext property') + try: + json_data['properties']['_metadata-anytext'] = ''.join([ + json_data['properties']['title'], + json_data['properties']['description'] + ]) + except KeyError: + LOGGER.debug('Missing title and description') + json_data['properties']['_metadata_anytext'] = '' + + return json_data + + def _add_search_query(self, query: list, search_term: str = None) -> str: + if search_term is not None: + LOGGER.debug('catalogue q= query') + for t in search_term.split(): + query.append(f"(Q.properties['_metadata-anytext'].search('{t}', flags=re.IGNORECASE))") # noqa + + return query def __repr__(self): return f' {self.data}' diff --git a/tests/cite/README.md b/tests/cite/README.md index 4341e8cbf..f8142f9a3 100644 --- a/tests/cite/README.md +++ b/tests/cite/README.md @@ -6,7 +6,7 @@ ## Test data ### OGC API - Features -Test data used is a subset of the [Canadian National Water Data Archive](https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html) as extracted from the [MSC GeoMet OGC API](https://eccc-msc.github.io/open-data/msc-geomet/web-services_en/#ogc-api-features) service. +Test data used is in `tests/data/canada-hydat-daily-mean-02HC003.tinydb` ### OGC API - Processes The `hello-world` test process that is provided with pygeoapi by default is used. @@ -23,7 +23,6 @@ rm -f /tmp/pygeoapi-process-manager.db* pip3 install gunicorn cd tests/cite . cite.env -python3 ../load_es_data.py ./canada-hydat-daily-mean-02hc003.geojson IDENTIFIER pygeoapi openapi generate $PYGEOAPI_CONFIG --output-file $PYGEOAPI_OPENAPI gunicorn pygeoapi.flask_app:APP -b 0.0.0.0:5001 --access-logfile '-' ``` diff --git a/tests/cite/canada-hydat-daily-mean-02HC003.geojson b/tests/cite/canada-hydat-daily-mean-02HC003.geojson deleted file mode 100644 index 13f534411..000000000 --- a/tests/cite/canada-hydat-daily-mean-02HC003.geojson +++ /dev/null @@ -1,1240 +0,0 @@ -{ - "features": [ - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-03", - "properties": { - "DATE": "1975-10-03", - "FLOW": 2.039999961853028, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-03", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-02", - "properties": { - "DATE": "1975-10-02", - "FLOW": 2.059999942779541, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-02", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-01", - "properties": { - "DATE": "1975-10-01", - "FLOW": 2.140000104904175, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-01", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-07", - "properties": { - "DATE": "1975-10-07", - "FLOW": 1.940000057220459, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-07", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-06", - "properties": { - "DATE": "1975-10-06", - "FLOW": 1.9600000381469729, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-06", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-05", - "properties": { - "DATE": "1975-10-05", - "FLOW": 1.909999966621399, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-05", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1975-10-04", - "properties": { - "DATE": "1975-10-04", - "FLOW": 2.0, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1975-10-04", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1983-12-31", - "properties": { - "DATE": "1983-12-31", - "FLOW": 2.5999999046325684, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1983-12-31", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1983-12-30", - "properties": { - "DATE": "1983-12-30", - "FLOW": 2.650000095367432, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1983-12-30", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.2017-05-23", - "properties": { - "DATE": "2017-05-23", - "FLOW": 7.079999923706055, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.2017-05-23", - "LEVEL": 2.3320000171661377, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-09", - "properties": { - "DATE": "1955-09-09", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-09", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-08", - "properties": { - "DATE": "1955-09-08", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-08", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-07", - "properties": { - "DATE": "1955-09-07", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-07", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-06", - "properties": { - "DATE": "1955-09-06", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-06", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-05", - "properties": { - "DATE": "1955-09-05", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-05", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-04", - "properties": { - "DATE": "1955-09-04", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-04", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-03", - "properties": { - "DATE": "1955-09-03", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-03", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-02", - "properties": { - "DATE": "1955-09-02", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-02", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1955-09-01", - "properties": { - "DATE": "1955-09-01", - "FLOW": 1.4700000286102295, - "FLOW_SYMBOL_EN": "Estimated", - "FLOW_SYMBOL_FR": "Estim\u00e9", - "IDENTIFIER": "02HC003.1955-09-01", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-22", - "properties": { - "DATE": "1979-11-22", - "FLOW": 5.230000019073486, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-22", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-23", - "properties": { - "DATE": "1979-11-23", - "FLOW": 9.260000228881836, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-23", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-20", - "properties": { - "DATE": "1979-11-20", - "FLOW": 3.390000104904175, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-20", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-21", - "properties": { - "DATE": "1979-11-21", - "FLOW": 3.3299999237060547, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-21", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-26", - "properties": { - "DATE": "1979-11-26", - "FLOW": 25.799999237060547, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-26", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-27", - "properties": { - "DATE": "1979-11-27", - "FLOW": 24.399999618530277, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-27", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-24", - "properties": { - "DATE": "1979-11-24", - "FLOW": 20.5, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-24", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-25", - "properties": { - "DATE": "1979-11-25", - "FLOW": 17.899999618530273, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-25", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-28", - "properties": { - "DATE": "1979-11-28", - "FLOW": 18.600000381469727, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-28", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1979-11-29", - "properties": { - "DATE": "1979-11-29", - "FLOW": 17.5, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1979-11-29", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-09", - "properties": { - "DATE": "1967-02-09", - "FLOW": 4.53000020980835, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-09", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-08", - "properties": { - "DATE": "1967-02-08", - "FLOW": 5.659999847412109, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-08", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-01", - "properties": { - "DATE": "1967-02-01", - "FLOW": 3.259999990463257, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-01", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-03", - "properties": { - "DATE": "1967-02-03", - "FLOW": 3.109999895095825, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-03", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-02", - "properties": { - "DATE": "1967-02-02", - "FLOW": 2.8299999237060547, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-02", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-05", - "properties": { - "DATE": "1967-02-05", - "FLOW": 3.9600000381469727, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-05", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-04", - "properties": { - "DATE": "1967-02-04", - "FLOW": 3.400000095367432, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-04", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-07", - "properties": { - "DATE": "1967-02-07", - "FLOW": 6.369999885559082, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-07", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1967-02-06", - "properties": { - "DATE": "1967-02-06", - "FLOW": 4.53000020980835, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1967-02-06", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1995-01-29", - "properties": { - "DATE": "1995-01-29", - "FLOW": 3.309999942779541, - "FLOW_SYMBOL_EN": "Ice Conditions", - "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", - "IDENTIFIER": "02HC003.1995-01-29", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-13", - "properties": { - "DATE": "1987-03-13", - "FLOW": 9.720000267028809, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-13", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.2017-05-27", - "properties": { - "DATE": "2017-05-27", - "FLOW": 17.299999237060547, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.2017-05-27", - "LEVEL": 2.5420000553131104, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-12", - "properties": { - "DATE": "1987-03-12", - "FLOW": 12.600000381469728, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-12", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-11", - "properties": { - "DATE": "1987-03-11", - "FLOW": 16.200000762939453, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-11", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-10", - "properties": { - "DATE": "1987-03-10", - "FLOW": 29.100000381469727, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-10", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-17", - "properties": { - "DATE": "1987-03-17", - "FLOW": 7.440000057220459, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-17", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-16", - "properties": { - "DATE": "1987-03-16", - "FLOW": 6.989999771118164, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-16", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.2017-05-26", - "properties": { - "DATE": "2017-05-26", - "FLOW": 30.399999618530277, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.2017-05-26", - "LEVEL": 2.691999912261963, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1987-03-15", - "properties": { - "DATE": "1987-03-15", - "FLOW": 7.210000038146973, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1987-03-15", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.1983-11-17", - "properties": { - "DATE": "1983-11-17", - "FLOW": 13.0, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.1983-11-17", - "LEVEL": null, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -79.52039337158203, - 43.69894027709961 - ], - "type": "Point" - }, - "id": "02HC003.2014-07-07", - "properties": { - "DATE": "2014-07-07", - "FLOW": 6.110000133514404, - "FLOW_SYMBOL_EN": null, - "FLOW_SYMBOL_FR": null, - "IDENTIFIER": "02HC003.2014-07-07", - "LEVEL": 2.2950000762939453, - "LEVEL_SYMBOL_EN": null, - "LEVEL_SYMBOL_FR": null, - "PROV_TERR_STATE_LOC": "ON", - "STATION_NAME": "HUMBER RIVER AT WESTON", - "STATION_NUMBER": "02HC003" - }, - "type": "Feature" - } - ], - "links": [ - { - "href": "https://geo.weather.gc.ca/geomet/features/collections/hydrometric-daily-mean/items?f=json", - "rel": "self", - "title": "This document as GeoJSON", - "type": "application/geo+json" - }, - { - "href": "https://geo.weather.gc.ca/geomet/features/collections/hydrometric-daily-mean/items?f=html", - "rel": "alternate", - "title": "This document as HTML", - "type": "text/html" - }, - { - "href": "https://geo.weather.gc.ca/geomet/features/collections/hydrometric-daily-mean/items/?offset=0", - "rel": "prev", - "title": "items (prev)", - "type": "application/geo+json" - }, - { - "href": "https://geo.weather.gc.ca/geomet/features/collections/hydrometric-daily-mean/items/?offset=500", - "rel": "next", - "title": "items (next)", - "type": "application/geo+json" - }, - { - "href": "https://geo.weather.gc.ca/geomet/features/collections/hydrometric-daily-mean", - "rel": "collection", - "title": "Daily Mean of Water Level or Flow", - "type": "application/json" - } - ], - "numberMatched": 26516, - "numberReturned": 50, - "timeStamp": "2019-12-23T14:08:36.974150Z", - "type": "FeatureCollection" -} diff --git a/tests/cite/cite.config.yml b/tests/cite/cite.config.yml index cc0eb7344..66362bd96 100644 --- a/tests/cite/cite.config.yml +++ b/tests/cite/cite.config.yml @@ -99,8 +99,8 @@ resources: providers: - type: feature - name: Elasticsearch - data: http://localhost:9200/canada-hydat-daily-mean-02hc003 + name: TinyDB + data: ../data/canada-hydat-daily-mean-02hc003.tinydb id_field: IDENTIFIER time_field: DATE diff --git a/tests/data/README.md b/tests/data/README.md index 3cebdae30..96b2fed8f 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -101,3 +101,8 @@ This directory provides test data to demonstrate functionality. - Notes - `items.geojson` tests pygeoapi's capability to serialize all geometry types for individual collection items in [JSON-LD formats](https://docs.pygeoapi.io/en/latest/configuration.html#linked-data), including GeoSPARQL WKT and schema.org/geo - The features represent the range of GeoJSON geometry types, instead of real locations. Additionally, each feature has a uri defined in the properties block + +### `canada-hydat-daily-mean-02HC003.tinydb` +- source: [Canadian National Water Data Archive](https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html) as extracted from the [MSC GeoMet OGC API](https://eccc-msc.github.io/open-data/msc-geomet/web-services_en/#ogc-api-features) service +- URL: https://www.canada.ca/en/environment-climate-change/services/water-overview/quantity/monitoring/survey/data-products-services/national-archive-hydat.html +- License: https://eccc-msc.github.io/open-data/licence/readme_en diff --git a/tests/data/canada-hydat-daily-mean-02HC003.tinydb b/tests/data/canada-hydat-daily-mean-02HC003.tinydb new file mode 100644 index 000000000..f37010195 --- /dev/null +++ b/tests/data/canada-hydat-daily-mean-02HC003.tinydb @@ -0,0 +1 @@ +{"_default": {"1": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-03", "properties": {"DATE": "1975-10-03", "FLOW": 2.039999961853028, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-03", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "2": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-02", "properties": {"DATE": "1975-10-02", "FLOW": 2.059999942779541, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-02", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "3": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-01", "properties": {"DATE": "1975-10-01", "FLOW": 2.140000104904175, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-01", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "4": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-07", "properties": {"DATE": "1975-10-07", "FLOW": 1.940000057220459, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-07", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "5": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-06", "properties": {"DATE": "1975-10-06", "FLOW": 1.9600000381469729, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-06", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "6": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-05", "properties": {"DATE": "1975-10-05", "FLOW": 1.909999966621399, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-05", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "7": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1975-10-04", "properties": {"DATE": "1975-10-04", "FLOW": 2.0, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1975-10-04", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "8": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1983-12-31", "properties": {"DATE": "1983-12-31", "FLOW": 2.5999999046325684, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1983-12-31", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "9": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1983-12-30", "properties": {"DATE": "1983-12-30", "FLOW": 2.650000095367432, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1983-12-30", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "10": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.2017-05-23", "properties": {"DATE": "2017-05-23", "FLOW": 7.079999923706055, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.2017-05-23", "LEVEL": 2.3320000171661377, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "11": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-09", "properties": {"DATE": "1955-09-09", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-09", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "12": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-08", "properties": {"DATE": "1955-09-08", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-08", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "13": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-07", "properties": {"DATE": "1955-09-07", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-07", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "14": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-06", "properties": {"DATE": "1955-09-06", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-06", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "15": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-05", "properties": {"DATE": "1955-09-05", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-05", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "16": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-04", "properties": {"DATE": "1955-09-04", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-04", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "17": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-03", "properties": {"DATE": "1955-09-03", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-03", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "18": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-02", "properties": {"DATE": "1955-09-02", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-02", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "19": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1955-09-01", "properties": {"DATE": "1955-09-01", "FLOW": 1.4700000286102295, "FLOW_SYMBOL_EN": "Estimated", "FLOW_SYMBOL_FR": "Estim\u00e9", "IDENTIFIER": "02HC003.1955-09-01", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "20": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-22", "properties": {"DATE": "1979-11-22", "FLOW": 5.230000019073486, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-22", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "21": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-23", "properties": {"DATE": "1979-11-23", "FLOW": 9.260000228881836, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-23", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "22": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-20", "properties": {"DATE": "1979-11-20", "FLOW": 3.390000104904175, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-20", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "23": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-21", "properties": {"DATE": "1979-11-21", "FLOW": 3.3299999237060547, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-21", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "24": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-26", "properties": {"DATE": "1979-11-26", "FLOW": 25.799999237060547, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-26", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "25": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-27", "properties": {"DATE": "1979-11-27", "FLOW": 24.399999618530277, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-27", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "26": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-24", "properties": {"DATE": "1979-11-24", "FLOW": 20.5, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-24", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "27": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-25", "properties": {"DATE": "1979-11-25", "FLOW": 17.899999618530273, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-25", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "28": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-28", "properties": {"DATE": "1979-11-28", "FLOW": 18.600000381469727, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-28", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "29": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1979-11-29", "properties": {"DATE": "1979-11-29", "FLOW": 17.5, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1979-11-29", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "30": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-09", "properties": {"DATE": "1967-02-09", "FLOW": 4.53000020980835, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-09", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "31": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-08", "properties": {"DATE": "1967-02-08", "FLOW": 5.659999847412109, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-08", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "32": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-01", "properties": {"DATE": "1967-02-01", "FLOW": 3.259999990463257, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-01", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "33": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-03", "properties": {"DATE": "1967-02-03", "FLOW": 3.109999895095825, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-03", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "34": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-02", "properties": {"DATE": "1967-02-02", "FLOW": 2.8299999237060547, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-02", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "35": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-05", "properties": {"DATE": "1967-02-05", "FLOW": 3.9600000381469727, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-05", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "36": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-04", "properties": {"DATE": "1967-02-04", "FLOW": 3.400000095367432, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-04", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "37": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-07", "properties": {"DATE": "1967-02-07", "FLOW": 6.369999885559082, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-07", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "38": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1967-02-06", "properties": {"DATE": "1967-02-06", "FLOW": 4.53000020980835, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1967-02-06", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "39": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1995-01-29", "properties": {"DATE": "1995-01-29", "FLOW": 3.309999942779541, "FLOW_SYMBOL_EN": "Ice Conditions", "FLOW_SYMBOL_FR": "Conditions \u00e0 glace", "IDENTIFIER": "02HC003.1995-01-29", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "40": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-13", "properties": {"DATE": "1987-03-13", "FLOW": 9.720000267028809, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-13", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "41": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.2017-05-27", "properties": {"DATE": "2017-05-27", "FLOW": 17.299999237060547, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.2017-05-27", "LEVEL": 2.5420000553131104, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "42": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-12", "properties": {"DATE": "1987-03-12", "FLOW": 12.600000381469728, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-12", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "43": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-11", "properties": {"DATE": "1987-03-11", "FLOW": 16.200000762939453, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-11", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "44": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-10", "properties": {"DATE": "1987-03-10", "FLOW": 29.100000381469727, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-10", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "45": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-17", "properties": {"DATE": "1987-03-17", "FLOW": 7.440000057220459, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-17", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "46": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-16", "properties": {"DATE": "1987-03-16", "FLOW": 6.989999771118164, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-16", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "47": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.2017-05-26", "properties": {"DATE": "2017-05-26", "FLOW": 30.399999618530277, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.2017-05-26", "LEVEL": 2.691999912261963, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "48": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1987-03-15", "properties": {"DATE": "1987-03-15", "FLOW": 7.210000038146973, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1987-03-15", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "49": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.1983-11-17", "properties": {"DATE": "1983-11-17", "FLOW": 13.0, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.1983-11-17", "LEVEL": null, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}, "50": {"geometry": {"coordinates": [-79.52039337158203, 43.69894027709961], "type": "Point"}, "id": "02HC003.2014-07-07", "properties": {"DATE": "2014-07-07", "FLOW": 6.110000133514404, "FLOW_SYMBOL_EN": null, "FLOW_SYMBOL_FR": null, "IDENTIFIER": "02HC003.2014-07-07", "LEVEL": 2.2950000762939453, "LEVEL_SYMBOL_EN": null, "LEVEL_SYMBOL_FR": null, "PROV_TERR_STATE_LOC": "ON", "STATION_NAME": "HUMBER RIVER AT WESTON", "STATION_NUMBER": "02HC003"}, "type": "Feature"}}} \ No newline at end of file diff --git a/tests/test_tinydb_provider.py b/tests/test_tinydb_provider.py new file mode 100644 index 000000000..c90f8ae0c --- /dev/null +++ b/tests/test_tinydb_provider.py @@ -0,0 +1,191 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2024 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +import shutil + +import pytest + +from pygeoapi.provider.base import ProviderItemNotFoundError +from pygeoapi.provider.tinydb_ import TinyDBProvider + +from .util import get_test_file_path + +path = get_test_file_path('tests/data/canada-hydat-daily-mean-02HC003.tinydb') + + +@pytest.fixture() +def data(): + return json.dumps({ + 'type': 'Feature', + 'geometry': { + 'type': 'Polygon', + 'coordinates': [[ + [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], + [100.0, 1.0], [100.0, 0.0] + ]] + }, + 'properties': { + 'identifier': 123, + 'title': 'test item', + 'description': 'test item' + } + }) + + +@pytest.fixture() +def data_no_id(): + return json.dumps({ + 'type': 'Feature', + 'geometry': { + 'type': 'Polygon', + 'coordinates': [[ + [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], + [100.0, 1.0], [100.0, 0.0] + ]] + }, + 'properties': { + 'title': 'test item', + 'description': 'test item' + } + }) + + +@pytest.fixture() +def config(tmp_path): + tmp_file = tmp_path / 'sample-features.tinydb' + shutil.copy(path, tmp_file) + return { + 'name': 'TinyDB', + 'type': 'feature', + 'data': tmp_file, + 'id_field': 'IDENTIFIER', + 'time_field': 'DATE' + } + + +def test_query(config): + p = TinyDBProvider(config) + + fields = p.get_fields() + assert len(fields) == 11 + assert fields['FLOW']['type'] == 'number' + assert fields['DATE']['type'] == 'string' + assert fields['DATE']['format'] == 'date' + + results = p.query() + assert len(results['features']) == 10 + assert results['numberMatched'] == 50 + assert results['numberReturned'] == 10 + assert results['features'][0]['id'] == '02HC003.1975-10-03' + assert results['features'][0]['properties']['STATION_NUMBER'] == '02HC003' + + results = p.query(properties=[('FLOW', 2.039999961853028)]) + assert len(results['features']) == 1 + assert results['numberMatched'] == 1 + assert results['numberReturned'] == 1 + + results = p.query(limit=1) + assert len(results['features']) == 1 + assert results['features'][0]['id'] == '02HC003.1975-10-03' + + results = p.query(datetime_='2017/..') + assert len(results['features']) == 3 + assert results['features'][0]['id'] == '02HC003.2017-05-23' + + results = p.query(datetime_='../2017') + assert len(results['features']) == 10 + assert results['features'][0]['id'] == '02HC003.1975-10-03' + + results = p.query(datetime_='1987-11-11/2000-11-11') + assert len(results['features']) == 1 + assert results['features'][0]['id'] == '02HC003.1995-01-29' + + results = p.query(bbox=[-154, 42, -52, 84]) + assert len(results['features']) == 10 + assert results['features'][0]['id'] == '02HC003.1975-10-03' + + results = p.query(offset=1, limit=1) + assert len(results['features']) == 1 + assert results['features'][0]['id'] == '02HC003.1975-10-02' + + results = p.query(offset=2, limit=2) + assert len(results['features']) == 2 + assert results['features'][0]['id'] == '02HC003.1975-10-01' + + results = p.query(sortby=[{'property': 'DATE', 'order': '+'}]) + assert results['features'][0]['id'] == '02HC003.1955-09-01' + + results = p.query(sortby=[{'property': 'DATE', 'order': '-'}]) + assert results['features'][0]['id'] == '02HC003.2017-05-27' + + +def test_get(config): + p = TinyDBProvider(config) + + result = p.get('02HC003.1975-10-02') + assert result['id'] == '02HC003.1975-10-02' + assert result['properties']['FLOW'] == 2.059999942779541 + + +def test_get_not_existing_item_raise_exception(config): + """Testing query for a not existing object""" + p = TinyDBProvider(config) + with pytest.raises(ProviderItemNotFoundError): + p.get('404') + + +def test_transactions_create(config, data): + """Testing transactional capabilities""" + + p = TinyDBProvider(config) + + new_id = p.create(data) + assert new_id == 123 + + assert p.update(123, data) + + assert p.delete(123) + + +def test_transactions_create_no_id(config, data_no_id): + """Testing transactional capabilities with incoming feature without ID""" + + p = TinyDBProvider(config) + + new_id = p.create(data_no_id) + assert new_id is not None + + data_got = p.get(new_id) + assert data_got['id'] == new_id + assert data_got['geometry'] == json.loads(data_no_id)['geometry'] + + assert p.update(new_id, json.dumps(data_got)) + + assert p.delete(new_id) From 533151831cac3790de0c478a9e009a6ec517dc1c Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:42:33 -0400 Subject: [PATCH 05/74] Add CRS transform to remaining providers (#1729) * Add CRS transform to remaining providers * Update tinydb_.py --------- Co-authored-by: Tom Kralidis --- docs/source/data-publishing/ogcapi-features.rst | 4 ++-- pygeoapi/provider/erddap.py | 3 +++ pygeoapi/provider/tinydb_.py | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index 32e657de7..ad84adea1 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -21,7 +21,7 @@ parameters. `CSV`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,✅ `Elasticsearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅ - `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌,❌ + `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌,✅ `ESRI Feature Service`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ `GeoJSON`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,✅ `MongoDB`_,✅/❌,results,✅,✅,✅,✅,❌,❌,✅ @@ -31,7 +31,7 @@ parameters. `SQLiteGPKG`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,✅ `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ `Socrata`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ - `TinyDB`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,❌ + `TinyDB`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅ .. note:: diff --git a/pygeoapi/provider/erddap.py b/pygeoapi/provider/erddap.py index a81876e1f..392ad24ec 100644 --- a/pygeoapi/provider/erddap.py +++ b/pygeoapi/provider/erddap.py @@ -51,6 +51,7 @@ from pygeoapi.provider.base import ( BaseProvider, ProviderNotFoundError, ProviderQueryError) +from pygeoapi.util import crs_transform LOGGER = logging.getLogger(__name__) @@ -80,6 +81,7 @@ def get_fields(self): return properties + @crs_transform def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], select_properties=[], skip_geometry=False, q=None, @@ -164,6 +166,7 @@ def query(self, offset=0, limit=10, resulttype='results', 'numberReturned': returned } + @crs_transform def get(self, identifier, **kwargs): query_params = [] diff --git a/pygeoapi/provider/tinydb_.py b/pygeoapi/provider/tinydb_.py index e46730aba..393ed2108 100644 --- a/pygeoapi/provider/tinydb_.py +++ b/pygeoapi/provider/tinydb_.py @@ -38,7 +38,7 @@ from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderItemNotFoundError) -from pygeoapi.util import get_typed_value +from pygeoapi.util import crs_transform, get_typed_value LOGGER = logging.getLogger(__name__) @@ -116,6 +116,7 @@ def get_fields(self): return fields + @crs_transform def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], select_properties=[], skip_geometry=False, q=None, **kwargs): @@ -234,6 +235,7 @@ def query(self, offset=0, limit=10, resulttype='results', return feature_collection + @crs_transform def get(self, identifier, **kwargs): """ Get TinyDB document by id From 0be6de01b84d146b69c91cae5284e99c87daf327 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:51:03 -0400 Subject: [PATCH 06/74] Update default docker bbox extents (#1732) --- docker/default.config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/default.config.yml b/docker/default.config.yml index bc20d7b58..6b5af029e 100644 --- a/docker/default.config.yml +++ b/docker/default.config.yml @@ -200,7 +200,7 @@ resources: hreflang: nl-NL extents: spatial: - bbox: [50.7539, 7.21097, 53.4658, 3.37087] + bbox: [3.37,50.75,7.21,53.47] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: @@ -301,7 +301,7 @@ resources: hreflang: en-US extents: spatial: - bbox: [36.0, 17.0, 46.0, 18.0] + bbox: [5.0,36.0,20.0,46.0] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: @@ -438,7 +438,7 @@ resources: hreflang: nl-NL extents: spatial: - bbox: [50.7539, 7.21097, 53.4658, 3.37087] + bbox: [3.37,50.75,7.21,53.47] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: @@ -481,7 +481,7 @@ resources: hreflang: nl-NL extents: spatial: - bbox: [50.7539, 7.21097, 53.4658, 3.37087] + bbox: [3.37,50.75,7.21,53.47] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: From 3d81115b1cc3208a52b83a9613ad34694e7a8d68 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:27:06 -0400 Subject: [PATCH 07/74] Allow setting docker repository via repository secret (#1734) --- .github/workflows/containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 67544fed2..04897a440 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -12,7 +12,7 @@ on: branches: [master] env: - DOCKER_REPOSITORY: geopython/pygeoapi + DOCKER_REPOSITORY: ${{ secrets.DOCKER_REPOSITORY || 'geopython/pygeoapi' }} # DOCKER_TEST_IMAGE: geopython/pygeoapi:test jobs: From 510875027e8483ce2916e7cf315fb6a7f6105807 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 17 Jul 2024 10:13:17 -0400 Subject: [PATCH 08/74] EDR: add HTML table of parameters for collections with EDR support (#1733) (#1736) --- .../templates/collections/collection.html | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pygeoapi/templates/collections/collection.html b/pygeoapi/templates/collections/collection.html index 5edccc97f..65bac7b33 100644 --- a/pygeoapi/templates/collections/collection.html +++ b/pygeoapi/templates/collections/collection.html @@ -85,6 +85,25 @@

{% trans %}Tiles{% endtrans %}

{% endif %} {% endfor %} {% endif %} + + {% if 'parameter_names' in data %} +

Parameters

+ + + + + + + {% for parameter in data['parameter_names'].values() %} + + + + + + {% endfor %} +
idnameunits
{{ parameter['id'] }}{{ parameter['name'] }}{{ parameter['unit']['symbol']['value'] }}
+ {% endif %} +

{% trans %}Links{% endtrans %}

    {% for link in data['links'] %} From e91f051ea06507c41e3e3e2ef807b9e59027bae5 Mon Sep 17 00:00:00 2001 From: francescoingv <9592487+francescoingv@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:00:59 +0200 Subject: [PATCH 09/74] Update processes.py (#1738) Co-authored-by: FrancescoIngv --- pygeoapi/api/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 3eb9ea695..d045bfa39 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -171,7 +171,7 @@ def describe_processes(api: API, request: APIRequest, 'type': FORMAT_TYPES[F_JSON], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', 'href': f'{jobs_url}?f={F_JSON}', - 'title': l10n.translate('Jobs for this process as HTML', request.locale), # noqa + 'title': l10n.translate('Jobs for this process as JSON', request.locale), # noqa 'hreflang': api.default_locale } p2['links'].append(link) From c322d3fb40d53eb332cca89f5f42057dc206fd1c Mon Sep 17 00:00:00 2001 From: francescoingv <9592487+francescoingv@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:10:21 +0200 Subject: [PATCH 10/74] Release Process Manager backed by Postgresql (#1730) * Release Process Manager backed by Postgresql * Update postgresdb_.py * Changes as for Tom comment on 16.7 * Update postgresql.py * Included further Tom suggestions * Renamed Manager requirements file * renamed table job_info -> jobs * Update postgres_manager_full_structure.backup.sql * Update postgresql.py * Update requirements-manager.txt Accepted @Tom requirement. I found the issue about compiling psycopg2: it was a bug in my configuration. * Modified to adhere to CI * Update postgresql.py --------- Co-authored-by: FrancescoIngv --- .github/workflows/main.yml | 3 + pygeoapi/plugin.py | 3 +- pygeoapi/process/manager/postgresql.py | 270 ++++++++++++++++++ requirements-manager.txt | 2 + ...postgres_manager_full_structure.backup.sql | 68 +++++ .../pygeoapi-test-config-postgres-manager.yml | 113 ++++++++ tests/test_postgresql_manager.py | 177 ++++++++++++ 7 files changed, 635 insertions(+), 1 deletion(-) create mode 100644 pygeoapi/process/manager/postgresql.py create mode 100644 requirements-manager.txt create mode 100644 tests/data/postgres_manager_full_structure.backup.sql create mode 100644 tests/pygeoapi-test-config-postgres-manager.yml create mode 100644 tests/test_postgresql_manager.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 224c07247..fa9836cac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,6 +91,7 @@ jobs: pip3 install -r requirements-starlette.txt pip3 install -r requirements-dev.txt pip3 install -r requirements-provider.txt + pip3 install -r requirements-manager.txt pip3 install -r requirements-django.txt python3 setup.py install pip3 install --upgrade numpy elasticsearch @@ -104,6 +105,7 @@ jobs: gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_types_data.sql + psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/postgres_manager_full_structure.backup.sql docker ps python3 tests/load_oracle_data.py - name: run unit tests ⚙️ @@ -127,6 +129,7 @@ jobs: pytest tests/test_ogr_shapefile_provider.py pytest tests/test_ogr_sqlite_provider.py pytest tests/test_ogr_wfs_provider.py + pytest tests/test_postgresql_manager.py # pytest tests/test_ogr_wfs_provider_live.py # NOTE: these are skipped in the file but listed here for completeness pytest tests/test_openapi.py pytest tests/test_oracle_provider.py diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index 7e4225562..74ef7324c 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -78,7 +78,8 @@ 'process_manager': { 'Dummy': 'pygeoapi.process.manager.dummy.DummyManager', 'MongoDB': 'pygeoapi.process.manager.mongodb_.MongoDBManager', - 'TinyDB': 'pygeoapi.process.manager.tinydb_.TinyDBManager' + 'TinyDB': 'pygeoapi.process.manager.tinydb_.TinyDBManager', + 'PostgreSQL': 'pygeoapi.process.manager.postgresql.PostgreSQLManager' } } diff --git a/pygeoapi/process/manager/postgresql.py b/pygeoapi/process/manager/postgresql.py new file mode 100644 index 000000000..017de7f55 --- /dev/null +++ b/pygeoapi/process/manager/postgresql.py @@ -0,0 +1,270 @@ +# ================================================================= +# +# Authors: Francesco Martinelli +# +# Copyright (c) 2024 Francesco Martinelli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# Requires postgresql database structure. +# Create the database: +# e.g. +# CREATE DATABASE test +# WITH TEMPLATE = template0 +# ENCODING = 'UTF8' +# LOCALE = 'en_US.UTF-8'; +# ALTER DATABASE test OWNER TO postgres; +# +# Import dump: +# psql -U postgres -h 127.0.0.1 -p 5432 test < +# tests/data/postgres_manager_full_structure.backup.sql + +import json +import logging +from pathlib import Path +from typing import Any, Tuple + +import psycopg2 +import psycopg2.extras + +from pygeoapi.process.manager.base import BaseManager +from pygeoapi.process.base import ( + JobNotFoundError, + JobResultNotFoundError, + ProcessorGenericError, +) +from pygeoapi.util import JobStatus + + +LOGGER = logging.getLogger(__name__) + + +class PostgreSQLManager(BaseManager): + """PostgreSql Manager""" + + def __init__(self, manager_def: dict): + """ + Initialize object + + :param manager_def: manager definition + + :returns: `pygeoapi.process.manager.postgresqs.PostgreSQLManager` + """ + + super().__init__(manager_def) + self.is_async = True + self.supports_subscribing = True + + self.__database_connection_parameters = manager_def['connection'] + try: + # Test connection parameters: + test_query = """SELECT version()""" + with self.get_db_connection() as conn: + with conn.cursor() as cur: + cur.execute(test_query) + cur.fetchone() + except Exception as err: + LOGGER.error(f'Test connecting to DB failed: {err}') + raise ProcessorGenericError('Test connecting to DB failed.') + + def get_db_connection(self): + """ + Get and return a new connection to the DB. + """ + if isinstance(self.__database_connection_parameters, str): + conn = psycopg2.connect(self.__database_connection_parameters) + else: + conn = psycopg2.connect(**self.__database_connection_parameters) + + return conn + + def get_jobs(self, status: JobStatus = None) -> list: + """ + Get jobs + + :param status: job status (accepted, running, successful, + failed, results) (default is all) + + :returns: 'list` of jobs (type (default='process'), identifier, + status, process_id, job_start_datetime, job_end_datetime, location, + mimetype, message, progress) + """ + + with self.get_db_connection() as conn: + with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: + query_select = """SELECT * FROM jobs """ + if status is not None: + query_select = query_select + "WHERE status = %s" + query_params = [status.value] + else: + query_params = [] + cur.execute(query_select, query_params) + return cur.fetchall() + + def add_job(self, job_metadata: dict) -> str: + """ + Add a job + + :param job_metadata: `dict` of job metadata + + :returns: identifier of added job + """ + + query_insert = """INSERT INTO jobs( + type, process_id, identifier, status, message, + progress, job_start_datetime, job_end_datetime + ) VALUES(%(type)s, %(process_id)s, %(identifier)s, %(status)s, + %(message)s, %(progress)s, %(job_start_datetime)s, + %(job_end_datetime)s);""" + with self.get_db_connection() as conn: + with conn.cursor() as cur: + cur.execute(query_insert, job_metadata) + conn.commit() + return job_metadata['identifier'] + + def update_job(self, job_id: str, update_dict: dict) -> bool: + """ + Updates a job + + :param job_id: job identifier + :param update_dict: `dict` of property updates + + :returns: `bool` of status result + """ + + query_update = "UPDATE jobs SET (" + keys_to_update = 0 + for key in update_dict.keys(): + if keys_to_update: + query_update = query_update + (", ") + query_update = query_update + key + keys_to_update = keys_to_update + 1 + + query_update = query_update + ") = (" + keys_to_update = 0 + for key in update_dict.keys(): + if keys_to_update: + query_update = query_update + (", ") + query_update = query_update + "%(" + key + ")s" + keys_to_update = keys_to_update + 1 + query_update = query_update + (") WHERE identifier = %(identifier)s") + + update_dict['identifier'] = job_id + + with self.get_db_connection() as conn: + with conn.cursor() as cur: + cur.execute(query_update, update_dict) + rowcount = cur.rowcount + conn.commit() + + return rowcount == 1 + + def get_job(self, job_id: str) -> dict: + """ + Get a single job + + :param job_id: job identifier + + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :returns: `dict` # `pygeoapi.process.manager.Job` + """ + + with self.get_db_connection() as conn: + with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: + query_select = \ + """SELECT * FROM jobs WHERE identifier = %s""" + query_params = [job_id] + cur.execute(query_select, query_params) + found = cur.fetchone() + + if found is not None: + return found + else: + raise JobNotFoundError() + + def delete_job(self, job_id: str) -> bool: + """ + Deletes a job + + :param job_id: job identifier + + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :return `bool` of status result + """ + # delete result file if present + job_result = self.get_job(job_id) + location = job_result.get('location') + if location and self.output_dir is not None: + try: + Path(location).unlink() + except FileNotFoundError: + pass + + query_delete = "DELETE FROM jobs WHERE identifier = %s" + with self.get_db_connection() as conn: + with conn.cursor() as cur: + cur.execute(query_delete, [job_id]) + rowcount = cur.rowcount + conn.commit() + + return rowcount == 1 + + def get_job_result(self, job_id: str) -> Tuple[str, Any]: + """ + Get a job's status, and actual output of executing the process + + :param job_id: job identifier + + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :raises JobResultNotFoundError: if the job-related result cannot + be returned + :returns: `tuple` of mimetype and raw output + """ + + job_result = self.get_job(job_id) + location = job_result.get('location') + mimetype = job_result.get('mimetype') + job_status = JobStatus[job_result['status']] + + if job_status != JobStatus.successful: + # Job is incomplete + return (None,) + if not location: + LOGGER.warning(f'job {job_id!r} - unknown result location') + raise JobResultNotFoundError() + else: + try: + location = Path(location) + with location.open(encoding='utf-8') as fh: + result = json.load(fh) + except (TypeError, FileNotFoundError, json.JSONDecodeError): + raise JobResultNotFoundError() + else: + return mimetype, result + + def __repr__(self): + return f' {self.name}' diff --git a/requirements-manager.txt b/requirements-manager.txt new file mode 100644 index 000000000..6ccfa4b1a --- /dev/null +++ b/requirements-manager.txt @@ -0,0 +1,2 @@ +psycopg2 + diff --git a/tests/data/postgres_manager_full_structure.backup.sql b/tests/data/postgres_manager_full_structure.backup.sql new file mode 100644 index 000000000..804b34b67 --- /dev/null +++ b/tests/data/postgres_manager_full_structure.backup.sql @@ -0,0 +1,68 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 14.12 (Ubuntu 14.12-1.pgdg20.04+1) +-- Dumped by pg_dump version 16.3 (Ubuntu 16.3-1.pgdg20.04+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: public; Type: SCHEMA; Schema: -; Owner: postgres +-- + +ALTER SCHEMA public OWNER TO postgres; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: jobs; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.jobs ( + type character varying DEFAULT 'process'::character varying NOT NULL, + identifier character varying NOT NULL, + process_id character varying NOT NULL, + job_start_datetime timestamp without time zone, + job_end_datetime timestamp without time zone, + status character varying NOT NULL, + location character varying, + mimetype character varying, + message character varying, + progress integer NOT NULL +); + + +ALTER TABLE public.jobs OWNER TO postgres; + +-- +-- Name: jobs jobs_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.jobs + ADD CONSTRAINT jobs_pkey PRIMARY KEY (identifier); + + +-- +-- Name: SCHEMA public; Type: ACL; Schema: -; Owner: postgres +-- + +REVOKE USAGE ON SCHEMA public FROM PUBLIC; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/tests/pygeoapi-test-config-postgres-manager.yml b/tests/pygeoapi-test-config-postgres-manager.yml new file mode 100644 index 000000000..496f5582b --- /dev/null +++ b/tests/pygeoapi-test-config-postgres-manager.yml @@ -0,0 +1,113 @@ +# ================================================================= +# +# Authors: Francesco Martinelli +# +# Copyright (c) 2024 Francesco Martinelli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + bind: + host: 0.0.0.0 + port: 5000 + url: http://localhost:5000/ + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + gzip: false + languages: + # First language is the default language + - en-US + - fr-CA + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + manager: + name: PostgreSQL + connection: + host: localhost + port: 5432 + database: test + user: postgres + password: ${POSTGRESQL_PASSWORD:-postgres} + # Alternative accepted connection definition: + # connection: postgresql://postgres:postgres@localhost:5432/test + output_dir: /tmp + +logging: + level: DEBUG + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: + en: pygeoapi instance to test Process Manager backed by PostgreSql + fr: test instance de pygeoapi + description: + en: pygeoapi provides an API to geospatial data + fr: pygeoapi fournit une API aux données géospatiales + keywords: + en: + - geospatial + - data + - api + fr: + - géospatiale + - données + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + + hello-world: + type: process + processor: + name: HelloWorld + diff --git a/tests/test_postgresql_manager.py b/tests/test_postgresql_manager.py new file mode 100644 index 000000000..cd3c86f38 --- /dev/null +++ b/tests/test_postgresql_manager.py @@ -0,0 +1,177 @@ +# ================================================================= +# +# Authors: Francesco Martinelli +# +# Copyright (c) 2024 Francesco Martinelli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# See pygeoapi/process/manager/postgresql.py +# for instructions on setting up database structure. + +import json + +import pytest +from werkzeug.wrappers import Request +from werkzeug.test import create_environ + +from .util import get_test_file_path +from pygeoapi.api import API, APIRequest +import pygeoapi.api.processes as processes_api +from pygeoapi.util import yaml_load + + +@pytest.fixture() +def config(): + with open(get_test_file_path( + 'pygeoapi-test-config-postgres-manager.yml') + ) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def openapi(): + with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def api_(config, openapi): + return API(config, openapi) + + +def _create_execute_request(name, message, locales): + data = { + "response": "raw", + "inputs": { + "name": name, + "message": message + } + } + environ = create_environ( + base_url='http://localhost:5000/processes/hello-world/execution', + method="POST", json=data) + req = Request(environ) + return APIRequest.with_data(req, locales) + + +def _create_job_request(job_id, locales): + environ = create_environ( + base_url=f'http://localhost:5000/jobs/{job_id}', + query_string="f=json", + method="GET") + req = Request(environ) + return APIRequest.with_data(req, locales) + + +def _create_results_request(job_id, locales): + environ = create_environ( + base_url=f'http://localhost:5000/jobs/{job_id}/results', + query_string="f=json", + method="GET") + req = Request(environ) + return APIRequest.with_data(req, locales) + + +def _create_delete_request(job_id, locales): + environ = create_environ( + base_url=f'http://localhost:5000/jobs/{job_id}', + query_string="f=json", + method="DELETE") + req = Request(environ) + return APIRequest.with_data(req, locales) + + +def test_job_sync_hello_world(api_, config): + """ + Create a new job for hello-world, + which mplicitly tests add_job() and update_job(); + then: + -) get the job info, whch tests get_job(), + -) get the job results, whch tests get_job_result(), + -) get all present jobs, whch tests get_jobs(), + -) delete the newly inserted job, whch tests delete_job(). + """ + process_id = "hello-world" + + # Create new job + req = _create_execute_request("World", "Hello", api_.locales) + headers, http_status, response = processes_api.execute_process( + api_, req, process_id) + assert http_status == 200 + out_json = json.loads(response) + assert out_json["id"] == "echo" + assert out_json["value"] == "Hello World! Hello" + + # Save job_id for later use + job_id = headers['Location'].split('/')[-1] + mimetype = headers['Content-Type'] + + # Get job info + req = _create_job_request(job_id, api_.locales) + headers, http_status, response = processes_api.get_jobs( + api_, req, job_id) + assert http_status == 200 + out_json = json.loads(response) + assert out_json["type"] == "process" + assert out_json["processID"] == process_id + assert out_json["jobID"] == job_id + + # Get job results + req = _create_results_request(job_id, api_.locales) + headers, http_status, response = processes_api.get_job_result( + api_, req, job_id) + assert http_status == 200 + assert mimetype == headers['Content-Type'] + out_json = json.loads(response) + assert out_json["id"] == "echo" + assert out_json["value"] == "Hello World! Hello" + + # Get all present jobs + req = _create_job_request(None, api_.locales) + headers, http_status, response = processes_api.get_jobs( + api_, req, None) + assert http_status == 200 + # check the inserted job is in the list + out_json = json.loads(response) + jobs = out_json["jobs"] + assert any(job["jobID"] == job_id for job in jobs) + + # Delete the inserted job + req = _create_delete_request(job_id, api_.locales) + headers, http_status, response = processes_api.delete_job( + api_, req, job_id) + assert http_status == 200 + out_json = json.loads(response) + assert out_json["jobID"] == job_id + assert out_json["status"] == "dismissed" + + # Try again to delete the inserted job + req = _create_delete_request(job_id, api_.locales) + headers, http_status, response = processes_api.get_jobs( + api_, req, job_id) + assert http_status == 404 + out_json = json.loads(response) + assert out_json["code"] == "InvalidParameterValue" + assert out_json["description"] == job_id From 05a7de433d6f73fe5b8d8c69d9deeb62032993be Mon Sep 17 00:00:00 2001 From: pgValero <67552686+pgValero@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:14:10 +0200 Subject: [PATCH 11/74] Update setup.py to restrict python version >= 3.9 (#1725) * Update setup.py Release 0.17.0 is incompatible with Python < 3.8 "python_requires" added to only install this version with python >= 3.9 * Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9ff9c3180..7013179aa 100644 --- a/setup.py +++ b/setup.py @@ -155,6 +155,7 @@ def get_package_version(): maintainer='Tom Kralidis', maintainer_email='tomkralidis@gmail.com', url='https://pygeoapi.io', + python_requires='>=3.10', install_requires=read('requirements.txt').splitlines(), packages=find_packages(exclude=['pygeoapi.tests']), include_package_data=True, From 9d53f1b8f74727db6648ac63f48c2749b1e1040d Mon Sep 17 00:00:00 2001 From: Paul Tomasula <31142705+ptomasula@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:12:57 -0400 Subject: [PATCH 12/74] Fix for Starlette CORS (#1741) * Fix for Starlette CORS The Starlette app with setting cors=true enables all origins, but only permits requests with the GET method. The CORSMiddleware for Starlette requires specifying `allow_methods=['*']` to enable all common request methods. * Add noqa to break line break This line was flagged by flake 8 for being 81 characters. I think that this line it more readable as is rather than splitting up into multiple lines. * Revise indentation on CORS middleware --- pygeoapi/starlette_app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index b2ddf4858..9f4e33f79 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -742,7 +742,11 @@ async def __call__(self, scope: Scope, # CORS: optionally enable from config. if CONFIG['server'].get('cors', False): from starlette.middleware.cors import CORSMiddleware - APP.add_middleware(CORSMiddleware, allow_origins=['*']) + APP.add_middleware( + CORSMiddleware, + allow_origins=['*'], + allow_methods=['*'], + ) try: OGC_SCHEMAS_LOCATION = Path(CONFIG['server']['ogc_schemas_location']) From 298f069ffa4a2fd1c837441ded6e656317afc941 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Fri, 19 Jul 2024 12:34:58 -0400 Subject: [PATCH 13/74] update jobs endpoint link title (#1739) (#1742) --- pygeoapi/api/processes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index d045bfa39..236f16e16 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -162,7 +162,7 @@ def describe_processes(api: API, request: APIRequest, 'type': FORMAT_TYPES[F_HTML], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', 'href': f'{jobs_url}?f={F_HTML}', - 'title': l10n.translate('Jobs for this process as HTML', request.locale), # noqa + 'title': l10n.translate('Jobs list as HTML', request.locale), # noqa 'hreflang': api.default_locale } p2['links'].append(link) @@ -171,7 +171,7 @@ def describe_processes(api: API, request: APIRequest, 'type': FORMAT_TYPES[F_JSON], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', 'href': f'{jobs_url}?f={F_JSON}', - 'title': l10n.translate('Jobs for this process as JSON', request.locale), # noqa + 'title': l10n.translate('Jobs list as JSON', request.locale), # noqa 'hreflang': api.default_locale } p2['links'].append(link) From a000fe558fa5638d5b502e65eab9005644dcb97b Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 20 Jul 2024 07:19:08 -0400 Subject: [PATCH 14/74] port PostgreSQL process manager to SQLAlchemy (#1745) --- pygeoapi/process/manager/postgresql.py | 196 ++++++++++-------- ...geoapi-test-config-postgresql-manager.yml} | 2 +- tests/test_postgresql_manager.py | 2 +- 3 files changed, 110 insertions(+), 90 deletions(-) rename tests/{pygeoapi-test-config-postgres-manager.yml => pygeoapi-test-config-postgresql-manager.yml} (97%) diff --git a/pygeoapi/process/manager/postgresql.py b/pygeoapi/process/manager/postgresql.py index 017de7f55..8215b1241 100644 --- a/pygeoapi/process/manager/postgresql.py +++ b/pygeoapi/process/manager/postgresql.py @@ -45,15 +45,16 @@ from pathlib import Path from typing import Any, Tuple -import psycopg2 -import psycopg2.extras +from sqlalchemy import insert, update, delete +from sqlalchemy.orm import Session -from pygeoapi.process.manager.base import BaseManager from pygeoapi.process.base import ( JobNotFoundError, JobResultNotFoundError, - ProcessorGenericError, + ProcessorGenericError ) +from pygeoapi.process.manager.base import BaseManager +from pygeoapi.provider.postgresql import get_engine, get_table_model from pygeoapi.util import JobStatus @@ -61,7 +62,7 @@ class PostgreSQLManager(BaseManager): - """PostgreSql Manager""" + """PostgreSQL Manager""" def __init__(self, manager_def: dict): """ @@ -74,30 +75,39 @@ def __init__(self, manager_def: dict): super().__init__(manager_def) self.is_async = True + self.id_field = 'identifier' self.supports_subscribing = True + self.connection = manager_def['connection'] - self.__database_connection_parameters = manager_def['connection'] try: - # Test connection parameters: - test_query = """SELECT version()""" - with self.get_db_connection() as conn: - with conn.cursor() as cur: - cur.execute(test_query) - cur.fetchone() - except Exception as err: - LOGGER.error(f'Test connecting to DB failed: {err}') - raise ProcessorGenericError('Test connecting to DB failed.') + self.db_search_path = tuple(self.connection.get('search_path', + ['public'])) + except Exception: + self.db_search_path = 'public' - def get_db_connection(self): - """ - Get and return a new connection to the DB. - """ - if isinstance(self.__database_connection_parameters, str): - conn = psycopg2.connect(self.__database_connection_parameters) - else: - conn = psycopg2.connect(**self.__database_connection_parameters) + try: + LOGGER.debug('Connecting to database') + if isinstance(self.connection, str): + self._engine = get_engine(self.connection) + else: + self._engine = get_engine(**self.connection) + except Exception as err: + msg = 'Test connecting to DB failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) - return conn + try: + LOGGER.debug('Getting table model') + self.table_model = get_table_model( + 'jobs', + self.id_field, + self.db_search_path, + self._engine + ) + except Exception as err: + msg = 'Table model fetch failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) def get_jobs(self, status: JobStatus = None) -> list: """ @@ -111,16 +121,14 @@ def get_jobs(self, status: JobStatus = None) -> list: mimetype, message, progress) """ - with self.get_db_connection() as conn: - with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: - query_select = """SELECT * FROM jobs """ - if status is not None: - query_select = query_select + "WHERE status = %s" - query_params = [status.value] - else: - query_params = [] - cur.execute(query_select, query_params) - return cur.fetchall() + LOGGER.debug('Querying for jobs') + with Session(self._engine) as session: + results = session.query(self.table_model) + if status is not None: + column = getattr(self.table_model, 'status') + results = results.filter(column == status.value) + + return [r.__dict__ for r in results.all()] def add_job(self, job_metadata: dict) -> str: """ @@ -131,16 +139,18 @@ def add_job(self, job_metadata: dict) -> str: :returns: identifier of added job """ - query_insert = """INSERT INTO jobs( - type, process_id, identifier, status, message, - progress, job_start_datetime, job_end_datetime - ) VALUES(%(type)s, %(process_id)s, %(identifier)s, %(status)s, - %(message)s, %(progress)s, %(job_start_datetime)s, - %(job_end_datetime)s);""" - with self.get_db_connection() as conn: - with conn.cursor() as cur: - cur.execute(query_insert, job_metadata) - conn.commit() + LOGGER.debug('Adding job') + with Session(self._engine) as session: + try: + session.execute(insert(self.table_model) + .values(**job_metadata)) + session.commit() + except Exception as err: + session.rollback() + msg = 'Insert failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + return job_metadata['identifier'] def update_job(self, job_id: str, update_dict: dict) -> bool: @@ -153,30 +163,25 @@ def update_job(self, job_id: str, update_dict: dict) -> bool: :returns: `bool` of status result """ - query_update = "UPDATE jobs SET (" - keys_to_update = 0 - for key in update_dict.keys(): - if keys_to_update: - query_update = query_update + (", ") - query_update = query_update + key - keys_to_update = keys_to_update + 1 - - query_update = query_update + ") = (" - keys_to_update = 0 - for key in update_dict.keys(): - if keys_to_update: - query_update = query_update + (", ") - query_update = query_update + "%(" + key + ")s" - keys_to_update = keys_to_update + 1 - query_update = query_update + (") WHERE identifier = %(identifier)s") - - update_dict['identifier'] = job_id - - with self.get_db_connection() as conn: - with conn.cursor() as cur: - cur.execute(query_update, update_dict) - rowcount = cur.rowcount - conn.commit() + rowcount = 0 + + LOGGER.debug('Updating job') + with Session(self._engine) as session: + try: + column = getattr(self.table_model, self.id_field) + stmt = ( + update(self.table_model) + .where(column == job_id) + .values(**update_dict) + ) + result = session.execute(stmt) + session.commit() + rowcount = result.rowcount + except Exception as err: + session.rollback() + msg = 'Update failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) return rowcount == 1 @@ -191,18 +196,18 @@ def get_job(self, job_id: str) -> dict: :returns: `dict` # `pygeoapi.process.manager.Job` """ - with self.get_db_connection() as conn: - with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: - query_select = \ - """SELECT * FROM jobs WHERE identifier = %s""" - query_params = [job_id] - cur.execute(query_select, query_params) - found = cur.fetchone() + LOGGER.debug('Querying for job') + with Session(self._engine) as session: + results = session.query(self.table_model) + column = getattr(self.table_model, self.id_field) + results = session.query(self.table_model).filter(column == job_id) - if found is not None: - return found - else: - raise JobNotFoundError() + first = results.first() + + if first is not None: + return first.__dict__ + else: + raise JobNotFoundError() def delete_job(self, job_id: str) -> bool: """ @@ -214,22 +219,37 @@ def delete_job(self, job_id: str) -> bool: known job :return `bool` of status result """ - # delete result file if present + + rowcount = 0 + + # get result file if present for deletion job_result = self.get_job(job_id) location = job_result.get('location') - if location and self.output_dir is not None: + + LOGGER.debug('Deleting job') + with Session(self._engine) as session: + try: + column = getattr(self.table_model, self.id_field) + stmt = ( + delete(self.table_model) + .where(column == job_id) + ) + result = session.execute(stmt) + session.commit() + rowcount = result.rowcount + except Exception as err: + session.rollback() + msg = 'Delete failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + + # delete result file if present + if None not in [location, self.output_dir]: try: Path(location).unlink() except FileNotFoundError: pass - query_delete = "DELETE FROM jobs WHERE identifier = %s" - with self.get_db_connection() as conn: - with conn.cursor() as cur: - cur.execute(query_delete, [job_id]) - rowcount = cur.rowcount - conn.commit() - return rowcount == 1 def get_job_result(self, job_id: str) -> Tuple[str, Any]: diff --git a/tests/pygeoapi-test-config-postgres-manager.yml b/tests/pygeoapi-test-config-postgresql-manager.yml similarity index 97% rename from tests/pygeoapi-test-config-postgres-manager.yml rename to tests/pygeoapi-test-config-postgresql-manager.yml index 496f5582b..fb3d63583 100644 --- a/tests/pygeoapi-test-config-postgres-manager.yml +++ b/tests/pygeoapi-test-config-postgresql-manager.yml @@ -55,7 +55,7 @@ server: user: postgres password: ${POSTGRESQL_PASSWORD:-postgres} # Alternative accepted connection definition: - # connection: postgresql://postgres:postgres@localhost:5432/test + # connection: postgresql://postgres:${POSTGRESQL_PASSWORD:-postgres}@localhost:5432/test output_dir: /tmp logging: diff --git a/tests/test_postgresql_manager.py b/tests/test_postgresql_manager.py index cd3c86f38..43464f6d9 100644 --- a/tests/test_postgresql_manager.py +++ b/tests/test_postgresql_manager.py @@ -45,7 +45,7 @@ @pytest.fixture() def config(): with open(get_test_file_path( - 'pygeoapi-test-config-postgres-manager.yml') + 'pygeoapi-test-config-postgresql-manager.yml') ) as fh: return yaml_load(fh) From 274646b6ae538467f8709e20f663b8b289acc345 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Sun, 21 Jul 2024 11:05:56 -0400 Subject: [PATCH 15/74] Add TRS to pygeoapi config schema (#1743) * Add TRS to pygeoapi config schema * Update configuration.rst --------- Co-authored-by: Tom Kralidis --- docs/source/configuration.rst | 1 + pygeoapi/schemas/config/pygeoapi-config-0.x.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 0c81b7126..b807ee961 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -206,6 +206,7 @@ default. temporal: # optional begin: 2000-10-30T18:24:39Z # start datetime in RFC3339 end: 2007-10-30T08:57:29Z # end datetime in RFC3339 + trs: http://www.opengis.net/def/uom/ISO-8601/0/Gregorian # TRS providers: # list of 1..n required connections information # provider name # see pygeoapi.plugin for supported providers diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml index 5d567c93a..ecc146c8c 100644 --- a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -411,6 +411,10 @@ properties: type: [string, 'null'] format: date-time nullable: true + trs: + type: string + description: temporal reference system of features + default: 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' required: - spatial providers: From b6c00b0cff1397c23f3945bdd7c6146bd49aa493 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 21 Jul 2024 16:39:08 -0400 Subject: [PATCH 16/74] update CITE setup (#1728) (#1731) --- tests/cite/README.md | 10 +++++++++- tests/cite/cite.config.yml | 39 +++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tests/cite/README.md b/tests/cite/README.md index f8142f9a3..e81afe17f 100644 --- a/tests/cite/README.md +++ b/tests/cite/README.md @@ -1,12 +1,20 @@ # CITE testing for OGC API capabilities - OGC API - Features +- OGC API - Tiles +- OGC API - Environmental Data Retrieval - OGC API - Processes ## Test data ### OGC API - Features -Test data used is in `tests/data/canada-hydat-daily-mean-02HC003.tinydb` +Test data used is in `tests/data/canada-hydat-daily-mean-02HC003.tinydb`. + +### OGC API - Tiles +Test data used is in `tests/data/tiles/ne_110m_lakes`. + +### OGC API - Environmental Data Retrieval +TODO ### OGC API - Processes The `hello-world` test process that is provided with pygeoapi by default is used. diff --git a/tests/cite/cite.config.yml b/tests/cite/cite.config.yml index 66362bd96..22d39510d 100644 --- a/tests/cite/cite.config.yml +++ b/tests/cite/cite.config.yml @@ -96,7 +96,6 @@ resources: temporal: begin: 1850-01-01T00:00:00Z end: null # or empty - providers: - type: feature name: TinyDB @@ -104,6 +103,44 @@ resources: id_field: IDENTIFIER time_field: DATE + lakes: + type: collection + title: Large Lakes + description: lakes of the world, public domain + keywords: + - lakes + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/ + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2011-11-11T00:00:00Z + end: null # or empty + providers: + - type: feature + name: GeoJSON + data: ../data/ne_110m_lakes.geojson + id_field: id + - type: tile + name: MVT-tippecanoe + data: ../data/tiles/ne_110m_lakes + options: + bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] + zoom: + min: 0 + max: 5 + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + hello-world: type: process processor: From f55aa875c25b895dbccca909d9bef83f3cf8065b Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:52:42 -0400 Subject: [PATCH 17/74] Remove spatial parameter from OAS for single location (#1747) --- pygeoapi/api/environmental_data_retrieval.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index 494e695d1..e371c4d0b 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -305,7 +305,6 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, 'tags': [k], 'operationId': f'queryLOCATIONSBYID{k.capitalize()}', 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa From e2676bdc56770237af9052c410269c69e2d50f45 Mon Sep 17 00:00:00 2001 From: Vincent Privat <146961743+vprivat-ads@users.noreply.github.com> Date: Mon, 22 Jul 2024 22:59:11 +0200 Subject: [PATCH 18/74] Document PostgreSQL process manager (#1746) * Document PostgreSQL process manager * Update ogcapi-processes.rst --------- Co-authored-by: Tom Kralidis --- .../data-publishing/ogcapi-processes.rst | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-processes.rst b/docs/source/data-publishing/ogcapi-processes.rst index c11f9f338..e62d3ff0c 100644 --- a/docs/source/data-publishing/ogcapi-processes.rst +++ b/docs/source/data-publishing/ogcapi-processes.rst @@ -52,11 +52,11 @@ can be requested by including the ``Prefer: respond-async`` HTTP header in the r output_dir: /tmp/ MongoDB --------------------- -As an alternative to the default a manager employing `MongoDB`_ can be used. -The connection to an installed `MongoDB`_ instance must be provided in the configuration. -`MongoDB`_ uses the localhost and port 27017 by default. Jobs are stored in a collection named -job_manager_pygeoapi. +------- +As an alternative to the default, a manager employing `MongoDB`_ can be used. +The connection to a `MongoDB`_ instance must be provided in the configuration. +`MongoDB`_ uses ``localhost`` and port ``27017`` by default. Jobs are stored in a collection named +``job_manager_pygeoapi``. .. code-block:: yaml @@ -67,10 +67,33 @@ job_manager_pygeoapi. output_dir: /tmp/ +PostgreSQL +---------- +As another alternative to the default, a manager employing `PostgreSQL`_ can be used. +The connection to a `PostgreSQL`_ database must be provided in the configuration. +`PostgreSQL`_ uses ``localhost`` and port ``5432`` by default. Jobs are stored in a table named ``jobs``. + +.. code-block:: yaml + + server: + manager: + name: PostgreSQL + connection: + host: localhost + port: 5432 + database: test + user: postgres + password: ${POSTGRESQL_PASSWORD:-postgres} + # Alternative accepted connection definition: + # connection: postgresql://postgres:postgres@localhost:5432/test + # connection: postgresql://postgres:${POSTGRESQL_PASSWORD:-postgres}@localhost:5432/test + output_dir: /tmp + + Putting it all together ----------------------- -To summarize how pygeoapi processes and managers work together:: +To summarize how pygeoapi processes and managers work together: * process plugins implement the core processing / workflow functionality * manager plugins control and manage how processes are executed From 31480af845dd770b22f2e5616f7fd1f67b65938a Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:58:06 -0400 Subject: [PATCH 19/74] Use consistent get_field ref in providers (#1727) * Use consistent get_field ref in providers * Fix flake8 * Update remaining providers * Fix recursive call * Fix recursive call * Fix tinydb_.py * Refresh TinyDB catalog fields * s/self.fields_/self._fields/g * Update BaseProvider.fields based on feedback * Fix flake8 --- docs/source/plugins.rst | 5 ++- pygeoapi/provider/base.py | 18 +++++++- pygeoapi/provider/csv_.py | 53 ++++++++++++------------ pygeoapi/provider/csw_facade.py | 19 +++++---- pygeoapi/provider/elasticsearch_.py | 64 +++++++++++++++-------------- pygeoapi/provider/erddap.py | 25 +++++------ pygeoapi/provider/esri.py | 6 +-- pygeoapi/provider/geojson.py | 37 +++++++++-------- pygeoapi/provider/mongo.py | 33 ++++++++------- pygeoapi/provider/ogr.py | 60 +++++++++++++-------------- pygeoapi/provider/oracle.py | 6 +-- pygeoapi/provider/postgresql.py | 23 +++++------ pygeoapi/provider/rasterio_.py | 41 +++++++++--------- pygeoapi/provider/sensorthings.py | 8 ++-- pygeoapi/provider/socrata.py | 6 +-- pygeoapi/provider/sqlite.py | 6 +-- pygeoapi/provider/tinydb_.py | 62 ++++++++++++++-------------- pygeoapi/provider/xarray_.py | 31 +++++++------- pygeoapi/provider/xarray_edr.py | 4 +- 19 files changed, 263 insertions(+), 244 deletions(-) diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 0a4138498..76248fbcb 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -240,15 +240,16 @@ The below template provides a minimal example (let's call the file ``mycoolraste super().__init__(provider_def) self.num_bands = 4 self.axes = ['Lat', 'Long'] - self.fields = self.get_fields() + self.get_fields() def get_fields(self): # generate a JSON Schema of coverage band metadata - return { + self._fields = { 'b1': { 'type': 'number' } } + return self._fields def query(self, bands=[], subsets={}, format_='json', **kwargs): # process bands and subsets parameters diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index 39dcc2a4a..c81c2b4f4 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -73,7 +73,7 @@ def __init__(self, provider_def): self.title_field = provider_def.get('title_field') self.properties = provider_def.get('properties', []) self.file_types = provider_def.get('file_types', []) - self.fields = {} + self._fields = {} self.filename = None # for coverage providers @@ -92,6 +92,22 @@ def get_fields(self): raise NotImplementedError() + @property + def fields(self) -> dict: + """ + Store provider field information (names, types) + + Example response: {'field1': {'type': 'string'}} + + :returns: dict of dicts (field names and their + associated JSON Schema definitions) + """ + + if hasattr(self, '_fields'): + return self._fields + else: + return self.get_fields() + def get_schema(self, schema_type: SchemaType = SchemaType.item): """ Get provider schema model diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index a8800ff0f..15aad908a 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -54,7 +54,7 @@ def __init__(self, provider_def): super().__init__(provider_def) self.geometry_x = provider_def['geometry']['x_field'] self.geometry_y = provider_def['geometry']['y_field'] - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -62,32 +62,31 @@ def get_fields(self): :returns: dict of fields """ - - LOGGER.debug('Treating all columns as string types') - with open(self.data) as ff: - LOGGER.debug('Serializing DictReader') - data_ = csv.DictReader(ff) - fields = {} - - row = next(data_) - - for key, value in row.items(): - LOGGER.debug(f'key: {key}, value: {value}') - value2 = get_typed_value(value) - if key in [self.geometry_x, self.geometry_y]: - continue - if key == self.id_field: - type_ = 'string' - elif isinstance(value2, float): - type_ = 'number' - elif isinstance(value2, int): - type_ = 'integer' - else: - type_ = 'string' - - fields[key] = {'type': type_} - - return fields + if not self._fields: + LOGGER.debug('Treating all columns as string types') + with open(self.data) as ff: + LOGGER.debug('Serializing DictReader') + data_ = csv.DictReader(ff) + + row = next(data_) + + for key, value in row.items(): + LOGGER.debug(f'key: {key}, value: {value}') + value2 = get_typed_value(value) + if key in [self.geometry_x, self.geometry_y]: + continue + if key == self.id_field: + type_ = 'string' + elif isinstance(value2, float): + type_ = 'number' + elif isinstance(value2, int): + type_ = 'integer' + else: + type_ = 'string' + + self._fields[key] = {'type': type_} + + return self._fields def _load(self, offset=0, limit=10, resulttype='results', identifier=None, bbox=[], datetime_=None, properties=[], diff --git a/pygeoapi/provider/csw_facade.py b/pygeoapi/provider/csw_facade.py index cfb5bb826..69cd0ddee 100644 --- a/pygeoapi/provider/csw_facade.py +++ b/pygeoapi/provider/csw_facade.py @@ -69,7 +69,8 @@ def __init__(self, provider_def): 'language': ('dc:language', 'language') } - self.fields = self.get_fields() + self._fields = {} + self.get_fields() def get_fields(self): """ @@ -78,17 +79,17 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - date_fields = ['date', 'created', 'updated'] + if not self._fields: + date_fields = ['date', 'created', 'updated'] - for key in self.record_mappings.keys(): - LOGGER.debug(f'key: {key}') - fields[key] = {'type': 'string'} + for key in self.record_mappings.keys(): + LOGGER.debug(f'key: {key}') + self._fields[key] = {'type': 'string'} - if key in date_fields: - fields[key]['format'] = 'date-time' + if key in date_fields: + self._fields[key]['format'] = 'date-time' - return fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/elasticsearch_.py b/pygeoapi/provider/elasticsearch_.py index cf5b4fd99..5cd90a8bb 100644 --- a/pygeoapi/provider/elasticsearch_.py +++ b/pygeoapi/provider/elasticsearch_.py @@ -87,7 +87,7 @@ def __init__(self, provider_def): LOGGER.debug('Grabbing field information') try: - self.fields = self.get_fields() + self.get_fields() except exceptions.NotFoundError as err: LOGGER.error(err) raise ProviderQueryError(err) @@ -98,38 +98,40 @@ def get_fields(self): :returns: dict of fields """ + if not self._fields: + ii = self.es.indices.get(index=self.index_name, + allow_no_indices=False) - fields_ = {} - ii = self.es.indices.get(index=self.index_name, allow_no_indices=False) - - LOGGER.debug(f'Response: {ii}') - try: - if '*' not in self.index_name: - p = ii[self.index_name]['mappings']['properties']['properties'] - else: - LOGGER.debug('Wildcard index; setting from first match') - index_name_ = list(ii.keys())[0] - p = ii[index_name_]['mappings']['properties']['properties'] - except KeyError: - LOGGER.warning('Trying for alias') - alias_name = next(iter(ii)) - p = ii[alias_name]['mappings']['properties']['properties'] - except IndexError: - LOGGER.warning('could not get fields; returning empty set') - return {} - - for k, v in p['properties'].items(): - if 'type' in v: - if v['type'] == 'text': - fields_[k] = {'type': 'string'} - elif v['type'] == 'date': - fields_[k] = {'type': 'string', 'format': 'date'} - elif v['type'] in ('float', 'long'): - fields_[k] = {'type': 'number', 'format': v['type']} + LOGGER.debug(f'Response: {ii}') + try: + if '*' not in self.index_name: + mappings = ii[self.index_name]['mappings'] + p = mappings['properties']['properties'] else: - fields_[k] = {'type': v['type']} - - return fields_ + LOGGER.debug('Wildcard index; setting from first match') + index_name_ = list(ii.keys())[0] + p = ii[index_name_]['mappings']['properties']['properties'] + except KeyError: + LOGGER.warning('Trying for alias') + alias_name = next(iter(ii)) + p = ii[alias_name]['mappings']['properties']['properties'] + except IndexError: + LOGGER.warning('could not get fields; returning empty set') + return {} + + for k, v in p['properties'].items(): + if 'type' in v: + if v['type'] == 'text': + self._fields[k] = {'type': 'string'} + elif v['type'] == 'date': + self._fields[k] = {'type': 'string', 'format': 'date'} + elif v['type'] in ('float', 'long'): + self._fields[k] = {'type': 'number', + 'format': v['type']} + else: + self._fields[k] = {'type': v['type']} + + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/erddap.py b/pygeoapi/provider/erddap.py index 392ad24ec..2fc71c064 100644 --- a/pygeoapi/provider/erddap.py +++ b/pygeoapi/provider/erddap.py @@ -62,24 +62,25 @@ def __init__(self, provider_def): LOGGER.debug('Setting provider query filters') self.filters = self.options.get('filters') - self.fields = self.get_fields() + self.get_fields() def get_fields(self): - LOGGER.debug('Fetching one feature for field definitions') - properties = self.query(limit=1)['features'][0]['properties'] + if not self._fields: + LOGGER.debug('Fetching one feature for field definitions') + properties = self.query(limit=1)['features'][0]['properties'] - for key, value in properties.items(): - LOGGER.debug(f'Field: {key}={value}') + for key, value in properties.items(): + LOGGER.debug(f'Field: {key}={value}') - data_type = type(value).__name__ + data_type = type(value).__name__ - if data_type == 'str': - data_type = 'string' - if data_type == 'float': - data_type = 'number' - properties[key] = {'type': data_type} + if data_type == 'str': + data_type = 'string' + if data_type == 'float': + data_type = 'number' + self._fields[key] = {'type': data_type} - return properties + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/esri.py b/pygeoapi/provider/esri.py index 0d22a8805..8179a705b 100644 --- a/pygeoapi/provider/esri.py +++ b/pygeoapi/provider/esri.py @@ -76,7 +76,7 @@ def get_fields(self): :returns: `dict` of fields """ - if not self.fields: + if not self._fields: # Load fields params = {'f': 'pjson'} resp = self.get_response(self.data, params=params) @@ -102,9 +102,9 @@ def get_fields(self): raise ProviderTypeError(msg) for _ in resp['fields']: - self.fields.update({_['name']: {'type': _['type']}}) + self._fields.update({_['name']: {'type': _['type']}}) - return self.fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index 180cf0746..257166a7c 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -68,7 +68,7 @@ def __init__(self, provider_def): """initializer""" super().__init__(provider_def) - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -77,23 +77,24 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - LOGGER.debug('Treating all columns as string types') - if os.path.exists(self.data): - with open(self.data) as src: - data = json.loads(src.read()) - for key, value in data['features'][0]['properties'].items(): - if isinstance(value, float): - type_ = 'number' - elif isinstance(value, int): - type_ = 'integer' - else: - type_ = 'string' - - fields[key] = {'type': type_} - else: - LOGGER.warning(f'File {self.data} does not exist.') - return fields + if not self._fields: + LOGGER.debug('Treating all columns as string types') + if os.path.exists(self.data): + with open(self.data) as src: + data = json.loads(src.read()) + for key, value in data['features'][0]['properties'].items(): + if isinstance(value, float): + type_ = 'number' + elif isinstance(value, int): + type_ = 'integer' + else: + type_ = 'string' + + self._fields[key] = {'type': type_} + else: + LOGGER.warning(f'File {self.data} does not exist.') + + return self._fields def _load(self, skip_geometry=None, properties=[], select_properties=[]): """Load and validate the source GeoJSON file diff --git a/pygeoapi/provider/mongo.py b/pygeoapi/provider/mongo.py index 1656de496..ca258018c 100644 --- a/pygeoapi/provider/mongo.py +++ b/pygeoapi/provider/mongo.py @@ -66,7 +66,7 @@ def __init__(self, provider_def): self.featuredb = dbclient.get_default_database() self.collection = provider_def['collection'] self.featuredb[self.collection].create_index([("geometry", GEOSPHERE)]) - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -75,25 +75,24 @@ def get_fields(self): :returns: dict of fields """ - pipeline = [ - {"$project": {"properties": 1}}, - {"$unwind": "$properties"}, - {"$group": {"_id": "$properties", "count": {"$sum": 1}}}, - {"$project": {"_id": 1}} - ] + if not self._fields: + pipeline = [ + {"$project": {"properties": 1}}, + {"$unwind": "$properties"}, + {"$group": {"_id": "$properties", "count": {"$sum": 1}}}, + {"$project": {"_id": 1}} + ] - result = list(self.featuredb[self.collection].aggregate(pipeline)) + result = list(self.featuredb[self.collection].aggregate(pipeline)) - # prepare a dictionary with fields - # set the field type to 'string'. - # by operating without a schema, mongo can query any data type. - fields = {} + # prepare a dictionary with fields + # set the field type to 'string'. + # by operating without a schema, mongo can query any data type. + for i in result: + for key in result[0]['_id'].keys(): + self._fields[key] = {'type': 'string'} - for i in result: - for key in result[0]['_id'].keys(): - fields[key] = {'type': 'string'} - - return fields + return self._fields def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1, skip_geometry=False): diff --git a/pygeoapi/provider/ogr.py b/pygeoapi/provider/ogr.py index 1584e6dcc..3132c2cb8 100644 --- a/pygeoapi/provider/ogr.py +++ b/pygeoapi/provider/ogr.py @@ -188,7 +188,7 @@ def __init__(self, provider_def): self.conn = None LOGGER.debug('Grabbing field information') - self.fields = self.get_fields() + self.get_fields() def _list_open_options(self): return [ @@ -260,43 +260,43 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - try: - layer_defn = self._get_layer().GetLayerDefn() - for fld in range(layer_defn.GetFieldCount()): - field_defn = layer_defn.GetFieldDefn(fld) - fieldName = field_defn.GetName() - fieldTypeCode = field_defn.GetType() - fieldType = field_defn.GetFieldTypeName(fieldTypeCode) + if not self._fields: + try: + layer_defn = self._get_layer().GetLayerDefn() + for fld in range(layer_defn.GetFieldCount()): + field_defn = layer_defn.GetFieldDefn(fld) + fieldName = field_defn.GetName() + fieldTypeCode = field_defn.GetType() + fieldType = field_defn.GetFieldTypeName(fieldTypeCode) - fieldName2 = fieldType.lower() + fieldName2 = fieldType.lower() - if fieldName2 == 'integer64': - fieldName2 = 'integer' - elif fieldName2 == 'real': - fieldName2 = 'number' + if fieldName2 == 'integer64': + fieldName2 = 'integer' + elif fieldName2 == 'real': + fieldName2 = 'number' - fields[fieldName] = {'type': fieldName2} + self._fields[fieldName] = {'type': fieldName2} - if fieldName2 == 'datetime': - fields[fieldName] = { - 'type': 'string', - 'format': 'date-time' - } + if fieldName2 == 'datetime': + self._fields[fieldName] = { + 'type': 'string', + 'format': 'date-time' + } - # fieldWidth = layer_defn.GetFieldDefn(fld).GetWidth() - # GetPrecision = layer_defn.GetFieldDefn(fld).GetPrecision() + # fieldWidth = layer_defn.GetFieldDefn(fld).GetWidth() + # GetPrecision = layer_defn.GetFieldDefn(fld).GetPrecision() # noqa - except RuntimeError as err: - LOGGER.error(err) - raise ProviderConnectionError(err) - except Exception as err: - LOGGER.error(err) + except RuntimeError as err: + LOGGER.error(err) + raise ProviderConnectionError(err) + except Exception as err: + LOGGER.error(err) - finally: - self._close() + finally: + self._close() - return fields + return self._fields def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], diff --git a/pygeoapi/provider/oracle.py b/pygeoapi/provider/oracle.py index 16ce2ced5..4bedde773 100644 --- a/pygeoapi/provider/oracle.py +++ b/pygeoapi/provider/oracle.py @@ -435,12 +435,12 @@ def get_fields(self): """ LOGGER.debug("Get available fields/properties") - if not self.fields: + if not self._fields: with DatabaseConnection( self.conn_dic, self.table, properties=self.properties ) as db: - self.fields = db.fields - return self.fields + self._fields = db.fields + return self._fields def _get_where_clauses( self, diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index bf0a5799f..d56bb7878 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -124,7 +124,7 @@ def __init__(self, provider_def): ) LOGGER.debug(f'DB connection: {repr(self._engine.url)}') - self.fields = self.get_fields() + self.get_fields() def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], @@ -204,8 +204,6 @@ def get_fields(self): LOGGER.debug('Get available fields/properties') - fields = {} - # sql-schema only allows these types, so we need to map from sqlalchemy # string, number, integer, object, array, boolean, null, # https://json-schema.org/understanding-json-schema/reference/type.html @@ -248,17 +246,18 @@ def _column_format_to_json_schema_format(column_type): LOGGER.debug('No string format detected') return None - for column in self.table_model.__table__.columns: - LOGGER.debug(f'Testing {column.name}') - if column.name == self.geom: - continue + if not self._fields: + for column in self.table_model.__table__.columns: + LOGGER.debug(f'Testing {column.name}') + if column.name == self.geom: + continue - fields[str(column.name)] = { - 'type': _column_type_to_json_schema_type(column.type), - 'format': _column_format_to_json_schema_format(column.type) - } + self._fields[str(column.name)] = { + 'type': _column_type_to_json_schema_type(column.type), + 'format': _column_format_to_json_schema_format(column.type) + } - return fields + return self._fields def get(self, identifier, crs_transform_spec=None, **kwargs): """ diff --git a/pygeoapi/provider/rasterio_.py b/pygeoapi/provider/rasterio_.py index 2bf25e3fb..69d2dbca8 100644 --- a/pygeoapi/provider/rasterio_.py +++ b/pygeoapi/provider/rasterio_.py @@ -59,38 +59,37 @@ def __init__(self, provider_def): self.axes = self._coverage_properties['axes'] self.crs = self._coverage_properties['bbox_crs'] self.num_bands = self._coverage_properties['num_bands'] - self.fields = self.get_fields() + self.get_fields() self.native_format = provider_def['format']['name'] except Exception as err: LOGGER.warning(err) raise ProviderConnectionError(err) def get_fields(self): - fields = {} + if not self._fields: + for i, dtype in zip(self._data.indexes, self._data.dtypes): + LOGGER.debug(f'Adding field for band {i}') + i2 = str(i) - for i, dtype in zip(self._data.indexes, self._data.dtypes): - LOGGER.debug(f'Adding field for band {i}') - i2 = str(i) + parameter = _get_parameter_metadata( + self._data.profile['driver'], self._data.tags(i)) - parameter = _get_parameter_metadata( - self._data.profile['driver'], self._data.tags(i)) + name = parameter['description'] + units = parameter.get('unit_label') - name = parameter['description'] - units = parameter.get('unit_label') + dtype2 = dtype + if dtype.startswith('float'): + dtype2 = 'number' - dtype2 = dtype - if dtype.startswith('float'): - dtype2 = 'number' - - fields[i2] = { - 'title': name, - 'type': dtype2, - '_meta': self._data.tags(i) - } - if units is not None: - fields[i2]['x-ogc-unit'] = units + self._fields[i2] = { + 'title': name, + 'type': dtype2, + '_meta': self._data.tags(i) + } + if units is not None: + self._fields[i2]['x-ogc-unit'] = units - return fields + return self._fields def query(self, properties=[], subsets={}, bbox=None, bbox_crs=4326, datetime_=None, format_='json', **kwargs): diff --git a/pygeoapi/provider/sensorthings.py b/pygeoapi/provider/sensorthings.py index 6b497b44b..c5281875c 100644 --- a/pygeoapi/provider/sensorthings.py +++ b/pygeoapi/provider/sensorthings.py @@ -150,7 +150,7 @@ def get_fields(self): :returns: dict of fields """ - if not self.fields: + if not self._fields: r = self._get_response(self._url, {'$top': 1}) try: results = r['value'][0] @@ -161,11 +161,11 @@ def get_fields(self): for (n, v) in results.items(): if isinstance(v, (int, float)) or \ (isinstance(v, (dict, list)) and n in ENTITY): - self.fields[n] = {'type': 'number'} + self._fields[n] = {'type': 'number'} elif isinstance(v, str): - self.fields[n] = {'type': 'string'} + self._fields[n] = {'type': 'string'} - return self.fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/socrata.py b/pygeoapi/provider/socrata.py index 0f402a735..9d2292549 100644 --- a/pygeoapi/provider/socrata.py +++ b/pygeoapi/provider/socrata.py @@ -75,7 +75,7 @@ def get_fields(self): :returns: dict of fields """ - if not self.fields: + if not self._fields: try: [dataset] = self.client.datasets(ids=[self.resource_id]) @@ -87,9 +87,9 @@ def get_fields(self): fields = self.properties or resource[FIELD_NAME] for field in fields: idx = resource[FIELD_NAME].index(field) - self.fields[field] = {'type': resource[DATA_TYPE][idx]} + self._fields[field] = {'type': resource[DATA_TYPE][idx]} - return self.fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/sqlite.py b/pygeoapi/provider/sqlite.py index bb046ac86..9ba796bb9 100644 --- a/pygeoapi/provider/sqlite.py +++ b/pygeoapi/provider/sqlite.py @@ -88,7 +88,7 @@ def get_fields(self): :returns: dict of fields """ - if not self.fields: + if not self._fields: results = self.cursor.execute( f'PRAGMA table_info({self.table})').fetchall() for item in results: @@ -100,9 +100,9 @@ def get_fields(self): json_type = 'string' if json_type is not None: - self.fields[item['name']] = {'type': json_type} + self._fields[item['name']] = {'type': json_type} - return self.fields + return self._fields def __get_where_clauses(self, properties=[], bbox=[]): """ diff --git a/pygeoapi/provider/tinydb_.py b/pygeoapi/provider/tinydb_.py index 393ed2108..bb86d3ae2 100644 --- a/pygeoapi/provider/tinydb_.py +++ b/pygeoapi/provider/tinydb_.py @@ -74,7 +74,7 @@ def __init__(self, provider_def): else: self.db = TinyDB(self.data) - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -83,38 +83,37 @@ def get_fields(self): :returns: dict of fields """ - fields = {} + if not self._fields: + try: + r = self.db.all()[0] + except IndexError as err: + LOGGER.debug(err) + return {} + + for key, value in r['properties'].items(): + if key not in self._excludes: + typed_value = get_typed_value(str(value)) + if isinstance(typed_value, float): + typed_value_type = 'number' + elif isinstance(typed_value, int): + typed_value_type = 'integer' + else: + typed_value_type = 'string' - try: - r = self.db.all()[0] - except IndexError as err: - LOGGER.debug(err) - return fields - - for key, value in r['properties'].items(): - if key not in self._excludes: - typed_value = get_typed_value(str(value)) - if isinstance(typed_value, float): - typed_value_type = 'number' - elif isinstance(typed_value, int): - typed_value_type = 'integer' - else: - typed_value_type = 'string' - - fields[key] = {'type': typed_value_type} + self._fields[key] = {'type': typed_value_type} - try: - LOGGER.debug('Attempting to detect date types') - _ = parse_date(value) - if len(value) > 11: - fields[key]['format'] = 'date-time' - else: - fields[key]['format'] = 'date' - except Exception: - LOGGER.debug('No date types detected') - pass + try: + LOGGER.debug('Attempting to detect date types') + _ = parse_date(value) + if len(value) > 11: + self._fields[key]['format'] = 'date-time' + else: + self._fields[key]['format'] = 'date' + except Exception: + LOGGER.debug('No date types detected') + pass - return fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', @@ -349,7 +348,10 @@ class TinyDBCatalogueProvider(TinyDBProvider): def __init__(self, provider_def): super().__init__(provider_def) + LOGGER.debug('Refreshing fields') self._excludes = ['_metadata-anytext'] + self._fields = {} + self.get_fields() def get_fields(self): fields = super().get_fields() diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index 9a82daec0..15315145f 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -88,28 +88,27 @@ def __init__(self, provider_def): self._coverage_properties['y_axis_label'], self._coverage_properties['time_axis_label']] - self.fields = self.get_fields() + self.get_fields() except Exception as err: LOGGER.warning(err) raise ProviderConnectionError(err) def get_fields(self): - fields = {} - - for key, value in self._data.variables.items(): - if len(value.shape) >= 3: - LOGGER.debug('Adding variable') - dtype = value.dtype - if dtype.name.startswith('float'): - dtype = 'number' - - fields[key] = { - 'type': dtype, - 'title': value.attrs['long_name'], - 'x-ogc-unit': value.attrs.get('units') - } + if not self._fields: + for key, value in self._data.variables.items(): + if len(value.shape) >= 3: + LOGGER.debug('Adding variable') + dtype = value.dtype + if dtype.name.startswith('float'): + dtype = 'number' + + self._fields[key] = { + 'type': dtype, + 'title': value.attrs['long_name'], + 'x-ogc-unit': value.attrs.get('units') + } - return fields + return self._fields def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, datetime_=None, format_='json', **kwargs): diff --git a/pygeoapi/provider/xarray_edr.py b/pygeoapi/provider/xarray_edr.py index e724f1d64..546b229ac 100644 --- a/pygeoapi/provider/xarray_edr.py +++ b/pygeoapi/provider/xarray_edr.py @@ -109,7 +109,7 @@ def position(self, **kwargs): try: if select_properties: - self.fields = {k: v for k, v in self.fields.items() if k in select_properties} # noqa + self._fields = {k: v for k, v in self._fields.items() if k in select_properties} # noqa data = self._data[[*select_properties]] else: data = self._data @@ -206,7 +206,7 @@ def cube(self, **kwargs): LOGGER.debug(f'query parameters: {query_params}') try: if select_properties: - self.fields = {k: v for k, v in self.fields.items() if k in select_properties} # noqa + self._fields = {k: v for k, v in self._fields.items() if k in select_properties} # noqa data = self._data[[*select_properties]] else: data = self._data From bbb5035508e8c0aae1a793dc47e358b1fab85563 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:06:23 -0400 Subject: [PATCH 20/74] Render covjson and geojson in EDR HTML view (#1749) --- pygeoapi/templates/collections/edr/query.html | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pygeoapi/templates/collections/edr/query.html b/pygeoapi/templates/collections/edr/query.html index 7d90dba55..af426c766 100644 --- a/pygeoapi/templates/collections/edr/query.html +++ b/pygeoapi/templates/collections/edr/query.html @@ -14,9 +14,15 @@ + {% if data.type == "Coverage" or data.type == "CoverageCollection" %} + {% elif data.type == "Feature" or data.type == "FeatureCollection" %} + + + + {% endif %} {% endblock %} {% block body %} @@ -36,6 +42,7 @@ } )); + {% if data.type == "Coverage" or data.type == "CoverageCollection" %} var layers = L.control.layers(null, null, {collapsed: false}).addTo(map) CovJSON.read(JSON.parse('{{ data | to_json | safe }}')).then(function (cov) { @@ -57,7 +64,28 @@ layers: [layer] }).setLatLng(e.latlng).openOn(map) }) + {% elif data.type == "Feature" or data.type == "FeatureCollection" %} + var geojson_data = {{ data | to_json | safe }}; + var items = new L.GeoJSON(geojson_data, { + onEachFeature: function (feature, layer) { + var html = '' + {% if data['title_field'] %} feature['properties']['{{ data['title_field'] }}'] {% else %} feature.id {% endif %} + ''; + layer.bindPopup(html); + } + }); + {% if data.type == "FeatureCollection" and data['features'][0]['geometry']['type'] == 'Point' %} + var markers = L.markerClusterGroup({ + disableClusteringAtZoom: 9, + chunkedLoading: true, + chunkInterval: 500, + }); + markers.clearLayers().addLayer(items); + map.addLayer(markers); + {% else %} + map.addLayer(items); + {% endif %} + map.fitBounds(items.getBounds(), {maxZoom: 15}); + {% endif %} {% endif %} {% endblock %} From d600f552145a2388189140ee1b707a48949d7906 Mon Sep 17 00:00:00 2001 From: francescoingv <9592487+francescoingv@users.noreply.github.com> Date: Wed, 24 Jul 2024 12:12:43 +0200 Subject: [PATCH 21/74] Update test_postgresql_manager.py (#1751) * Update test_postgresql_manager.py Test issue #1750 is fixed * Update test_postgresql_manager.py * Update test_postgresql_manager.py * Update postgresql.py * Update requirements-manager.txt * Update postgresql.py * Update test_postgresql_manager.py * Update test_postgresql_manager.py * Update test_postgresql_manager.py * Update postgresql.py Initialization problem with search path in case of exception. * Update requirements-manager.txt --- pygeoapi/process/manager/postgresql.py | 11 +++++++++-- tests/test_postgresql_manager.py | 9 +++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pygeoapi/process/manager/postgresql.py b/pygeoapi/process/manager/postgresql.py index 8215b1241..30b7b8df6 100644 --- a/pygeoapi/process/manager/postgresql.py +++ b/pygeoapi/process/manager/postgresql.py @@ -46,6 +46,7 @@ from typing import Any, Tuple from sqlalchemy import insert, update, delete +from sqlalchemy.engine import make_url from sqlalchemy.orm import Session from pygeoapi.process.base import ( @@ -83,12 +84,18 @@ def __init__(self, manager_def: dict): self.db_search_path = tuple(self.connection.get('search_path', ['public'])) except Exception: - self.db_search_path = 'public' + self.db_search_path = ('public',) try: LOGGER.debug('Connecting to database') if isinstance(self.connection, str): - self._engine = get_engine(self.connection) + _url = make_url(self.connection) + self._engine = get_engine( + _url.host, + _url.port, + _url.database, + _url.username, + _url.password) else: self._engine = get_engine(**self.connection) except Exception as err: diff --git a/tests/test_postgresql_manager.py b/tests/test_postgresql_manager.py index 43464f6d9..db20fba46 100644 --- a/tests/test_postgresql_manager.py +++ b/tests/test_postgresql_manager.py @@ -103,6 +103,15 @@ def _create_delete_request(job_id, locales): return APIRequest.with_data(req, locales) +def test_api_connection_rfc3986(config, openapi): + connection = config['server']['manager']['connection'] + connection_string = ( + f"postgresql://{connection['user']}:{connection['password']}" + f"@{connection['host']}:{connection['port']}/{connection['database']}") + config['server']['manager']['connection'] = connection_string + API(config, openapi) + + def test_job_sync_hello_world(api_, config): """ Create a new job for hello-world, From 7bb7b380160b541f718fbabd1f1a9b66ac4affdf Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 24 Jul 2024 08:56:50 -0400 Subject: [PATCH 22/74] fix pagination for features/records (#1658) (#1754) * fix pagination for features/records (#1658) * remove given tests already existing --- pygeoapi/api/itemtypes.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 33cb517b3..b43d75964 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -531,17 +531,23 @@ def get_collection_items( 'href': f'{uri}?offset={prev}{serialized_query_params}' }) - if 'numberMatched' in content: - if content['numberMatched'] > (limit + offset): - next_ = offset + limit - next_href = f'{uri}?offset={next_}{serialized_query_params}' - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'next', - 'title': l10n.translate('Items (next)', request.locale), - 'href': next_href - }) + next_link = False + + if content.get('numberMatched', -1) > (limit + offset): + next_link = True + elif len(content['features']) == limit: + next_link = True + + if next_link: + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'next', + 'title': l10n.translate('Items (next)', request.locale), + 'href': next_href + }) content['links'].append( { From 0281732c5c1a04d6f698223f62d83697e6bd0ea6 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 24 Jul 2024 11:08:15 -0400 Subject: [PATCH 23/74] add CORS expose headers setting (#1689) (#1755) --- pygeoapi/flask_app.py | 2 +- pygeoapi/starlette_app.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index cc140e276..4e4f0097e 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -77,7 +77,7 @@ if CONFIG['server'].get('cors', False): try: from flask_cors import CORS - CORS(APP) + CORS(APP, CORS_EXPOSE_HEADERS=['*']) except ModuleNotFoundError: print('Python package flask-cors required for CORS support') diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 9f4e33f79..00ed727d2 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -746,6 +746,7 @@ async def __call__(self, scope: Scope, CORSMiddleware, allow_origins=['*'], allow_methods=['*'], + expose_headers=['*'] ) try: From af8483a25b8c680b662dfe45fde14d7ae0fc4842 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 24 Jul 2024 14:58:27 -0400 Subject: [PATCH 24/74] OAProc: handle binary data when response: document (#1285) (#1756) --- .../data-publishing/ogcapi-processes.rst | 61 ++++++++++++++++--- docs/source/plugins.rst | 2 + pygeoapi/api/processes.py | 6 +- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-processes.rst b/docs/source/data-publishing/ogcapi-processes.rst index e62d3ff0c..6cd214c5c 100644 --- a/docs/source/data-publishing/ogcapi-processes.rst +++ b/docs/source/data-publishing/ogcapi-processes.rst @@ -14,15 +14,47 @@ The pygeoapi offers two processes: a default ``hello-world`` process which allow Configuration ------------- +The below configuration is an example of a process defined within the pygeoapi internal plugin registry: + .. code-block:: yaml processes: - - # enabled by default + # enabled by default hello-world: processor: name: HelloWorld +The below configuration is an example of a process defined as part of a custom Python process: + +.. code-block:: yaml + + processes: + # enabled by default + hello-world: + processor: + # refer to a process in the standard PYTHONPATH + # e.g. my_package/my_module/my_file.py (class MyProcess) + # the MyProcess class must subclass from pygeoapi.process.base.BaseProcessor + name: my_package.my_module.my_file.MyProcess + +See :ref:`example-custom-pygeoapi-processing-plugin` for processing plugin examples. + +Processing and response handling +-------------------------------- + +pygeoapi processing plugins must return a tuple of media type and native outputs. Multipart +responses are not supported at this time, and it is up to the process plugin implementor to return a single +payload defining multiple artifacts (or references to them). + +By default (or via the OGC API - Processes ``response: raw`` execution parameter), pygeoapi provides +processing responses in their native encoding and media type, as defined by a given +plugin (which needs to set the response content type and payload accordingly). + +pygeoapi also supports a JSON-based response type (via the OGC API - Processes ``response: document`` +execution parameter). When this mode is requested, the response will always be a JSON encoding, embedding +the resulting payload (part of which may be Base64 encoded for binary data, for example). + + Asynchronous support -------------------- @@ -33,15 +65,27 @@ an asynchronous design pattern. This means that when a job is submitted in asyn mode, the server responds immediately with a reference to the job, which allows the client to periodically poll the server for the processing status of a given job. -pygeoapi provides asynchronous support by providing a 'manager' concept which, well, +In keeping with the OGC API - Processes specification, asynchronous process execution +can be requested by including the ``Prefer: respond-async`` HTTP header in the request. + +Job management is required for asynchronous functionality. + +Job management +-------------- + +pygeoapi provides job management by providing a 'manager' concept which, well, manages job execution. The manager concept is implemented as part of the pygeoapi :ref:`plugins` architecture. pygeoapi provides a default manager implementation based on `TinyDB`_ for simplicity. Custom manager plugins can be developed for more advanced job management capabilities (e.g. Kubernetes, databases, etc.). -In keeping with the OGC API - Processes specification, asynchronous process execution -can be requested by including the ``Prefer: respond-async`` HTTP header in the request +Job managers +------------ + +TinyDB +^^^^^^ +TinyDB is the default job manager for pygeoapi when enabled. .. code-block:: yaml @@ -52,7 +96,8 @@ can be requested by including the ``Prefer: respond-async`` HTTP header in the r output_dir: /tmp/ MongoDB -------- +^^^^^^^ + As an alternative to the default, a manager employing `MongoDB`_ can be used. The connection to a `MongoDB`_ instance must be provided in the configuration. `MongoDB`_ uses ``localhost`` and port ``27017`` by default. Jobs are stored in a collection named @@ -66,9 +111,9 @@ The connection to a `MongoDB`_ instance must be provided in the configuration. connection: mongodb://host:port output_dir: /tmp/ - PostgreSQL ----------- +^^^^^^^^^^ + As another alternative to the default, a manager employing `PostgreSQL`_ can be used. The connection to a `PostgreSQL`_ database must be provided in the configuration. `PostgreSQL`_ uses ``localhost`` and port ``5432`` by default. Jobs are stored in a table named ``jobs``. diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 76248fbcb..cadf93a73 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -273,6 +273,8 @@ implementation. Each base class documents the functions, arguments and return types required for implementation. +.. _example-custom-pygeoapi-processing-plugin: + Example: custom pygeoapi processing plugin ------------------------------------------ diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 236f16e16..d547a984e 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -379,6 +379,8 @@ def execute_process(api: API, request: APIRequest, requested_outputs = data.get('outputs') LOGGER.debug(f'outputs: {requested_outputs}') + response_requested = data.get('response', 'raw') + subscriber = None subscriber_dict = data.get('subscriber') if subscriber_dict: @@ -420,7 +422,7 @@ def execute_process(api: API, request: APIRequest, if status == JobStatus.failed: response = outputs - if data.get('response', 'raw') == 'raw': + if response_requested == 'raw': headers['Content-Type'] = mime_type response = outputs elif status not in (JobStatus.failed, JobStatus.accepted): @@ -433,7 +435,7 @@ def execute_process(api: API, request: APIRequest, else: http_status = HTTPStatus.OK - if mime_type == 'application/json': + if mime_type == 'application/json' or response_requested == 'document': response2 = to_json(response, api.pretty_print) else: response2 = response From 6c538ca33008bcf618597a96dbe930f0525c7be4 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 24 Jul 2024 16:11:44 -0400 Subject: [PATCH 25/74] raise error for collections without queryables (#1757) --- pygeoapi/api/itemtypes.py | 6 ++++++ tests/api/test_itemtypes.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index b43d75964..1fa70c235 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -134,6 +134,12 @@ def get_collection_queryables(api: API, request: Union[APIRequest, Any], LOGGER.debug('Loading record provider') p = load_plugin('provider', get_provider_by_type( api.config['resources'][dataset]['providers'], 'record')) + finally: + msg = 'queryables not available for this collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableError', msg) + except ProviderGenericError as err: return api.get_exception( err.http_status_code, headers, request.format, diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py index 20822836c..283bb0c23 100644 --- a/tests/api/test_itemtypes.py +++ b/tests/api/test_itemtypes.py @@ -62,6 +62,11 @@ def test_get_collection_queryables(config, api_): api_, req, 'notfound') assert code == HTTPStatus.NOT_FOUND + req = mock_api_request() + rsp_headers, code, response = get_collection_queryables( + api_, req, 'mapserver_world_map') + assert code == HTTPStatus.BAD_REQUEST + req = mock_api_request({'f': 'html'}) rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] From 3adfdb2341fe70e790011dfe3bdc9de63e777820 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:25:23 -0400 Subject: [PATCH 26/74] Describe required collection level metadata about EDR Queries (#1744) * Add data_queries to describe EDR Queries * Fix flake8 * Use covjson media type * Update pygeoapi-config-0.x.yml Update schema definition to match https://schemas.opengis.net/ogcapi/edr/1.1/openapi/schemas/collections/extent.yaml * Add data_queries to describe EDR Queries * Fix flake8 * Use covjson media type * Update pygeoapi-config-0.x.yml Update schema definition to match https://schemas.opengis.net/ogcapi/edr/1.1/openapi/schemas/collections/extent.yaml * Enable query type registration * Update base_edr.py * Add GeoJSON as a valid response type * Preserve query_types as list * Revert changes to min required by EDR spec --- pygeoapi/api/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 793b5969d..65d85587d 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -1209,6 +1209,7 @@ def describe_collections(self, request: Union[APIRequest, Any], if edr: # TODO: translate LOGGER.debug('Adding EDR links') + collection['data_queries'] = {} parameters = p.get_fields() if parameters: collection['parameter_names'] = {} @@ -1229,6 +1230,14 @@ def describe_collections(self, request: Union[APIRequest, Any], } for qt in p.get_query_types(): + data_query = { + 'link': { + 'href': f'{self.get_collections_url()}/{k}/{qt}', + 'rel': 'data' + } + } + collection['data_queries'][qt] = data_query + title1 = l10n.translate('query for this collection as JSON', request.locale) # noqa title1 = f'{qt} {title1}' title2 = l10n.translate('query for this collection as HTML', request.locale) # noqa From b2a8e0678d95969feab2118cc2467bbffde5f94b Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 24 Jul 2024 18:26:00 -0400 Subject: [PATCH 27/74] safeguard OpenAPI detection on startup (#1650) (#1758) --- pygeoapi/openapi.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 274e19dad..3cf5c0b8f 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -903,6 +903,17 @@ def load_openapi_document() -> dict: pygeoapi_openapi = os.environ.get('PYGEOAPI_OPENAPI') + if pygeoapi_openapi is None: + msg = 'PYGEOAPI_OPENAPI environment not set' + LOGGER.error(msg) + raise RuntimeError(msg) + + if not os.path.exists(pygeoapi_openapi): + msg = (f'OpenAPI document {pygeoapi_openapi} does not exist. ' + 'Please generate before starting pygeoapi') + LOGGER.error(msg) + raise RuntimeError(msg) + with open(pygeoapi_openapi, encoding='utf8') as ff: if pygeoapi_openapi.endswith(('.yaml', '.yml')): openapi_ = yaml_load(ff) From 86390a6f12ff51f6b3f0947a82e14bca1166c7ac Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 24 Jul 2024 21:27:04 -0400 Subject: [PATCH 28/74] OAProc: fix response: document encoding for results (#1579) (#1759) --- pygeoapi/api/processes.py | 16 ++++-- pygeoapi/process/manager/base.py | 93 ++++++++++++++++++++----------- pygeoapi/process/manager/dummy.py | 26 +++++++-- pygeoapi/util.py | 5 ++ tests/api/test_processes.py | 14 +++++ 5 files changed, 113 insertions(+), 41 deletions(-) diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index d547a984e..e95d90a13 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -379,7 +379,7 @@ def execute_process(api: API, request: APIRequest, requested_outputs = data.get('outputs') LOGGER.debug(f'outputs: {requested_outputs}') - response_requested = data.get('response', 'raw') + requested_response = data.get('response', 'raw') subscriber = None subscriber_dict = data.get('subscriber') @@ -409,10 +409,14 @@ def execute_process(api: API, request: APIRequest, result = api.manager.execute_process( process_id, data_dict, execution_mode=execution_mode, requested_outputs=requested_outputs, - subscriber=subscriber) + subscriber=subscriber, + requested_response=requested_response) job_id, mime_type, outputs, status, additional_headers = result headers.update(additional_headers or {}) - headers['Location'] = f'{api.base_url}/jobs/{job_id}' + + if api.manager.is_async: + headers['Location'] = f'{api.base_url}/jobs/{job_id}' + except ProcessorExecuteError as err: return api.get_exception( err.http_status_code, headers, @@ -422,11 +426,11 @@ def execute_process(api: API, request: APIRequest, if status == JobStatus.failed: response = outputs - if response_requested == 'raw': + if requested_response == 'raw': headers['Content-Type'] = mime_type response = outputs elif status not in (JobStatus.failed, JobStatus.accepted): - response['outputs'] = [outputs] + response = outputs if status == JobStatus.accepted: http_status = HTTPStatus.CREATED @@ -435,7 +439,7 @@ def execute_process(api: API, request: APIRequest, else: http_status = HTTPStatus.OK - if mime_type == 'application/json' or response_requested == 'document': + if mime_type == 'application/json' or requested_response == 'document': response2 = to_json(response, api.pretty_print) else: response2 = response diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index 548883b1d..f0d31482e 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -54,6 +54,7 @@ JobStatus, ProcessExecutionMode, RequestedProcessExecutionMode, + RequestedResponse, Subscriber ) @@ -187,6 +188,7 @@ def _execute_handler_async(self, p: BaseProcessor, job_id: str, data_dict: dict, requested_outputs: Optional[dict] = None, subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa ) -> Tuple[str, None, JobStatus]: """ This private execution handler executes a process in a background @@ -197,27 +199,34 @@ def _execute_handler_async(self, p: BaseProcessor, job_id: str, :param p: `pygeoapi.process` object :param job_id: job identifier :param data_dict: `dict` of data parameters - :param requested_outputs: `dict` specify the subset of required - outputs - defaults to all outputs. - The value of any key may be an object and include the property - `transmissionMode` - defaults to `value`. - Note: 'optional' is for backward compatibility. + :param requested_outputs: `dict` optionally specifying the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and + include the property `transmissionMode` + (defaults to `value`) + Note: 'optional' is for backward + compatibility. :param subscriber: optional `Subscriber` specifying callback URLs + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) :returns: tuple of None (i.e. initial response payload) and JobStatus.accepted (i.e. initial job status) """ - _process = dummy.Process( - target=self._execute_handler_sync, - args=(p, job_id, data_dict, requested_outputs, subscriber) - ) + + args = (p, job_id, data_dict, requested_outputs, subscriber, + requested_response) + + _process = dummy.Process(target=self._execute_handler_sync, args=args) _process.start() + return 'application/json', None, JobStatus.accepted def _execute_handler_sync(self, p: BaseProcessor, job_id: str, data_dict: dict, requested_outputs: Optional[dict] = None, subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa ) -> Tuple[str, Any, JobStatus]: """ Synchronous execution handler @@ -229,15 +238,27 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, :param p: `pygeoapi.process` object :param job_id: job identifier :param data_dict: `dict` of data parameters - :param requested_outputs: `dict` specify the subset of required - outputs - defaults to all outputs. - The value of any key may be an object and include the property - `transmissionMode` - defaults to `value`. - Note: 'optional' is for backward compatibility. + :param requested_outputs: `dict` optionally specifying the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and + include the property `transmissionMode` + (defaults to `value`) + Note: 'optional' is for backward + compatibility. :param subscriber: optional `Subscriber` specifying callback URLs + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) :returns: tuple of MIME type, response payload and status """ + + extra_execute_parameters = {} + + # only pass requested_outputs if supported, + # otherwise this breaks existing processes + if p.supports_outputs: + extra_execute_parameters['outputs'] = requested_outputs + self._send_in_progress_notification(subscriber) try: @@ -248,13 +269,12 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, job_filename = None current_status = JobStatus.running - jfmt, outputs = p.execute( - data_dict, - # only pass requested_outputs if supported, - # otherwise this breaks existing processes - **({'outputs': requested_outputs} - if p.supports_outputs else {}) - ) + jfmt, outputs = p.execute(data_dict, **extra_execute_parameters) + + if requested_response == RequestedResponse.document.value: + outputs = { + 'outputs': [outputs] + } self.update_job(job_id, { 'status': current_status.value, @@ -330,7 +350,8 @@ def execute_process( data_dict: dict, execution_mode: Optional[RequestedProcessExecutionMode] = None, requested_outputs: Optional[dict] = None, - subscriber: Optional[Subscriber] = None + subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa ) -> Tuple[str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -339,12 +360,17 @@ def execute_process( :param data_dict: `dict` of data parameters :param execution_mode: `str` optionally specifying sync or async processing. - :param requested_outputs: `dict` optionally specify the subset of - required outputs - defaults to all outputs. - The value of any key may be an object and include the property - `transmissionMode` - defaults to `value`. - Note: 'optional' is for backward compatibility. + :param requested_outputs: `dict` optionally specifying the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and + include the property `transmissionMode` + (default is `value`) + Note: 'optional' is for backward + compatibility. :param subscriber: `Subscriber` optionally specifying callback urls + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) + :raises UnknownProcessError: if the input process_id does not correspond to a known process @@ -356,6 +382,9 @@ def execute_process( job_id = str(uuid.uuid1()) processor = self.get_processor(process_id) processor.set_job_id(job_id) + extra_execute_handler_parameters = { + 'requested_response': requested_response + } if execution_mode == RequestedProcessExecutionMode.respond_async: job_control_options = processor.metadata.get( @@ -406,6 +435,11 @@ def execute_process( } self.add_job(job_metadata) + # only pass subscriber if supported, otherwise this breaks + # existing managers + if self.supports_subscribing: + extra_execute_handler_parameters['subscriber'] = subscriber + # TODO: handler's response could also be allowed to include more HTTP # headers mime_type, outputs, status = handler( @@ -413,10 +447,7 @@ def execute_process( job_id, data_dict, requested_outputs, - # only pass subscriber if supported, otherwise this breaks existing - # managers - **({'subscriber': subscriber} if self.supports_subscribing else {}) - ) + **extra_execute_handler_parameters) return job_id, mime_type, outputs, status, response_headers diff --git a/pygeoapi/process/manager/dummy.py b/pygeoapi/process/manager/dummy.py index 8360154a0..4a9866bc7 100644 --- a/pygeoapi/process/manager/dummy.py +++ b/pygeoapi/process/manager/dummy.py @@ -33,8 +33,9 @@ from pygeoapi.process.manager.base import BaseManager from pygeoapi.util import ( - RequestedProcessExecutionMode, JobStatus, + RequestedProcessExecutionMode, + RequestedResponse, Subscriber ) @@ -73,7 +74,8 @@ def execute_process( data_dict: dict, execution_mode: Optional[RequestedProcessExecutionMode] = None, requested_outputs: Optional[dict] = None, - subscriber: Optional[Subscriber] = None + subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa ) -> Tuple[str, str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -81,9 +83,19 @@ def execute_process( :param process_id: process identifier :param data_dict: `dict` of data parameters :param execution_mode: requested execution mode - + :param requested_outputs: `dict` optionally specify the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and include the property + `transmissionMode` - defaults to `value`. + Note: 'optional' is for backward compatibility. + :param subscriber: `Subscriber` optionally specifying callback urls + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) + + :raises UnknownProcessError: if the input process_id does not + correspond to a known process :returns: tuple of job_id, MIME type, response payload, status and - optionally additional HTTP headers to include in the + optionally additional HTTP headers to include in the final response """ @@ -111,6 +123,12 @@ def execute_process( current_status = JobStatus.failed LOGGER.exception(err) self._send_failed_notification(subscriber) + + if requested_response == RequestedResponse.document.value: + outputs = { + 'outputs': [outputs] + } + job_id = str(uuid.uuid1()) return job_id, jfmt, outputs, current_status, response_headers diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 4a1ddfd57..b0db50cd8 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -597,6 +597,11 @@ class RequestedProcessExecutionMode(Enum): respond_async = 'respond-async' +class RequestedResponse(Enum): + raw = 'raw' + document = 'document' + + class JobStatus(Enum): """ Enum for the job status options specified in the WPS 2.0 specification diff --git a/tests/api/test_processes.py b/tests/api/test_processes.py index bfd5ca98f..ee569c402 100644 --- a/tests/api/test_processes.py +++ b/tests/api/test_processes.py @@ -198,6 +198,12 @@ def test_execute_process(config, api_): 'failedUri': 'https://example.com/failed', } } + req_body_8 = { + 'inputs': { + 'name': 'Test document' + }, + 'response': 'document' + } cleanup_jobs = set() @@ -346,6 +352,14 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) + req = mock_api_request(data=req_body_8) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + response = json.loads(response) + assert code == HTTPStatus.OK + assert 'outputs' in response + assert isinstance(response['outputs'], list) + # Cleanup time.sleep(2) # Allow time for any outstanding async jobs for _, job_id in cleanup_jobs: From b8dcf6a8855b20870c4859b76e94bd0562786106 Mon Sep 17 00:00:00 2001 From: francescoingv <9592487+francescoingv@users.noreply.github.com> Date: Thu, 25 Jul 2024 18:20:08 +0200 Subject: [PATCH 29/74] Fixed typo (#1763) --- docs/source/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index b807ee961..d7b34125a 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -241,7 +241,7 @@ default. option_name: option_value hello-world: # name of process - type: collection # REQUIRED (collection, process, or stac-collection) + type: process # REQUIRED (collection, process, or stac-collection) processor: name: HelloWorld # Python path of process definition From b712cb2695d35064835bfbd4fdf9b60663a0f759 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Thu, 25 Jul 2024 14:06:51 -0400 Subject: [PATCH 30/74] Fix typo in docs (#1762) * Fix typo in docs * Fix doc x/y fields --- docs/source/data-publishing/ogcapi-coverages.rst | 2 +- docs/source/data-publishing/ogcapi-edr.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-coverages.rst b/docs/source/data-publishing/ogcapi-coverages.rst index 6e06b2e30..97230df35 100644 --- a/docs/source/data-publishing/ogcapi-coverages.rst +++ b/docs/source/data-publishing/ogcapi-coverages.rst @@ -72,8 +72,8 @@ The `Xarray`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data. data: tests/data/coads_sst.nc # optionally specify x/y/time fields, else provider will attempt # to derive automagically - x_field: lat x_field: lon + y_field: lat time_field: time format: name: netcdf diff --git a/docs/source/data-publishing/ogcapi-edr.rst b/docs/source/data-publishing/ogcapi-edr.rst index 2ad82a166..599a711b3 100644 --- a/docs/source/data-publishing/ogcapi-edr.rst +++ b/docs/source/data-publishing/ogcapi-edr.rst @@ -44,8 +44,8 @@ The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data data: tests/data/coads_sst.nc # optionally specify x/y/time fields, else provider will attempt # to derive automagically - x_field: lat x_field: lon + y_field: lat time_field: time format: name: netcdf From 0a7bb7f5f497c5f00a0439614cf299054c7eeabe Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 25 Jul 2024 14:13:20 -0400 Subject: [PATCH 31/74] fix various deprecation warnings (#1761) --- pygeoapi/api/environmental_data_retrieval.py | 4 ++-- pygeoapi/provider/xarray_edr.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index e371c4d0b..7e7ba1248 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -42,7 +42,7 @@ import logging from typing import Tuple -from shapely.errors import WKTReadingError +from shapely.errors import ShapelyError from shapely.wkt import loads as shapely_loads from pygeoapi.plugin import load_plugin, PLUGINS @@ -124,7 +124,7 @@ def get_collection_edr_query(api: API, request: APIRequest, if wkt: try: wkt = shapely_loads(wkt) - except WKTReadingError: + except ShapelyError: msg = 'invalid coords parameter' return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, diff --git a/pygeoapi/provider/xarray_edr.py b/pygeoapi/provider/xarray_edr.py index 546b229ac..f5bf543f2 100644 --- a/pygeoapi/provider/xarray_edr.py +++ b/pygeoapi/provider/xarray_edr.py @@ -81,14 +81,14 @@ def position(self, **kwargs): wkt = kwargs.get('wkt') if wkt is not None: LOGGER.debug('Processing WKT') - LOGGER.debug(f'Geometry type: {wkt.type}') - if wkt.type == 'Point': + LOGGER.debug(f'Geometry type: {wkt.geom_type}') + if wkt.geom_type == 'Point': query_params[self._coverage_properties['x_axis_label']] = wkt.x query_params[self._coverage_properties['y_axis_label']] = wkt.y - elif wkt.type == 'LineString': + elif wkt.geom_type == 'LineString': query_params[self._coverage_properties['x_axis_label']] = wkt.xy[0] # noqa query_params[self._coverage_properties['y_axis_label']] = wkt.xy[1] # noqa - elif wkt.type == 'Polygon': + elif wkt.geom_type == 'Polygon': query_params[self._coverage_properties['x_axis_label']] = slice(wkt.bounds[0], wkt.bounds[2]) # noqa query_params[self._coverage_properties['y_axis_label']] = slice(wkt.bounds[1], wkt.bounds[3]) # noqa pass From a806f89a313f6ed07ee33171f746876b29df1697 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 27 Jul 2024 07:50:17 -0400 Subject: [PATCH 32/74] add installation note about Python version support (#1644) (#1760) --- docs/source/installation.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index c3ff21b7b..e3354da08 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -11,6 +11,13 @@ Requirements and dependencies pygeoapi runs on Python 3. +.. note:: + + The exact Python version requirements are aligned with the version of Python on the pygeoapi supported Ubuntu + operating system version. For example, as of 2024-07, the supported version of Python is bound to Ubuntu 22.04 + (Jammy) which supports Python 3.10. Ensure you have a Python version that is compatible with the current Ubuntu + version that is specified in pygeoapi's `Dockerfile`_. + Core dependencies are included as part of a given pygeoapi installation procedure. More specific requirements details are described below depending on the platform. @@ -32,7 +39,7 @@ For developers and the truly impatient vi example-config.yml # edit as required export PYGEOAPI_CONFIG=example-config.yml export PYGEOAPI_OPENAPI=example-openapi.yml - pygeoapi openapi generate $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI + pygeoapi openapi generate $PYGEOAPI_CONFIG --output-file $PYGEOAPI_OPENAPI pygeoapi serve curl http://localhost:5000 @@ -142,3 +149,4 @@ onto your system. .. _`Docker image`: https://github.com/geopython/pygeoapi/pkgs/container/pygeoapi +.. _`Dockerfile`: https://github.com/geopython/pygeoapi/blob/master/Dockerfile From d1dfa179b32d69c024dda75c97a741f1ca6b7f52 Mon Sep 17 00:00:00 2001 From: Moritz Langer <31667222+Moritz-Langer@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:04:02 +0200 Subject: [PATCH 33/74] Add Wallet for Session pool connections in oracle.py (#1768) * Added Wallet to Connection Pool * Flake8 changes * Flake8 changes * Feedback from Pull Request * Flake8 --- .../data-publishing/ogcapi-features.rst | 2 +- pygeoapi/provider/oracle.py | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index ad84adea1..30ccf0796 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -420,7 +420,7 @@ Configured using environment variables. export ORACLE_POOL_MAX=10 -The ``ORACLE_POOL_MIN`` and ``ORACLE_POOL_MAX`` environment variables are used to trigger session pool creation in the Oracle Provider and the ``DatabaseConnection`` class. See https://python-oracledb.readthedocs.io/en/latest/api_manual/module.html#oracledb.create_pool for documentation of the ``create_pool`` function. +The ``ORACLE_POOL_MIN`` and ``ORACLE_POOL_MAX`` environment variables are used to trigger session pool creation in the Oracle Provider and the ``DatabaseConnection`` class. Supports auth via user + password or wallet. For an example of the configuration see above at Oracle - Connection. See https://python-oracledb.readthedocs.io/en/latest/api_manual/module.html#oracledb.create_pool for documentation of the ``create_pool`` function. If none or only one of the environment variables is set, session pooling will not be activated and standalone connections are established at every request. diff --git a/pygeoapi/provider/oracle.py b/pygeoapi/provider/oracle.py index 4bedde773..1359779c7 100644 --- a/pygeoapi/provider/oracle.py +++ b/pygeoapi/provider/oracle.py @@ -66,17 +66,31 @@ def create_pool(cls, conn_dict, oracle_pool_min, oracle_pool_max): """Initialize the connection pool for the class Lock is implemented before function call at __init__""" dsn = cls._make_dsn(conn_dict) + + connect_kwargs = { + 'dsn': dsn, + 'min': oracle_pool_min, + 'max': oracle_pool_max, + 'increment': 1 + } + # Create the pool + if conn_dict.get("external_auth") == "wallet": + # If Auth is via Wallet you need to save a wallet under + # the directory returned by this bash command if apache is used + # cat /etc/passwd |grep apache + # except another directory is specified in the sqlnet.ora file + LOGGER.debug("Connection pool from wallet.") + connect_kwargs["externalauth"] = True + connect_kwargs["homogeneous"] = False - p = oracledb.create_pool( - user=conn_dict["user"], - password=conn_dict["password"], - dsn=dsn, - min=oracle_pool_min, - max=oracle_pool_max, - increment=1, - ) - LOGGER.debug("Connection pool created successfully.") + else: + LOGGER.debug("Connection pool from user and password.") + connect_kwargs["user"] = conn_dict["user"] + connect_kwargs["password"] = conn_dict["password"] + + p = oracledb.create_pool(**connect_kwargs) + LOGGER.debug("Connection pool created successfully") return p From 491ceaff48caeed0e5533d7df37140b3cce9bba8 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Fri, 2 Aug 2024 09:07:47 -0400 Subject: [PATCH 34/74] fix collection breadcrumbs on queryables and schemas HTML Jinja2 templates (#1769) --- pygeoapi/api/__init__.py | 1 + pygeoapi/api/itemtypes.py | 1 + pygeoapi/templates/collections/queryables.html | 2 +- pygeoapi/templates/collections/schema.html | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 65d85587d..297a22da9 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -1382,6 +1382,7 @@ def get_collection_schema(self, request: Union[APIRequest, Any], self.config['resources'][dataset]['title'], request.locale) schema['collections_path'] = self.get_collections_url() + schema['dataset_path'] = f'{self.get_collections_url()}/{dataset}' content = render_j2_template(self.tpl_config, 'collections/schema.html', diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 1fa70c235..c2f403556 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -188,6 +188,7 @@ def get_collection_queryables(api: API, request: Union[APIRequest, Any], api.config['resources'][dataset]['title'], request.locale) queryables['collections_path'] = api.get_collections_url() + queryables['dataset_path'] = f'{api.get_collections_url()}/{dataset}' content = render_j2_template(api.tpl_config, 'collections/queryables.html', diff --git a/pygeoapi/templates/collections/queryables.html b/pygeoapi/templates/collections/queryables.html index df4387135..49ca148f2 100644 --- a/pygeoapi/templates/collections/queryables.html +++ b/pygeoapi/templates/collections/queryables.html @@ -2,7 +2,7 @@ {% block title %}{{ super() }} {{ data['title'] }} {% endblock %} {% block crumbs %}{{ super() }} / {% trans %}Collections{% endtrans %} -/ {{ data['title'] | truncate( 25 ) }} +/ {{ data['title'] | truncate( 25 ) }} / {% trans %}Queryables{% endtrans %} {% endblock %} {% block body %} diff --git a/pygeoapi/templates/collections/schema.html b/pygeoapi/templates/collections/schema.html index e839d36f5..f83f42f84 100644 --- a/pygeoapi/templates/collections/schema.html +++ b/pygeoapi/templates/collections/schema.html @@ -2,7 +2,7 @@ {% block title %}{{ super() }} {{ data['title'] }} {% endblock %} {% block crumbs %}{{ super() }} / {% trans %}Collections{% endtrans %} -/ {{ data['title'] | truncate( 25 ) }} +/ {{ data['title'] | truncate( 25 ) }} / {% trans %}Schema{% endtrans %} {% endblock %} {% block body %} From 4b28de6d42aa125dfe0ee2c546976821bf08c877 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 5 Aug 2024 08:13:44 -0400 Subject: [PATCH 35/74] fix breadcrumbs again (follow on of #1769) (#1770) --- pygeoapi/templates/collections/queryables.html | 2 +- pygeoapi/templates/collections/schema.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/templates/collections/queryables.html b/pygeoapi/templates/collections/queryables.html index 49ca148f2..3a5af610e 100644 --- a/pygeoapi/templates/collections/queryables.html +++ b/pygeoapi/templates/collections/queryables.html @@ -3,7 +3,7 @@ {% block crumbs %}{{ super() }} / {% trans %}Collections{% endtrans %} / {{ data['title'] | truncate( 25 ) }} -/ {% trans %}Queryables{% endtrans %} +/ {% trans %}Queryables{% endtrans %} {% endblock %} {% block body %}
    diff --git a/pygeoapi/templates/collections/schema.html b/pygeoapi/templates/collections/schema.html index f83f42f84..6a0383373 100644 --- a/pygeoapi/templates/collections/schema.html +++ b/pygeoapi/templates/collections/schema.html @@ -3,7 +3,7 @@ {% block crumbs %}{{ super() }} / {% trans %}Collections{% endtrans %} / {{ data['title'] | truncate( 25 ) }} -/ {% trans %}Schema{% endtrans %} +/ {% trans %}Schema{% endtrans %} {% endblock %} {% block body %}
    From d4063f360e156dae4a0f8d8e399b8a0e5059632c Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 6 Aug 2024 06:03:00 -0400 Subject: [PATCH 36/74] fix EDR HTML breadcrumbs (#1764) * fix EDR HTML breadcrumbs * add translations * do not include CoverageJSON to format types * set JSON-LD link for HTML templating * add Locations and Instances to translations --- locale/bs/LC_MESSAGES/messages.po | 24 +++++++++++++ locale/de/LC_MESSAGES/messages.po | 24 +++++++++++++ locale/en/LC_MESSAGES/messages.po | 24 +++++++++++++ locale/es/LC_MESSAGES/messages.po | 24 +++++++++++++ locale/fr/LC_MESSAGES/messages.po | 24 +++++++++++++ locale/sr/LC_MESSAGES/messages.po | 24 +++++++++++++ pygeoapi/api/__init__.py | 1 + pygeoapi/api/environmental_data_retrieval.py | 35 ++++++++++++++++++- pygeoapi/templates/_base.html | 2 +- pygeoapi/templates/collections/edr/query.html | 2 +- 10 files changed, 181 insertions(+), 3 deletions(-) diff --git a/locale/bs/LC_MESSAGES/messages.po b/locale/bs/LC_MESSAGES/messages.po index 40c6573f6..b140d4b2c 100644 --- a/locale/bs/LC_MESSAGES/messages.po +++ b/locale/bs/LC_MESSAGES/messages.po @@ -656,3 +656,27 @@ msgstr "" msgid "not specified" msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/de/LC_MESSAGES/messages.po b/locale/de/LC_MESSAGES/messages.po index e7d1229b6..770816f1c 100644 --- a/locale/de/LC_MESSAGES/messages.po +++ b/locale/de/LC_MESSAGES/messages.po @@ -706,3 +706,27 @@ msgstr "" msgid "not specified" msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/en/LC_MESSAGES/messages.po b/locale/en/LC_MESSAGES/messages.po index 4f2e08a03..d9ff71247 100644 --- a/locale/en/LC_MESSAGES/messages.po +++ b/locale/en/LC_MESSAGES/messages.po @@ -708,3 +708,27 @@ msgstr "" msgid "not specified" msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/es/LC_MESSAGES/messages.po b/locale/es/LC_MESSAGES/messages.po index 9cd90fc2a..7c31304c3 100644 --- a/locale/es/LC_MESSAGES/messages.po +++ b/locale/es/LC_MESSAGES/messages.po @@ -521,3 +521,27 @@ msgstr "" msgid "not specified" msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/fr/LC_MESSAGES/messages.po b/locale/fr/LC_MESSAGES/messages.po index 72ab27727..ea63df8c7 100644 --- a/locale/fr/LC_MESSAGES/messages.po +++ b/locale/fr/LC_MESSAGES/messages.po @@ -715,3 +715,27 @@ msgstr "" msgid "not specified" msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/sr/LC_MESSAGES/messages.po b/locale/sr/LC_MESSAGES/messages.po index 9a043592e..71d7210f0 100644 --- a/locale/sr/LC_MESSAGES/messages.po +++ b/locale/sr/LC_MESSAGES/messages.po @@ -656,3 +656,27 @@ msgstr "" msgid "not specified" msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 297a22da9..a51b86258 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -80,6 +80,7 @@ CHARSET = ['utf-8'] F_JSON = 'json' +F_COVERAGEJSON = 'json' F_HTML = 'html' F_JSONLD = 'jsonld' F_GZIP = 'gzip' diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index 7e7ba1248..a72b97655 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -41,10 +41,12 @@ from http import HTTPStatus import logging from typing import Tuple +import urllib from shapely.errors import ShapelyError from shapely.wkt import loads as shapely_loads +from pygeoapi import l10n from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ProviderGenericError from pygeoapi.util import ( @@ -52,7 +54,8 @@ to_json, filter_dict_by_key_value ) -from . import APIRequest, API, F_HTML, validate_datetime, validate_bbox +from . import (APIRequest, API, F_COVERAGEJSON, F_HTML, F_JSONLD, + validate_datetime, validate_bbox) LOGGER = logging.getLogger(__name__) @@ -195,6 +198,36 @@ def get_collection_edr_query(api: API, request: APIRequest, err.ogc_exception_code, err.message) if request.format == F_HTML: # render + uri = f'{api.get_collections_url()}/{dataset}/{query_type}' + serialized_query_params = '' + for k, v in request.params.items(): + if k != 'f': + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + data['query_type'] = query_type.capitalize() + data['query_path'] = uri + data['dataset_path'] = '/'.join(uri.split('/')[:-1]) + data['collections_path'] = api.get_collections_url() + + data['links'] = [{ + 'rel': 'collection', + 'title': collections[dataset]['title'], + 'href': data['dataset_path'] + }, { + 'type': 'application/prs.coverage+json', + 'rel': request.get_linkrel(F_COVERAGEJSON), + 'title': l10n.translate('This document as CoverageJSON', request.locale), # noqa + 'href': f'{uri}?f={F_COVERAGEJSON}{serialized_query_params}' + }, { + 'type': 'application/ld+json', + 'rel': 'alternate', + 'title': l10n.translate('This document as JSON-LD', request.locale), # noqa + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }] + content = render_j2_template(api.tpl_config, 'collections/edr/query.html', data, api.default_locale) diff --git a/pygeoapi/templates/_base.html b/pygeoapi/templates/_base.html index b53701054..5b57d5f54 100644 --- a/pygeoapi/templates/_base.html +++ b/pygeoapi/templates/_base.html @@ -70,7 +70,7 @@ {% set links_found = namespace(json=0, jsonld=0) %} {% for link in data['links'] %} - {% if link['rel'] == 'alternate' and link['type'] and link['type'] in ['application/json', 'application/geo+json'] %} + {% if link['rel'] == 'alternate' and link['type'] and link['type'] in ['application/json', 'application/geo+json', 'application/prs.coverage+json'] %} {% set links_found.json = 1 %} {% trans %}json{% endtrans %} {% elif link['rel'] == 'alternate' and link['type'] and link['type'] == 'application/ld+json' %} diff --git a/pygeoapi/templates/collections/edr/query.html b/pygeoapi/templates/collections/edr/query.html index af426c766..7563273f3 100644 --- a/pygeoapi/templates/collections/edr/query.html +++ b/pygeoapi/templates/collections/edr/query.html @@ -8,7 +8,7 @@ {% set col_title = link['title'] %} {% endif %} {% endfor %} -/ {% trans %}Items{% endtrans %} +/ {% trans query_type=data.query_type %}{{ query_type }}{% endtrans %} {% endblock %} {% block extrahead %} From 9ad870622352ba798426e40c5cf0c6d344111047 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 6 Aug 2024 06:03:27 -0400 Subject: [PATCH 37/74] fix item id breadcrumb (#1772) --- pygeoapi/templates/collections/items/item.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi/templates/collections/items/item.html b/pygeoapi/templates/collections/items/item.html index 9c98492f3..705f9057c 100644 --- a/pygeoapi/templates/collections/items/item.html +++ b/pygeoapi/templates/collections/items/item.html @@ -1,5 +1,5 @@ {% extends "_base.html" %} -{% set ptitle = data['properties'][data['title_field']] or '_(Item) '.format(data['id']) %} +{% set ptitle = data['properties'][data['title_field']] or data['id'] | string %} {% block desc %}{{ data.get('properties',{}).get('description', {}) | string | truncate(250) }}{% endblock %} {% block tags %}{{ data['properties'].get('themes', [{}])[0].get('concepts', []) | join(',') }}{% endblock %} {# Optionally renders an img element, otherwise standard value or link rendering #} From c1b90dc3acaa2534ae8997dc64ce4c477eef3fc8 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Fri, 9 Aug 2024 09:56:02 -0400 Subject: [PATCH 38/74] update basemap URL across all configurations (#1777) (#1778) * update basemap URL across all configurations (#1777) * update basemap URL across all configurations (#1777) --- docker/default.config.yml | 4 ++-- docs/source/configuration.rst | 4 ++-- tests/data/admin/admin-put.json | 6 +++--- tests/pygeoapi-test-config-admin.yml | 6 ++---- tests/pygeoapi-test-config-apirules.yml | 4 ++-- tests/pygeoapi-test-config-enclosure.yml | 4 ++-- tests/pygeoapi-test-config-envvars.yml | 4 ++-- tests/pygeoapi-test-config-hidden-resources.yml | 4 ++-- tests/pygeoapi-test-config-ogr.yml | 4 ++-- tests/pygeoapi-test-config-postgresql-manager.yml | 4 ++-- tests/pygeoapi-test-config.yml | 4 ++-- 11 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docker/default.config.yml b/docker/default.config.yml index 6b5af029e..81d9e2883 100644 --- a/docker/default.config.yml +++ b/docker/default.config.yml @@ -48,8 +48,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' ogc_schemas_location: /schemas.opengis.net logging: diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index d7b34125a..2fa41d7f0 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -57,8 +57,8 @@ For more information related to API design rules (the ``api_rules`` property in static: /path/to/static/folder # path to static folder containing css, js, images and other static files referenced by the template map: # leaflet map setup for HTML pages - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' ogc_schemas_location: /opt/schemas.opengis.net # local copy of https://schemas.opengis.net manager: # optional OGC API - Processes asynchronous job management diff --git a/tests/data/admin/admin-put.json b/tests/data/admin/admin-put.json index c09827254..c3b2a0c94 100644 --- a/tests/data/admin/admin-put.json +++ b/tests/data/admin/admin-put.json @@ -13,8 +13,8 @@ "pretty_print": true, "limit": 10, "map": { - "url": "https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png", - "attribution": "Wikimedia maps | Map data © OpenStreetMap contributors" + "url": "https://tile.openstreetmap.org/{z}/{x}/{y}.png", + "attribution": "© OpenStreetMap contributors" } }, "logging": { @@ -69,4 +69,4 @@ } } } - \ No newline at end of file + diff --git a/tests/pygeoapi-test-config-admin.yml b/tests/pygeoapi-test-config-admin.yml index aba8b9146..c7bdbc49d 100644 --- a/tests/pygeoapi-test-config-admin.yml +++ b/tests/pygeoapi-test-config-admin.yml @@ -41,10 +41,8 @@ server: pretty_print: true limit: 10 map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: Wikimedia - maps | Map data © OpenStreetMap - contributors + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' # manager: # name: TinyDB # connection: /tmp/pygeoapi-process-manager.db diff --git a/tests/pygeoapi-test-config-apirules.yml b/tests/pygeoapi-test-config-apirules.yml index 598903de6..6aba4c71c 100644 --- a/tests/pygeoapi-test-config-apirules.yml +++ b/tests/pygeoapi-test-config-apirules.yml @@ -46,8 +46,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' manager: name: TinyDB connection: /tmp/pygeoapi-test-process-manager.db diff --git a/tests/pygeoapi-test-config-enclosure.yml b/tests/pygeoapi-test-config-enclosure.yml index e4011622a..aa65b0595 100644 --- a/tests/pygeoapi-test-config-enclosure.yml +++ b/tests/pygeoapi-test-config-enclosure.yml @@ -44,8 +44,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' manager: name: TinyDB connection: /tmp/pygeoapi-test-process-manager.db diff --git a/tests/pygeoapi-test-config-envvars.yml b/tests/pygeoapi-test-config-envvars.yml index 0b2d4c8e9..f84fc79ac 100644 --- a/tests/pygeoapi-test-config-envvars.yml +++ b/tests/pygeoapi-test-config-envvars.yml @@ -41,8 +41,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' logging: level: DEBUG diff --git a/tests/pygeoapi-test-config-hidden-resources.yml b/tests/pygeoapi-test-config-hidden-resources.yml index 5be7a676a..3682bc2c9 100644 --- a/tests/pygeoapi-test-config-hidden-resources.yml +++ b/tests/pygeoapi-test-config-hidden-resources.yml @@ -44,8 +44,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' manager: name: TinyDB connection: /tmp/pygeoapi-test-process-manager.db diff --git a/tests/pygeoapi-test-config-ogr.yml b/tests/pygeoapi-test-config-ogr.yml index 703e0a7fa..e4237941a 100644 --- a/tests/pygeoapi-test-config-ogr.yml +++ b/tests/pygeoapi-test-config-ogr.yml @@ -41,8 +41,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' logging: level: DEBUG diff --git a/tests/pygeoapi-test-config-postgresql-manager.yml b/tests/pygeoapi-test-config-postgresql-manager.yml index fb3d63583..e0fe947d0 100644 --- a/tests/pygeoapi-test-config-postgresql-manager.yml +++ b/tests/pygeoapi-test-config-postgresql-manager.yml @@ -44,8 +44,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' manager: name: PostgreSQL connection: diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 588d06301..95a868631 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -44,8 +44,8 @@ server: limit: 10 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' manager: name: TinyDB connection: /tmp/pygeoapi-test-process-manager.db From 71ce03e5486221389b3d133adf2b89f4e9121e39 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:55:17 -0400 Subject: [PATCH 39/74] Add validation check to EDR query registration (#1774) * Add validation check to EDR query registration * Fix flake8 * fix edr query types --- pygeoapi/provider/base_edr.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pygeoapi/provider/base_edr.py b/pygeoapi/provider/base_edr.py index 3e7a259cb..021b2f403 100644 --- a/pygeoapi/provider/base_edr.py +++ b/pygeoapi/provider/base_edr.py @@ -29,10 +29,14 @@ import logging -from pygeoapi.provider.base import BaseProvider +from pygeoapi.provider.base import BaseProvider, ProviderInvalidDataError LOGGER = logging.getLogger(__name__) +EDR_QUERY_TYPES = ['position', 'radius', 'area', 'cube', + 'trajectory', 'corridor', 'items', + 'locations', 'instances'] + class BaseEDRProvider(BaseProvider): """Base EDR Provider""" @@ -55,6 +59,11 @@ def __init__(self, provider_def): @classmethod def register(cls): def inner(fn): + if fn.__name__ not in EDR_QUERY_TYPES: + msg = 'Invalid EDR Query type' + LOGGER.error(msg) + raise ProviderInvalidDataError(msg) + cls.query_types.append(fn.__name__) return fn return inner From 2a131c51311ad8c00bad475675abcb5cb3709fdc Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 13 Aug 2024 11:12:44 -0400 Subject: [PATCH 40/74] update docstrings for base provider fields functionality (#1783) * update docstrings for base provider fields functionality * fix flake8 --- pygeoapi/provider/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index c81c2b4f4..5f9456870 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -85,7 +85,8 @@ def get_fields(self): """ Get provider field information (names, types) - Example response: {'field1': 'string', 'field2': 'number'}} + Example response: + {'field1': {'type': 'string'}, 'field2': {'type': 'number'}} :returns: dict of field names and their associated JSON Schema types """ @@ -97,7 +98,8 @@ def fields(self) -> dict: """ Store provider field information (names, types) - Example response: {'field1': {'type': 'string'}} + Example response: + {'field1': {'type': 'string'}, 'field2': {'type': 'number'}} :returns: dict of dicts (field names and their associated JSON Schema definitions) From 60bd40385e5e83673c80b70f0878e89a75b6af72 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Thu, 15 Aug 2024 12:06:36 -0400 Subject: [PATCH 41/74] Add C3 plotting to EDR HTML view (#1784) * Add C3 plotting to EDR HTML view * Remove unused function --- pygeoapi/static/css/default.css | 9 + pygeoapi/templates/collections/edr/query.html | 247 ++++++++++++++++-- 2 files changed, 232 insertions(+), 24 deletions(-) diff --git a/pygeoapi/static/css/default.css b/pygeoapi/static/css/default.css index 2f87fc9e3..a8bed5164 100644 --- a/pygeoapi/static/css/default.css +++ b/pygeoapi/static/css/default.css @@ -27,6 +27,15 @@ main { height: 400px; } +#coverages-map { + width: 100%; + height: 80vh; +} + +.c3-tooltip-container { + z-index: 300; +} + /* cancel mini-css header>button uppercase */ header button, header [type="button"], header .button, header [role="button"] { text-transform: none; diff --git a/pygeoapi/templates/collections/edr/query.html b/pygeoapi/templates/collections/edr/query.html index 7563273f3..79ed714e4 100644 --- a/pygeoapi/templates/collections/edr/query.html +++ b/pygeoapi/templates/collections/edr/query.html @@ -15,6 +15,9 @@ {% if data.type == "Coverage" or data.type == "CoverageCollection" %} + + + @@ -27,42 +30,240 @@ {% block body %}
    -
    + {% if data.features or data.coverages %} +
    + {% else %} +
    +

    {% trans %}No items{% endtrans %}

    +
    + {% endif %}
    {% endblock %} {% block extrafoot %} {% if data %} diff --git a/pygeoapi/templates/collections/items/index.html b/pygeoapi/templates/collections/items/index.html index bb9b48855..d2049becd 100644 --- a/pygeoapi/templates/collections/items/index.html +++ b/pygeoapi/templates/collections/items/index.html @@ -169,7 +169,6 @@

    {% for l in data['links'] if l.rel == 'collection' %} {{ l['title'] }} {% en layer.bindPopup(html); } }); - {% if data['features'][0]['geometry']['type'] == 'Point' %} var markers = L.markerClusterGroup({ disableClusteringAtZoom: 9, chunkedLoading: true, @@ -177,9 +176,6 @@

    {% for l in data['links'] if l.rel == 'collection' %} {{ l['title'] }} {% en }); markers.clearLayers().addLayer(items); map.addLayer(markers); - {% else %} - map.addLayer(items); - {% endif %} map.fitBounds(items.getBounds()); From 6ad14a6d54c6b0d913ea701439f203f16c85ff31 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 9 Sep 2024 12:02:33 -0400 Subject: [PATCH 55/74] drop unicodecsv package (#1805) * remove unicodecsv (#1804) * add test --- Dockerfile | 1 - pygeoapi/formatter/csv_.py | 8 ++++---- requirements.txt | 1 - tests/api/test_itemtypes.py | 7 +++++++ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98e7ffc18..71505ba98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,7 +98,6 @@ ENV TZ=${TZ} \ python3-greenlet \ python3-pip \ python3-tz \ - python3-unicodecsv \ python3-yaml \ ${ADD_DEB_PACKAGES}" diff --git a/pygeoapi/formatter/csv_.py b/pygeoapi/formatter/csv_.py index 664bc8807..51a6ded17 100644 --- a/pygeoapi/formatter/csv_.py +++ b/pygeoapi/formatter/csv_.py @@ -27,11 +27,10 @@ # # ================================================================= +import csv import io import logging -import unicodecsv as csv - from pygeoapi.formatter.base import BaseFormatter, FormatterSerializationError LOGGER = logging.getLogger(__name__) @@ -83,10 +82,11 @@ def write(self, options: dict = {}, data: dict = None) -> str: # TODO: implement wkt geometry serialization LOGGER.debug('not a point geometry, skipping') + print("JJJ", fields) LOGGER.debug(f'CSV fields: {fields}') try: - output = io.BytesIO() + output = io.StringIO() writer = csv.DictWriter(output, fields) writer.writeheader() @@ -101,7 +101,7 @@ def write(self, options: dict = {}, data: dict = None) -> str: LOGGER.error(err) raise FormatterSerializationError('Error writing CSV output') - return output.getvalue() + return output.getvalue().encode('utf-8') def __repr__(self): return f' {self.name}' diff --git a/requirements.txt b/requirements.txt index ef0f47d81..601f51b83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,3 @@ requests shapely SQLAlchemy<2.0.0 tinydb -unicodecsv diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py index 283bb0c23..2cd445898 100644 --- a/tests/api/test_itemtypes.py +++ b/tests/api/test_itemtypes.py @@ -578,6 +578,13 @@ def test_get_collection_item(config, api_): assert 'prev' not in feature['links'] assert 'next' not in feature['links'] + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'norway_pop', + '790') + feature = json.loads(response) + + assert feature['properties']['name'] == 'Ålesund' + def test_get_collection_item_json_ld(config, api_): req = mock_api_request({'f': 'jsonld'}) From 28618034b8ce05730b2adeb5023554b7e8c03dce Mon Sep 17 00:00:00 2001 From: Simon Seyock <8100558+simonseyock@users.noreply.github.com> Date: Wed, 11 Sep 2024 18:24:26 +0200 Subject: [PATCH 56/74] feat: add version parameter to WMSFacade provider (#1806) --- docs/source/data-publishing/ogcapi-maps.rst | 9 +++++---- pygeoapi/provider/wms_facade.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-maps.rst b/docs/source/data-publishing/ogcapi-maps.rst index 30e12b1d1..03846eeaa 100644 --- a/docs/source/data-publishing/ogcapi-maps.rst +++ b/docs/source/data-publishing/ogcapi-maps.rst @@ -51,7 +51,7 @@ Currently supported style files (`options.style`): .. code-block:: yaml providers: - - type: map + - type: map name: MapScript data: /path/to/data.shp options: @@ -59,7 +59,7 @@ Currently supported style files (`options.style`): layer: foo_name style: ./foo.sld format: - name: png + name: png mimetype: image/png WMSFacade @@ -71,14 +71,15 @@ required. An optional style name can be defined via `options.style`. .. code-block:: yaml providers: - - type: map + - type: map name: WMSFacade data: https://demo.mapserver.org/cgi-bin/msautotest options: layer: world_latlong style: default + version: 1.3.0 format: - name: png + name: png mimetype: image/png diff --git a/pygeoapi/provider/wms_facade.py b/pygeoapi/provider/wms_facade.py index 8771b6395..fa3ffd064 100644 --- a/pygeoapi/provider/wms_facade.py +++ b/pygeoapi/provider/wms_facade.py @@ -84,7 +84,9 @@ def query(self, style=None, bbox=[-180, -90, 180, 90], width=500, self._transparent = 'TRUE' - if crs in [4326, 'CRS;84']: + version = self.options.get('version', '1.3.0') + + if crs in [4326, 'CRS;84'] and version == '1.3.0': LOGGER.debug('Swapping 4326 axis order to WMS 1.3 mode (yx)') bbox2 = ','.join(str(c) for c in [bbox[1], bbox[0], bbox[3], bbox[2]]) @@ -106,12 +108,14 @@ def query(self, style=None, bbox=[-180, -90, 180, 90], width=500, if not transparent: self._transparent = 'FALSE' + crs_param = 'crs' if version == '1.3.0' else 'srs' + params = { - 'version': '1.3.0', + 'version': version, 'service': 'WMS', 'request': 'GetMap', 'bbox': bbox2, - 'crs': CRS_CODES[crs], + crs_param: CRS_CODES[crs], 'layers': self.options['layer'], 'styles': self.options.get('style', 'default'), 'width': width, @@ -128,7 +132,7 @@ def query(self, style=None, bbox=[-180, -90, 180, 90], width=500, else: request_url = '?'.join([self.data, urlencode(params)]) - LOGGER.debug(f'WMS 1.3.0 request url: {request_url}') + LOGGER.debug(f'WMS {version} request url: {request_url}') response = requests.get(request_url) From 0677c2e646a6eef43da2e5d9a7f88f6e76c91131 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:29:40 -0400 Subject: [PATCH 57/74] SensorThings API provider cleanup (#1807) * Add support for ObservedProperties OAF and custom expand of entities * Update sensorthings.py * Respond to feedback * Use `pygeoapi.get_config` for SensorThings Intralinking --- pygeoapi/provider/sensorthings.py | 256 +++++++++++++++++------------- 1 file changed, 143 insertions(+), 113 deletions(-) diff --git a/pygeoapi/provider/sensorthings.py b/pygeoapi/provider/sensorthings.py index b888a60f7..defd48c43 100644 --- a/pygeoapi/provider/sensorthings.py +++ b/pygeoapi/provider/sensorthings.py @@ -30,14 +30,14 @@ # ================================================================= from json.decoder import JSONDecodeError -import os import logging from requests import Session +from pygeoapi.config import get_config from pygeoapi.provider.base import ( BaseProvider, ProviderQueryError, ProviderConnectionError) from pygeoapi.util import ( - yaml_load, url_join, get_provider_default, crs_transform, get_base_url) + url_join, get_provider_default, crs_transform, get_base_url) LOGGER = logging.getLogger(__name__) @@ -51,10 +51,10 @@ _EXPAND = { 'Things': 'Locations,Datastreams', 'Observations': 'Datastream,FeatureOfInterest', + 'ObservedProperties': 'Datastreams/Thing/Locations', 'Datastreams': """ Sensor ,ObservedProperty - ,Thing ,Thing/Locations ,Observations( $select=@iot.id; @@ -71,6 +71,7 @@ class SensorThingsProvider(BaseProvider): """SensorThings API (STA) Provider""" + expand = EXPAND def __init__(self, provider_def): """ @@ -82,68 +83,11 @@ def __init__(self, provider_def): :returns: pygeoapi.provider.sensorthings.SensorThingsProvider """ LOGGER.debug('Setting SensorThings API (STA) provider') - + self.linked_entity = {} super().__init__(provider_def) - self.data.rstrip('/') - try: - self.entity = provider_def['entity'] - self._url = url_join(self.data, self.entity) - except KeyError: - LOGGER.debug('Attempting to parse Entity from provider data') - if not self._get_entity(self.data): - raise RuntimeError('Entity type required') - self.entity = self._get_entity(self.data) - self._url = self.data - self.data = self._url.rstrip(f'/{self.entity}') - LOGGER.debug(f'STA endpoint: {self.data}, Entity: {self.entity}') - - # Default id - if self.id_field: - LOGGER.debug(f'Using id field: {self.id_field}') - else: - LOGGER.debug('Using default @iot.id for id field') - self.id_field = '@iot.id' - - # Create intra-links - self.links = {} - self.intralink = provider_def.get('intralink', False) - if self.intralink and provider_def.get('rel_link'): - # For pytest - self.rel_link = provider_def['rel_link'] - - elif self.intralink: - # Read from pygeoapi config - with open(os.getenv('PYGEOAPI_CONFIG'), encoding='utf8') as fh: - CONFIG = yaml_load(fh) - self.rel_link = get_base_url(CONFIG) - for (name, rs) in CONFIG['resources'].items(): - pvs = rs.get('providers') - - if pvs is None: - LOGGER.debug(f'Skipping collection: {name}') - continue - - p = get_provider_default(pvs) - e = p.get('entity') or self._get_entity(p['data']) - if any([ - not pvs, # No providers in resource - not p.get('intralink'), # No configuration for intralinks - not e, # No STA entity found - self.data not in p.get('data') # No common STA endpoint - ]): - continue - - if p.get('uri_field'): - LOGGER.debug(f'Linking {e} with field: {p["uri_field"]}') - else: - LOGGER.debug(f'Linking {e} with collection: {name}') - - self.links[e] = { - 'cnm': name, # OAPI collection name, - 'cid': p.get('id_field', '@iot.id'), # OAPI id_field - 'uri': p.get('uri_field') # STA uri_field - } + self._generate_mappings(provider_def) + LOGGER.debug(f'STA endpoint: {self.data}, Entity: {self.entity}') # Start session self.http = Session() @@ -277,17 +221,19 @@ def _load(self, offset=0, limit=10, resulttype='results', return fc - def _make_feature(self, entity, select_properties=[], skip_geometry=False): + def _make_feature(self, feature, select_properties=[], skip_geometry=False, + entity=None): """ Private function: Create feature from entity - :param entity: `dict` of STA entity + :param feature: `dict` of STA entity :param select_properties: list of property names :param skip_geometry: bool of whether to skip geometry (default False) + :param entity: SensorThings entity name :returns: dict of GeoJSON Feature """ - _ = entity.pop(self.id_field) + _ = feature.pop(self.id_field) id = f"'{_}'" if isinstance(_, str) else str(_) f = { 'type': 'Feature', 'id': id, 'properties': {}, 'geometry': None @@ -295,28 +241,35 @@ def _make_feature(self, entity, select_properties=[], skip_geometry=False): # Make geometry if not skip_geometry: - f['geometry'] = self._geometry(entity) + f['geometry'] = self._geometry(feature, entity) # Fill properties block try: f['properties'] = self._expand_properties( - entity, select_properties) + feature, select_properties, entity) except KeyError as err: LOGGER.error(err) raise ProviderQueryError(err) return f - def _get_response(self, url, params={}): + def _get_response(self, url, params={}, entity=None, expand=None): """ Private function: Get STA response :param url: request url :param params: query parameters + :param entity: SensorThings entity name + :param expand: SensorThings expand query + :returns: STA response """ - params.update({'$expand': EXPAND[self.entity]}) + if expand: + params.update({'$expand': expand}) + else: + entity_ = entity or self.entity + params.update({'$expand': self.expand[entity_]}) r = self.http.get(url, params=params) @@ -332,13 +285,15 @@ def _get_response(self, url, params={}): return response - def _make_filter(self, properties, bbox=[], datetime_=None): + def _make_filter(self, properties, bbox=[], datetime_=None, + entity=None): """ Private function: Make STA filter from query properties :param properties: list of tuples (name, value) :param bbox: bounding box [minx,miny,maxx,maxy] :param datetime_: temporal (datestamp or extent) + :param entity: SensorThings entity name :returns: STA $filter string of properties """ @@ -350,16 +305,8 @@ def _make_filter(self, properties, bbox=[], datetime_=None): ret.append(f'{name} eq {value}') if bbox: - minx, miny, maxx, maxy = bbox - bbox_ = f'POLYGON (({minx} {miny}, {maxx} {miny}, \ - {maxx} {maxy}, {minx} {maxy}, {minx} {miny}))' - if self.entity == 'Things': - loc = 'Locations/location' - elif self.entity == 'Datastreams': - loc = 'Thing/Locations/location' - elif self.entity == 'Observations': - loc = 'FeatureOfInterest/feature' - ret.append(f"st_within({loc}, geography'{bbox_}')") + entity_ = entity or self.entity + ret.append(self._make_bbox(bbox, entity_)) if datetime_ is not None: if self.time_field is None: @@ -378,6 +325,20 @@ def _make_filter(self, properties, bbox=[], datetime_=None): return ' and '.join(ret) + @staticmethod + def _make_bbox(bbox, entity): + minx, miny, maxx, maxy = bbox + bbox_ = f'POLYGON(({minx} {miny},{maxx} {miny},{maxx} {maxy},{minx} {maxy},{minx} {miny}))' # noqa + if entity == 'Things': + loc = 'Locations/location' + elif entity == 'Datastreams': + loc = 'Thing/Locations/location' + elif entity == 'Observations': + loc = 'FeatureOfInterest/feature' + elif entity == 'ObservedProperties': + loc = 'Datastreams/observedArea' + return f"st_within({loc},geography'{bbox_}')" + def _make_orderby(self, sortby): """ Private function: Make STA filter from query properties @@ -398,79 +359,85 @@ def _make_orderby(self, sortby): return ','.join(ret) - def _geometry(self, entity): + def _geometry(self, feature, entity=None): """ Private function: Retrieve STA geometry - :param entity: SensorThings entity + :param feature: SensorThings entity + :param entity: SensorThings entity name :returns: GeoJSON Geometry for feature """ + entity_ = entity or self.entity try: - if self.entity == 'Things': - return entity['Locations'][0]['location'] + if entity_ == 'Things': + return feature['Locations'][0]['location'] - elif self.entity == 'Observations': - return entity['FeatureOfInterest'].pop('feature') + elif entity_ == 'Observations': + return feature['FeatureOfInterest'].pop('feature') - elif self.entity == 'Datastreams': + elif entity_ == 'Datastreams': try: - return entity['Observations'][0]['FeatureOfInterest'].pop('feature') # noqa + return feature['Observations'][0]['FeatureOfInterest'].pop('feature') # noqa except (KeyError, IndexError): - return entity['Thing'].pop('Locations')[0]['location'] + return feature['Thing'].pop('Locations')[0]['location'] + + elif entity_ == 'ObservedProperties': + return feature['Datastreams'][0]['Thing']['Locations'][0]['location'] # noqa except (KeyError, IndexError): LOGGER.warning('No geometry found') return None - def _expand_properties(self, entity, keys=(), uri=''): + def _expand_properties(self, feature, keys=(), uri='', + entity=None): """ Private function: Parse STA entity into feature - :param entity: SensorThings entity + :param feature: `dict` of SensorThings entity :param keys: keys used in properties block :param uri: uri of STA entity + :param entity: SensorThings entity name :returns: dict of SensorThings feature properties """ - LOGGER.debug('Adding extra properties') - # Properties filter & display keys = (() if not self.properties and not keys else set(self.properties) | set(keys)) - if self.entity == 'Things': - self._expand_location(entity) - elif 'Thing' in entity.keys(): - self._expand_location(entity['Thing']) + entity = entity or self.entity + if entity == 'Things': + self._expand_location(feature) + elif 'Thing' in feature.keys(): + self._expand_location(feature['Thing']) # Retain URI if present - if entity.get('properties') and self.uri_field: - uri = entity['properties'] + if feature.get('properties') and self.uri_field: + uri = feature['properties'] # Create intra links - LOGGER.debug('Creating intralinks') - for k, v in entity.items(): - if k in self.links: - entity[k] = [self._get_uri(_v, **self.links[k]) for _v in v] + for k, v in feature.items(): + if k in self.linked_entity: + feature[k] = [self._get_uri(_v, **self.linked_entity[k]) + for _v in v] LOGGER.debug(f'Created link for {k}') - elif f'{k}s' in self.links: - entity[k] = self._get_uri(v, **self.links[f'{k}s']) + elif f'{k}s' in self.linked_entity: + feature[k] = \ + self._get_uri(v, **self.linked_entity[f'{k}s']) LOGGER.debug(f'Created link for {k}') # Make properties block - LOGGER.debug('Making properties block') - if entity.get('properties'): - entity.update(entity.pop('properties')) + if feature.get('properties'): + feature.update(feature.pop('properties')) if keys: - ret = {k: entity.pop(k) for k in keys} - entity = ret + ret = {k: feature.pop(k) for k in keys} + feature = ret if self.uri_field is not None and uri != '': - entity[self.uri_field] = uri + feature[self.uri_field] = uri - return entity + return feature @staticmethod def _expand_location(entity): @@ -522,5 +489,68 @@ def _get_entity(uri): else: return '' + def _generate_mappings(self, provider_def: dict): + """ + Generate mappings for the STA entity and set up intra-links. + + This function sets up the necessary mappings and configurations for + the STA entity based on the provided provider definition. + + :param provider_def: `dict` of provider definition containing + configuration details for the STA entity. + """ + self.data.rstrip('/') + try: + self.entity = provider_def['entity'] + self._url = url_join(self.data, self.entity) + except KeyError: + LOGGER.debug('Attempting to parse Entity from provider data') + if not self._get_entity(self.data): + raise RuntimeError('Entity type required') + self.entity = self._get_entity(self.data) + self._url = self.data + self.data = self._url.rstrip(f'/{self.entity}') + + # Default id + if self.id_field: + LOGGER.debug(f'Using id field: {self.id_field}') + else: + LOGGER.debug('Using default @iot.id for id field') + self.id_field = '@iot.id' + + # Create intra-links + self.intralink = provider_def.get('intralink', False) + if self.intralink and provider_def.get('rel_link'): + # For pytest + self.rel_link = provider_def['rel_link'] + + elif self.intralink: + # Read from pygeoapi config + CONFIG = get_config() + self.rel_link = get_base_url(CONFIG) + + for name, rs in CONFIG['resources'].items(): + pvs = rs.get('providers') + p = get_provider_default(pvs) + e = p.get('entity') or self._get_entity(p['data']) + if any([ + not pvs, # No providers in resource + not p.get('intralink'), # No configuration for intralinks + not e, # No STA entity found + self.data not in p.get('data') # No common STA endpoint + ]): + continue + + if p.get('uri_field'): + LOGGER.debug(f'Linking {e} with field: {p["uri_field"]}') + else: + LOGGER.debug(f'Linking {e} with collection: {name}') + + self.linked_entity[e] = { + 'cnm': name, # OAPI collection name, + 'cid': p.get('id_field', '@iot.id'), # OAPI id_field + 'uri': p.get('uri_field') # STA uri_field + } + def __repr__(self): return f' {self.data}, {self.entity}' From deb043f928076aa37b4bb7efbee114f9245c9b0c Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Thu, 12 Sep 2024 13:37:11 +0200 Subject: [PATCH 58/74] Jobs pagination (#1779) * Add pagination for job list Adds limit and offset parameter to `get_jobs`. Process manager `get_jobs` now also returns the number of matched jobs additionally to the jobs themselves so we can calculate whether we need a next link. Note that this is a breaking change. * Add pagination support to jobs UI This works exactly the same way as for itemtypes * Add note regarding job sorting * Formatting fixes --- pygeoapi/api/processes.py | 84 +++++++++++++++++++++++++- pygeoapi/process/manager/base.py | 11 +++- pygeoapi/process/manager/dummy.py | 10 ++- pygeoapi/process/manager/mongodb_.py | 7 ++- pygeoapi/process/manager/postgresql.py | 16 +++-- pygeoapi/process/manager/tinydb_.py | 21 ++++++- pygeoapi/templates/jobs/index.html | 33 ++++++++++ tests/api/test_processes.py | 51 +++++++++++++++- 8 files changed, 215 insertions(+), 18 deletions(-) diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index e95d90a13..b2a577757 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -46,6 +46,7 @@ import json import logging from typing import Tuple +import urllib.parse from pygeoapi import l10n from pygeoapi.util import ( @@ -240,10 +241,51 @@ def get_jobs(api: API, request: APIRequest, headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError: + limit = int(api.config['server']['limit']) + LOGGER.debug('returning all jobs') + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + if job_id is None: - jobs = sorted(api.manager.get_jobs(), + jobs_data = api.manager.get_jobs(limit=limit, offset=offset) + # TODO: For pagination to work, the provider has to do the sorting. + # Here we do sort again in case the provider doesn't support + # pagination yet and always returns all jobs. + jobs = sorted(jobs_data['jobs'], key=lambda k: k['job_start_datetime'], reverse=True) + numberMatched = jobs_data['numberMatched'] + else: try: jobs = [api.manager.get_job(job_id)] @@ -251,6 +293,7 @@ def get_jobs(api: API, request: APIRequest, return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'InvalidParameterValue', job_id) + numberMatched = 1 serialized_jobs = { 'jobs': [], @@ -309,6 +352,44 @@ def get_jobs(api: API, request: APIRequest, serialized_jobs['jobs'].append(job2) + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + uri = f'{api.base_url}/jobs' + + if offset > 0: + prev = max(0, offset - limit) + serialized_jobs['links'].append( + { + 'href': f'{uri}?offset={prev}{serialized_query_params}', + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'prev', + 'title': l10n.translate('Items (prev)', request.locale), + }) + + next_link = False + + if numberMatched > (limit + offset): + next_link = True + elif len(jobs) == limit: + next_link = True + + if next_link: + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + serialized_jobs['links'].append( + { + 'href': next_href, + 'rel': 'next', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Items (next)', request.locale), + }) + if job_id is None: j2_template = 'jobs/index.html' else: @@ -318,6 +399,7 @@ def get_jobs(api: API, request: APIRequest, if request.format == F_HTML: data = { 'jobs': serialized_jobs, + 'offset': offset, 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) } response = render_j2_template(api.tpl_config, j2_template, data, diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index f0d31482e..df5c41a8c 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -108,14 +108,21 @@ def get_processor(self, process_id: str) -> BaseProcessor: else: return load_plugin('process', process_conf['processor']) - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, + status: JobStatus = None, + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> dict: """ Get process jobs, optionally filtered by status :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: `list` of jobs (identifier, status, process identifier) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ raise NotImplementedError() diff --git a/pygeoapi/process/manager/dummy.py b/pygeoapi/process/manager/dummy.py index 4a9866bc7..7c3a7037b 100644 --- a/pygeoapi/process/manager/dummy.py +++ b/pygeoapi/process/manager/dummy.py @@ -56,17 +56,21 @@ def __init__(self, manager_def: dict): super().__init__(manager_def) - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, status: JobStatus = None, limit=None, offset=None + ) -> dict: """ Get process jobs, optionally filtered by status :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: `list` of jobs (identifier, status, process identifier) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ - return [] + return {'jobs': [], 'numberMatched': 0} def execute_process( self, diff --git a/pygeoapi/process/manager/mongodb_.py b/pygeoapi/process/manager/mongodb_.py index 66886f973..2b64806cf 100644 --- a/pygeoapi/process/manager/mongodb_.py +++ b/pygeoapi/process/manager/mongodb_.py @@ -70,7 +70,7 @@ def destroy(self): exc_info=(traceback)) return False - def get_jobs(self, status=None): + def get_jobs(self, status=None, limit=None, offset=None): try: self._connect() database = self.db.job_manager_pygeoapi @@ -80,7 +80,10 @@ def get_jobs(self, status=None): else: jobs = list(collection.find({})) LOGGER.info("JOBMANAGER - MongoDB jobs queried") - return jobs + return { + 'jobs': jobs, + 'numberMatched': len(jobs) + } except Exception: LOGGER.error("JOBMANAGER - get_jobs error", exc_info=(traceback)) diff --git a/pygeoapi/process/manager/postgresql.py b/pygeoapi/process/manager/postgresql.py index 30b7b8df6..72f3b75c4 100644 --- a/pygeoapi/process/manager/postgresql.py +++ b/pygeoapi/process/manager/postgresql.py @@ -116,16 +116,18 @@ def __init__(self, manager_def: dict): LOGGER.error(f'{msg}: {err}') raise ProcessorGenericError(msg) - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, status: JobStatus = None, limit=None, offset=None + ) -> dict: """ Get jobs :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: 'list` of jobs (type (default='process'), identifier, - status, process_id, job_start_datetime, job_end_datetime, location, - mimetype, message, progress) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ LOGGER.debug('Querying for jobs') @@ -135,7 +137,11 @@ def get_jobs(self, status: JobStatus = None) -> list: column = getattr(self.table_model, 'status') results = results.filter(column == status.value) - return [r.__dict__ for r in results.all()] + jobs = [r.__dict__ for r in results.all()] + return { + 'jobs': jobs, + 'numberMatched': len(jobs) + } def add_job(self, job_metadata: dict) -> str: """ diff --git a/pygeoapi/process/manager/tinydb_.py b/pygeoapi/process/manager/tinydb_.py index 3966e9dd1..2f022a33c 100644 --- a/pygeoapi/process/manager/tinydb_.py +++ b/pygeoapi/process/manager/tinydb_.py @@ -82,20 +82,35 @@ def destroy(self) -> bool: return True - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, status: JobStatus = None, limit=None, offset=None + ) -> dict: """ Get jobs :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: 'list` of jobs (identifier, status, process identifier) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ with self._db() as db: jobs_list = db.all() - return jobs_list + number_matched = len(jobs_list) + + if offset: + jobs_list = jobs_list[offset:] + + if limit: + jobs_list = jobs_list[:limit] + + return { + 'jobs': jobs_list, + 'numberMatched': number_matched + } def add_job(self, job_metadata: dict) -> str: """ diff --git a/pygeoapi/templates/jobs/index.html b/pygeoapi/templates/jobs/index.html index 05c12d918..1c1903520 100644 --- a/pygeoapi/templates/jobs/index.html +++ b/pygeoapi/templates/jobs/index.html @@ -48,5 +48,38 @@ +
    +
    + {% trans %}Limit{% endtrans %}: + + +
    +
    +
    +
    + {% for link in data['jobs']['links'] %} + {% if link['rel'] == 'prev' and data['offset'] > 0 %} + {% trans %}Prev{% endtrans %} + {% elif link['rel'] == 'next' and data['jobs']['jobs'] %} + {% trans %}Next{% endtrans %} + {% endif %} + {% endfor %} +
    +
    +

    {% endblock %} diff --git a/tests/api/test_processes.py b/tests/api/test_processes.py index ee569c402..62256e4b1 100644 --- a/tests/api/test_processes.py +++ b/tests/api/test_processes.py @@ -39,7 +39,7 @@ from pygeoapi.api import FORMAT_TYPES, F_HTML, F_JSON from pygeoapi.api.processes import ( - describe_processes, execute_process, delete_job, get_job_result, + describe_processes, execute_process, delete_job, get_job_result, get_jobs ) from tests.util import mock_api_request @@ -442,4 +442,51 @@ def test_get_job_result(api_): ) assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == 'application/json' - assert json.loads(response)['value'] == "Hello Sync Test!" + assert json.loads(response)['value'] == 'Hello Sync Test!' + + +def test_get_jobs_single(api_): + job_id = _execute_a_job(api_) + headers, code, response = get_jobs(api_, mock_api_request(), job_id=job_id) + assert code == HTTPStatus.OK + + job = json.loads(response) + assert job['jobID'] == job_id + assert job['status'] == 'successful' + + +def test_get_jobs_pagination(api_): + # generate test jobs for querying + for _ in range(11): + _execute_a_job(api_) + + # test default pagination limit + headers, code, response = get_jobs(api_, mock_api_request(), job_id=None) + job_response = json.loads(response) + assert len(job_response['jobs']) == 10 + assert next( + link for link in job_response['links'] if link['rel'] == 'next' + )['href'].endswith('/jobs?offset=10') + + headers, code, response = get_jobs( + api_, + mock_api_request({'limit': 10, 'offset': 9}), + job_id=None) + job_response_offset = json.loads(response) + # check to get 1 same job id with an offset of 9 and limit of 10 + same_job_ids = {job['jobID'] for job in job_response['jobs']}.intersection( + {job['jobID'] for job in job_response_offset['jobs']} + ) + assert len(same_job_ids) == 1 + assert next( + link for link in job_response_offset['links'] if link['rel'] == 'prev' + )['href'].endswith('/jobs?offset=0&limit=10') + + # test custom limit + headers, code, response = get_jobs( + api_, + mock_api_request({'limit': 20}), + job_id=None) + job_response = json.loads(response) + # might be more than 11 due to test interaction + assert len(job_response['jobs']) > 10 From 6682b44928d36e19d54a8edadbe39cc40ca0991a Mon Sep 17 00:00:00 2001 From: Sarah Jordan Date: Thu, 12 Sep 2024 11:22:25 -0500 Subject: [PATCH 59/74] CRS handling in xarray provider properties (#1641) * update crs handling * fix epsg code * config parsing, lean on pyproj * consolidate code and leverage prior crs work * update crs handling * fix epsg code * config parsing, lean on pyproj * consolidate code and leverage prior crs work * fix function call * bug and flake8 fixes * documentation updates * flake8 * Update ogcapi-coverages.rst * update crs handling * fix epsg code * config parsing, lean on pyproj * consolidate code and leverage prior crs work * update crs handling * fix epsg code * config parsing, lean on pyproj * consolidate code and leverage prior crs work * fix function call * bug and flake8 fixes * documentation updates * flake8 * Update ogcapi-coverages.rst * flake8 fix * rebase issues * update import formatting Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * update conditional logic Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * update error handling Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * parse storage crs in init --------- Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> --- .../data-publishing/ogcapi-coverages.rst | 9 ++ docs/source/data-publishing/ogcapi-edr.rst | 9 ++ pygeoapi/provider/xarray_.py | 96 +++++++++++++++++-- 3 files changed, 104 insertions(+), 10 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-coverages.rst b/docs/source/data-publishing/ogcapi-coverages.rst index 97230df35..76ed0de39 100644 --- a/docs/source/data-publishing/ogcapi-coverages.rst +++ b/docs/source/data-publishing/ogcapi-coverages.rst @@ -75,6 +75,9 @@ The `Xarray`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data. x_field: lon y_field: lat time_field: time + # optionally specify the coordinate reference system of your dataset + # else pygeoapi assumes it is WGS84 (EPSG:4326). + storage_crs: 4326 format: name: netcdf mimetype: application/x-netcdf @@ -96,6 +99,11 @@ The `Xarray`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data. be sure to provide the full S3 URL. Any parameters required to open the dataset using fsspec can be added to the config file under `options` and `s3`. +.. note:: + When providing a `storage_crs` value in the xarray configuration, specify the + coordinate reference system using any valid input for + `pyproj.CRS.from_user_input`_. + Data access examples -------------------- @@ -146,3 +154,4 @@ Data access examples .. _`NetCDF`: https://en.wikipedia.org/wiki/NetCDF .. _`Zarr`: https://zarr.readthedocs.io/en/stable .. _`GDAL raster driver short name`: https://gdal.org/drivers/raster/index.html +.. _`pyproj.CRS.from_user_input`: https://pyproj4.github.io/pyproj/stable/api/crs/coordinate_system.html#pyproj.crs.CoordinateSystem.from_user_input diff --git a/docs/source/data-publishing/ogcapi-edr.rst b/docs/source/data-publishing/ogcapi-edr.rst index 599a711b3..83a2a76bd 100644 --- a/docs/source/data-publishing/ogcapi-edr.rst +++ b/docs/source/data-publishing/ogcapi-edr.rst @@ -47,6 +47,9 @@ The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data x_field: lon y_field: lat time_field: time + # optionally specify the coordinate reference system of your dataset + # else pygeoapi assumes it is WGS84 (EPSG:4326). + storage_crs: 4326 format: name: netcdf mimetype: application/x-netcdf @@ -81,6 +84,11 @@ The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data S3 URL. Any parameters required to open the dataset using fsspec can be added to the config file under `options` and `s3`, as shown above. +.. note:: + When providing a `storage_crs` value in the EDR configuration, specify the + coordinate reference system using any valid input for + `pyproj.CRS.from_user_input`_. + Data access examples -------------------- @@ -105,6 +113,7 @@ Data access examples .. _`xarray`: https://docs.xarray.dev/en/stable/ .. _`NetCDF`: https://en.wikipedia.org/wiki/NetCDF .. _`Zarr`: https://zarr.readthedocs.io/en/stable +.. _`pyproj.CRS.from_user_input`: https://pyproj4.github.io/pyproj/stable/api/crs/coordinate_system.html#pyproj.crs.CoordinateSystem.from_user_input .. _`OGC Environmental Data Retrieval (EDR) (API)`: https://github.com/opengeospatial/ogcapi-coverages diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index f06bb1053..585879282 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -37,12 +37,16 @@ import xarray import fsspec import numpy as np +import pyproj +from pyproj.exceptions import CRSError + +from pygeoapi.api import DEFAULT_STORAGE_CRS from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderNoDataError, ProviderQueryError) -from pygeoapi.util import read_data +from pygeoapi.util import get_crs_from_uri, read_data LOGGER = logging.getLogger(__name__) @@ -82,6 +86,7 @@ def __init__(self, provider_def): data_to_open = self.data self._data = open_func(data_to_open) + self.storage_crs = self._parse_storage_crs(provider_def) self._coverage_properties = self._get_coverage_properties() self.axes = [self._coverage_properties['x_axis_label'], @@ -341,6 +346,7 @@ def gen_covjson(self, metadata, data, fields): def _get_coverage_properties(self): """ Helper function to normalize coverage properties + :param provider_def: provider definition :returns: `dict` of coverage properties """ @@ -401,16 +407,21 @@ def _get_coverage_properties(self): 'restime': self.get_time_resolution() } - if 'crs' in self._data.variables.keys(): - try: - properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa - - properties['inverse_flattening'] = self._data.crs.\ - inverse_flattening - + # Update properties based on the xarray's CRS + epsg_code = self.storage_crs.to_epsg() + LOGGER.debug(f'{epsg_code}') + if epsg_code == 4326 or self.storage_crs == 'OGC:CRS84': + pass + LOGGER.debug('Confirmed default of WGS 84') + else: + properties['bbox_crs'] = \ + f'https://www.opengis.net/def/crs/EPSG/0/{epsg_code}' + properties['inverse_flattening'] = \ + self.storage_crs.ellipsoid.inverse_flattening + if self.storage_crs.is_projected: properties['crs_type'] = 'ProjectedCRS' - except AttributeError: - pass + + LOGGER.debug(f'properties: {properties}') properties['axes'] = [ properties['x_axis_label'], @@ -476,6 +487,71 @@ def get_time_coverage_duration(self): return ', '.join(times) + def _parse_grid_mapping(self): + """ + Identifies grid_mapping. + + :returns: name of xarray data variable that contains CRS information. + """ + LOGGER.debug('Parsing grid mapping...') + spatiotemporal_dims = (self.time_field, self.y_field, self.x_field) + LOGGER.debug(spatiotemporal_dims) + grid_mapping_name = None + for var_name, var in self._data.variables.items(): + if all(dim in var.dims for dim in spatiotemporal_dims): + try: + grid_mapping_name = self._data[var_name].attrs['grid_mapping'] # noqa + LOGGER.debug(f'Grid mapping: {grid_mapping_name}') + except KeyError as err: + LOGGER.debug(err) + LOGGER.debug('No grid mapping information found.') + return grid_mapping_name + + def _parse_storage_crs( + self, + provider_def: dict + ) -> pyproj.CRS: + """ + Parse the storage CRS from an xarray dataset. + + :param provider_def: provider definition + + :returns: `pyproj.CRS` instance parsed from dataset + """ + storage_crs = None + + try: + storage_crs = provider_def['storage_crs'] + crs_function = pyproj.CRS.from_user_input + except KeyError as err: + LOGGER.debug(err) + LOGGER.debug('No storage_crs found. Attempting to parse the CRS.') + + if storage_crs is None: + grid_mapping = self._parse_grid_mapping() + if grid_mapping is not None: + storage_crs = self._data[grid_mapping].attrs + crs_function = pyproj.CRS.from_cf + elif 'crs' in self._data.variables.keys(): + storage_crs = self._data['crs'].attrs + crs_function = pyproj.CRS.from_dict + else: + storage_crs = DEFAULT_STORAGE_CRS + crs_function = get_crs_from_uri + LOGGER.debug('Failed to parse dataset CRS. Assuming WGS84.') + + LOGGER.debug(f'Parsing CRS {storage_crs} with {crs_function}') + try: + crs = crs_function(storage_crs) + except CRSError as err: + LOGGER.debug(f'Unable to parse projection with pyproj: {err}') + LOGGER.debug('Assuming default WGS84.') + crs = get_crs_from_uri(DEFAULT_STORAGE_CRS) + + LOGGER.debug(crs) + + return crs + def _to_datetime_string(datetime_obj): """ From 76fd1304938f4b01e59b9d867bbb00840afd9f18 Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Sat, 21 Sep 2024 06:06:18 +0300 Subject: [PATCH 60/74] Update Ubuntu Jammy docker base image to 20240911.1 (#1815) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 71505ba98..04f1a20db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ # # ================================================================= -FROM ubuntu:jammy-20240627.1 +FROM ubuntu:jammy-20240911.1 LABEL maintainer="Just van den Broecke " From 52bec0fa8905c0ed03c9c810dc9f3c577211cd66 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 21 Sep 2024 14:19:43 -0400 Subject: [PATCH 61/74] docs: update compliance for OGC API - Processes (#1817) * Update compliance for OGC API - Processes * Update introduction.rst --- docs/source/_templates/indexsidebar.html | 7 +++++-- docs/source/introduction.rst | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/_templates/indexsidebar.html b/docs/source/_templates/indexsidebar.html index 8000f32dd..a782c3ba6 100644 --- a/docs/source/_templates/indexsidebar.html +++ b/docs/source/_templates/indexsidebar.html @@ -16,11 +16,14 @@ OGC Reference Implementation + + OGC Reference Implementation + OSGeo Project - - FOSS4G Conference + + FOSS4G Conference

    diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index a98b218c2..d41ff33f4 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -14,12 +14,12 @@ Features * OGC API - Features * OGC API - Environmental Data Retrieval * OGC API - Tiles + * OGC API - Processes * additionally implements * OGC API - Coverages * OGC API - Maps - * OGC API - Processes * OGC API - Records * SpatioTemporal Asset Library @@ -52,7 +52,7 @@ Standards are at the core of pygeoapi. Below is the project's standards support `OGC API - Coverages`_,Implementing `OGC API - Maps`_,Implementing `OGC API - Tiles`_,Reference Implementation - `OGC API - Processes`_,Implementing + `OGC API - Processes`_,Compliant `OGC API - Records`_,Implementing `OGC API - Environmental Data Retrieval`_,Reference Implementation `SpatioTemporal Asset Catalog`_,Implementing From 6b91024aa53422e7377399100a6b04235ab96dc5 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Thu, 26 Sep 2024 19:23:18 -0400 Subject: [PATCH 62/74] Zoom to first layer on EDR (#1819) --- pygeoapi/templates/collections/edr/query.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi/templates/collections/edr/query.html b/pygeoapi/templates/collections/edr/query.html index ac7f17d2d..d3af4ce3d 100644 --- a/pygeoapi/templates/collections/edr/query.html +++ b/pygeoapi/templates/collections/edr/query.html @@ -139,7 +139,7 @@ if (!firstLayer) { firstLayer = layer; layer.on('afterAdd', () => { - zoomToLayers([layers]) + zoomToLayers([layer]) if (!cov.coverages) { if (isVerticalProfile(cov) || isTimeSeries(cov)) { layer.openPopup(); From 83ef1ac1743f5c62d2c73ca5dc5eae50d96173ea Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Fri, 27 Sep 2024 20:15:05 +0300 Subject: [PATCH 63/74] update release version --- docs/source/conf.py | 2 +- pygeoapi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index dff89f13c..8f39a4ace 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.18.dev0' +version = '0.18.0' # The full version, including alpha/beta/rc tags. release = version diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index e66240e72..b6c75d9e4 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -30,7 +30,7 @@ # # ================================================================= -__version__ = '0.18.dev0' +__version__ = '0.18.0' import click try: From b3a70719a2f2de6f0327989cde533b17322a1bdc Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Fri, 27 Sep 2024 20:39:45 +0300 Subject: [PATCH 64/74] back to dev --- docs/source/conf.py | 2 +- pygeoapi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8f39a4ace..6a9e11d61 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.18.0' +version = '0.19.dev0' # The full version, including alpha/beta/rc tags. release = version diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index b6c75d9e4..9c5d35688 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -30,7 +30,7 @@ # # ================================================================= -__version__ = '0.18.0' +__version__ = '0.19.dev0' import click try: From 474cb60d823f1986d9d463bb17e1c661eb91ea5a Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 29 Sep 2024 12:07:24 -0400 Subject: [PATCH 65/74] fix item queryables provider handling (#1820) * fix queryables provider handling * fix test --- pygeoapi/api/itemtypes.py | 29 ++++++++++---------------- tests/api/test_api.py | 4 ++-- tests/api/test_itemtypes.py | 8 +++++++ tests/pygeoapi-test-config.yml | 38 ++++++++++++++++++++++++++++++++++ tests/test_util.py | 2 +- 5 files changed, 60 insertions(+), 21 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 661f5cd80..008b28cb7 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -121,29 +121,22 @@ def get_collection_queryables(api: API, request: Union[APIRequest, Any], HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) LOGGER.debug('Creating collection queryables') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - api.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: + + p = None + for pt in ['feature', 'coverage', 'record']: try: - LOGGER.debug('Loading coverage provider') + LOGGER.debug(f'Loading {pt} provider') p = load_plugin('provider', get_provider_by_type( - api.config['resources'][dataset]['providers'], 'coverage')) # noqa + api.config['resources'][dataset]['providers'], pt)) + break except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - api.config['resources'][dataset]['providers'], 'record')) - finally: - msg = 'queryables not available for this collection' - return api.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableError', msg) + LOGGER.debug(f'Providing type {pt} not found') - except ProviderGenericError as err: + if p is None: + msg = 'queryables not available for this collection' return api.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableError', msg) queryables = { 'type': 'object', diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 0477ef81f..faad9ed09 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -575,7 +575,7 @@ def test_conformance(config, api_): assert isinstance(root, dict) assert 'conformsTo' in root - assert len(root['conformsTo']) == 37 + assert len(root['conformsTo']) == 42 assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ in root['conformsTo'] @@ -604,7 +604,7 @@ def test_describe_collections(config, api_): collections = json.loads(response) assert len(collections) == 2 - assert len(collections['collections']) == 9 + assert len(collections['collections']) == 10 assert len(collections['links']) == 3 rsp_headers, code, response = api_.describe_collections(req, 'foo') diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py index 2cd445898..ae19c28d6 100644 --- a/tests/api/test_itemtypes.py +++ b/tests/api/test_itemtypes.py @@ -79,6 +79,14 @@ def test_get_collection_queryables(config, api_): assert 'properties' in queryables assert len(queryables['properties']) == 5 + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'canada-metadata') # noqa + assert rsp_headers['Content-Type'] == 'application/schema+json' + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 10 + # test with provider filtered properties api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 95a868631..58b62484f 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -398,6 +398,44 @@ resources: name: png mimetype: image/png + canada-metadata: + type: collection + title: + en: Open Canada sample data + fr: Exemple de donn\u00e9es Canada Ouvert + description: + en: Sample metadata records from open.canada.ca + fr: Exemples d'enregistrements de m\u00e9tadonn\u00e9es sur ouvert.canada.ca + keywords: + en: + - canada + - open data + fr: + - canada + - donn\u00e9es ouvertes + links: + - type: text/html + rel: canonical + title: information + href: https://open.canada.ca/en/open-data + hreflang: en-CA + - type: text/html + rel: alternate + title: informations + href: https://ouvert.canada.ca/fr/donnees-ouvertes + hreflang: fr-CA + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: record + name: TinyDBCatalogue + data: tests/data/open.canada.ca/sample-records.tinydb + id_field: externalId + time_field: created + title_field: title + hello-world: type: process processor: diff --git a/tests/test_util.py b/tests/test_util.py index c71ce80a0..d15aac321 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -172,7 +172,7 @@ def test_path_basename(): def test_filter_dict_by_key_value(config): collections = util.filter_dict_by_key_value(config['resources'], 'type', 'collection') - assert len(collections) == 9 + assert len(collections) == 10 notfound = util.filter_dict_by_key_value(config['resources'], 'type', 'foo') From d240a8210e0ea35540030152529892dd70651aa2 Mon Sep 17 00:00:00 2001 From: Leo Ghignone Date: Mon, 30 Sep 2024 19:40:31 +1000 Subject: [PATCH 66/74] Improvements for xarray provider (#1800) * Manage non-cf-compliant time dimension * Manage datasets without a time dimension * Allow reversed slices also for axes * Convert also metadata to float64 for json output * Use named temporary file to enable netcdf4 engine * Make float64 conversion faster * Add netcdf output to xarray provider * Flake8 fixes * Fix bug when no time axis in data * Use new xarray interface * Add test for zarr dataset without time dimension * Avoid errors if missing long_name * Manage zarr and netcdf output in the same way * Revert "Manage zarr and netcdf output in the same way" This reverts commit 0b09281b608da95221951d05004f213379da168d. * Revert "Add netcdf output to xarray provider" This reverts commit 9f72bf7614775b418f53f4808fcaeab567c7024a. --- pygeoapi/provider/xarray_.py | 158 +++++++++++++++++++---------- tests/test_xarray_zarr_provider.py | 26 +++++ 2 files changed, 129 insertions(+), 55 deletions(-) diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index 585879282..ba835f033 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -85,13 +85,19 @@ def __init__(self, provider_def): else: data_to_open = self.data - self._data = open_func(data_to_open) + try: + self._data = open_func(data_to_open) + except ValueError as err: + # Manage non-cf-compliant time dimensions + if 'time' in str(err): + self._data = open_func(self.data, decode_times=False) + else: + raise err + self.storage_crs = self._parse_storage_crs(provider_def) self._coverage_properties = self._get_coverage_properties() - self.axes = [self._coverage_properties['x_axis_label'], - self._coverage_properties['y_axis_label'], - self._coverage_properties['time_axis_label']] + self.axes = self._coverage_properties['axes'] self.get_fields() except Exception as err: @@ -101,7 +107,7 @@ def __init__(self, provider_def): def get_fields(self): if not self._fields: for key, value in self._data.variables.items(): - if len(value.shape) >= 3: + if key not in self._data.coords: LOGGER.debug('Adding variable') dtype = value.dtype if dtype.name.startswith('float'): @@ -109,7 +115,7 @@ def get_fields(self): self._fields[key] = { 'type': dtype, - 'title': value.attrs['long_name'], + 'title': value.attrs.get('long_name'), 'x-ogc-unit': value.attrs.get('units') } @@ -142,9 +148,9 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, data = self._data[[*properties]] - if any([self._coverage_properties['x_axis_label'] in subsets, - self._coverage_properties['y_axis_label'] in subsets, - self._coverage_properties['time_axis_label'] in subsets, + if any([self._coverage_properties.get('x_axis_label') in subsets, + self._coverage_properties.get('y_axis_label') in subsets, + self._coverage_properties.get('time_axis_label') in subsets, datetime_ is not None]): LOGGER.debug('Creating spatio-temporal subset') @@ -163,18 +169,36 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, self._coverage_properties['y_axis_label'] in subsets, len(bbox) > 0]): msg = 'bbox and subsetting by coordinates are exclusive' - LOGGER.warning(msg) + LOGGER.error(msg) raise ProviderQueryError(msg) else: - query_params[self._coverage_properties['x_axis_label']] = \ - slice(bbox[0], bbox[2]) - query_params[self._coverage_properties['y_axis_label']] = \ - slice(bbox[1], bbox[3]) + x_axis_label = self._coverage_properties['x_axis_label'] + x_coords = data.coords[x_axis_label] + if x_coords.values[0] > x_coords.values[-1]: + LOGGER.debug( + 'Reversing slicing of x axis from high to low' + ) + query_params[x_axis_label] = slice(bbox[2], bbox[0]) + else: + query_params[x_axis_label] = slice(bbox[0], bbox[2]) + y_axis_label = self._coverage_properties['y_axis_label'] + y_coords = data.coords[y_axis_label] + if y_coords.values[0] > y_coords.values[-1]: + LOGGER.debug( + 'Reversing slicing of y axis from high to low' + ) + query_params[y_axis_label] = slice(bbox[3], bbox[1]) + else: + query_params[y_axis_label] = slice(bbox[1], bbox[3]) LOGGER.debug('bbox_crs is not currently handled') if datetime_ is not None: - if self._coverage_properties['time_axis_label'] in subsets: + if self._coverage_properties['time_axis_label'] is None: + msg = 'Dataset does not contain a time axis' + LOGGER.error(msg) + raise ProviderQueryError(msg) + elif self._coverage_properties['time_axis_label'] in subsets: msg = 'datetime and temporal subsetting are exclusive' LOGGER.error(msg) raise ProviderQueryError(msg) @@ -196,13 +220,15 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, LOGGER.warning(err) raise ProviderQueryError(err) - if (any([data.coords[self.x_field].size == 0, - data.coords[self.y_field].size == 0, - data.coords[self.time_field].size == 0])): + if any(size == 0 for size in data.sizes.values()): msg = 'No data found' LOGGER.warning(msg) raise ProviderNoDataError(msg) + if format_ == 'json': + # json does not support float32 + data = _convert_float32_to_float64(data) + out_meta = { 'bbox': [ data.coords[self.x_field].values[0], @@ -210,18 +236,20 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, data.coords[self.x_field].values[-1], data.coords[self.y_field].values[-1] ], - "time": [ - _to_datetime_string(data.coords[self.time_field].values[0]), - _to_datetime_string(data.coords[self.time_field].values[-1]) - ], "driver": "xarray", "height": data.sizes[self.y_field], "width": data.sizes[self.x_field], - "time_steps": data.sizes[self.time_field], "variables": {var_name: var.attrs for var_name, var in data.variables.items()} } + if self.time_field is not None: + out_meta['time'] = [ + _to_datetime_string(data.coords[self.time_field].values[0]), + _to_datetime_string(data.coords[self.time_field].values[-1]), + ] + out_meta["time_steps"] = data.sizes[self.time_field] + LOGGER.debug('Serializing data in memory') if format_ == 'json': LOGGER.debug('Creating output in CoverageJSON') @@ -230,9 +258,11 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, LOGGER.debug('Returning data in native zarr format') return _get_zarr_data(data) else: # return data in native format - with tempfile.TemporaryFile() as fp: + with tempfile.NamedTemporaryFile() as fp: LOGGER.debug('Returning data in native NetCDF format') - fp.write(data.to_netcdf()) + data.to_netcdf( + fp.name + ) # we need to pass a string to be able to use the "netcdf4" engine # noqa fp.seek(0) return fp.read() @@ -249,7 +279,6 @@ def gen_covjson(self, metadata, data, fields): LOGGER.debug('Creating CoverageJSON domain') minx, miny, maxx, maxy = metadata['bbox'] - mint, maxt = metadata['time'] selected_fields = { key: value for key, value in self.fields.items() @@ -285,11 +314,6 @@ def gen_covjson(self, metadata, data, fields): 'start': maxy, 'stop': miny, 'num': metadata['height'] - }, - self.time_field: { - 'start': mint, - 'stop': maxt, - 'num': metadata['time_steps'] } }, 'referencing': [{ @@ -304,6 +328,14 @@ def gen_covjson(self, metadata, data, fields): 'ranges': {} } + if self.time_field is not None: + mint, maxt = metadata['time'] + cj['domain']['axes'][self.time_field] = { + 'start': mint, + 'stop': maxt, + 'num': metadata['time_steps'], + } + for key, value in selected_fields.items(): parameter = { 'type': 'Parameter', @@ -322,7 +354,6 @@ def gen_covjson(self, metadata, data, fields): cj['parameters'][key] = parameter data = data.fillna(None) - data = _convert_float32_to_float64(data) try: for key, value in selected_fields.items(): @@ -330,13 +361,18 @@ def gen_covjson(self, metadata, data, fields): 'type': 'NdArray', 'dataType': value['type'], 'axisNames': [ - 'y', 'x', self._coverage_properties['time_axis_label'] + 'y', 'x' ], 'shape': [metadata['height'], - metadata['width'], - metadata['time_steps']] + metadata['width']] } cj['ranges'][key]['values'] = data[key].values.flatten().tolist() # noqa + + if self.time_field is not None: + cj['ranges'][key]['axisNames'].append( + self._coverage_properties['time_axis_label'] + ) + cj['ranges'][key]['shape'].append(metadata['time_steps']) except IndexError as err: LOGGER.warning(err) raise ProviderQueryError('Invalid query parameter') @@ -382,31 +418,37 @@ def _get_coverage_properties(self): self._data.coords[self.x_field].values[-1], self._data.coords[self.y_field].values[-1], ], - 'time_range': [ - _to_datetime_string( - self._data.coords[self.time_field].values[0] - ), - _to_datetime_string( - self._data.coords[self.time_field].values[-1] - ) - ], 'bbox_crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', 'crs_type': 'GeographicCRS', 'x_axis_label': self.x_field, 'y_axis_label': self.y_field, - 'time_axis_label': self.time_field, 'width': self._data.sizes[self.x_field], 'height': self._data.sizes[self.y_field], - 'time': self._data.sizes[self.time_field], - 'time_duration': self.get_time_coverage_duration(), 'bbox_units': 'degrees', - 'resx': np.abs(self._data.coords[self.x_field].values[1] - - self._data.coords[self.x_field].values[0]), - 'resy': np.abs(self._data.coords[self.y_field].values[1] - - self._data.coords[self.y_field].values[0]), - 'restime': self.get_time_resolution() + 'resx': np.abs( + self._data.coords[self.x_field].values[1] + - self._data.coords[self.x_field].values[0] + ), + 'resy': np.abs( + self._data.coords[self.y_field].values[1] + - self._data.coords[self.y_field].values[0] + ), } + if self.time_field is not None: + properties['time_axis_label'] = self.time_field + properties['time_range'] = [ + _to_datetime_string( + self._data.coords[self.time_field].values[0] + ), + _to_datetime_string( + self._data.coords[self.time_field].values[-1] + ), + ] + properties['time'] = self._data.sizes[self.time_field] + properties['time_duration'] = self.get_time_coverage_duration() + properties['restime'] = self.get_time_resolution() + # Update properties based on the xarray's CRS epsg_code = self.storage_crs.to_epsg() LOGGER.debug(f'{epsg_code}') @@ -425,10 +467,12 @@ def _get_coverage_properties(self): properties['axes'] = [ properties['x_axis_label'], - properties['y_axis_label'], - properties['time_axis_label'] + properties['y_axis_label'] ] + if self.time_field is not None: + properties['axes'].append(properties['time_axis_label']) + return properties @staticmethod @@ -455,7 +499,8 @@ def get_time_resolution(self): :returns: time resolution string """ - if self._data[self.time_field].size > 1: + if self.time_field is not None \ + and self._data[self.time_field].size > 1: time_diff = (self._data[self.time_field][1] - self._data[self.time_field][0]) @@ -472,6 +517,9 @@ def get_time_coverage_duration(self): :returns: time coverage duration string """ + if self.time_field is None: + return None + dur = self._data[self.time_field][-1] - self._data[self.time_field][0] ms_difference = dur.values.astype('timedelta64[ms]').astype(np.double) @@ -634,7 +682,7 @@ def _convert_float32_to_float64(data): for var_name in data.variables: if data[var_name].dtype == 'float32': og_attrs = data[var_name].attrs - data[var_name] = data[var_name].astype('float64') + data[var_name] = data[var_name].astype('float64', copy=False) data[var_name].attrs = og_attrs return data diff --git a/tests/test_xarray_zarr_provider.py b/tests/test_xarray_zarr_provider.py index 5163b32a6..ec014e655 100644 --- a/tests/test_xarray_zarr_provider.py +++ b/tests/test_xarray_zarr_provider.py @@ -30,6 +30,7 @@ from numpy import float64, int64 import pytest +import xarray as xr from pygeoapi.provider.xarray_ import XarrayProvider from pygeoapi.util import json_serial @@ -53,6 +54,20 @@ def config(): } +@pytest.fixture() +def config_no_time(tmp_path): + ds = xr.open_zarr(path) + ds = ds.sel(time=ds.time[0]) + ds = ds.drop_vars('time') + ds.to_zarr(tmp_path / 'no_time.zarr') + return { + 'name': 'zarr', + 'type': 'coverage', + 'data': str(tmp_path / 'no_time.zarr'), + 'format': {'name': 'zarr', 'mimetype': 'application/zip'}, + } + + def test_provider(config): p = XarrayProvider(config) @@ -85,3 +100,14 @@ def test_numpy_json_serial(): d = float64(500.00000005) assert json_serial(d) == 500.00000005 + + +def test_no_time(config_no_time): + p = XarrayProvider(config_no_time) + + assert len(p.fields) == 4 + assert p.axes == ['lon', 'lat'] + + coverage = p.query(format='json') + + assert sorted(coverage['domain']['axes'].keys()) == ['x', 'y'] From e736fa3b2fa7cb168ecd4736aa80de215245dc5f Mon Sep 17 00:00:00 2001 From: Colin Henderson Date: Tue, 1 Oct 2024 15:53:39 +0100 Subject: [PATCH 67/74] Custom esri token service (#1813) * Added ability for self-hosted token service to be specified. * Update documentation to show the available parameters * Update pygeoapi/provider/esri.py Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * Update pygeoapi/provider/esri.py Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * Update pygeoapi/provider/esri.py Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * Update pygeoapi/provider/esri.py Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> * Update pygeoapi/provider/esri.py * Update ogcapi-features.rst --------- Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Co-authored-by: Tom Kralidis --- .../data-publishing/ogcapi-features.rst | 24 ++++++++++++++++++- pygeoapi/provider/esri.py | 8 +++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index 00d48a020..12478c1d2 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -145,7 +145,11 @@ To publish an ESRI `Feature Service`_ or `Map Service`_ specify the URL for the * ``id_field`` will often be ``OBJECTID``, ``objectid``, or ``FID``. * If the map or feature service is not shared publicly, the ``username`` and ``password`` fields can be set in the - configuration to authenticate into the service. + configuration to authenticate to the service. +* If the map or feature service is self-hosted and not shared publicly, the ``token_service`` and optional ``referer`` fields + can be set in the configuration to authenticate to the service. + +To publish from an ArcGIS online hosted service: .. code-block:: yaml @@ -158,6 +162,24 @@ To publish an ESRI `Feature Service`_ or `Map Service`_ specify the URL for the crs: 4326 # Optional crs (default is EPSG:4326) username: username # Optional ArcGIS username password: password # Optional ArcGIS password + token_service: https://your.server.com/arcgis/sharing/rest/generateToken # optional URL to your generateToken service + referer: https://your.server.com # optional referer, defaults to https://www.arcgis.com if not set + +To publish from a self-hosted service that is not publicly accessible, the ``token_service`` field is required: + +.. code-block:: yaml + + providers: + - type: feature + name: ESRI + data: https://your.server.com/arcgis/rest/services/your-layer/MapServer/0 + id_field: objectid + time_field: date_in_your_device_time_zone # Optional time field + crs: 4326 # Optional crs (default is EPSG:4326) + username: username # Optional ArcGIS username + password: password # Optional ArcGIS password + token_service: https://your.server.com/arcgis/sharing/rest/generateToken # Optional url to your generateToken service + referer: https://your.server.com # Optional referer, defaults to https://www.arcgis.com if not set GeoJSON ^^^^^^^ diff --git a/pygeoapi/provider/esri.py b/pygeoapi/provider/esri.py index 8179a705b..47d74e2b9 100644 --- a/pygeoapi/provider/esri.py +++ b/pygeoapi/provider/esri.py @@ -62,8 +62,9 @@ def __init__(self, provider_def): self.crs = provider_def.get('crs', '4326') self.username = provider_def.get('username') self.password = provider_def.get('password') + self.token_url = provider_def.get('token_service', ARCGIS_URL) + self.token_referer = provider_def.get('referer', GENERATE_TOKEN_URL) self.token = None - self.session = Session() self.login() @@ -194,16 +195,15 @@ def login(self): msg = 'Missing ESRI login information, not setting token' LOGGER.debug(msg) return - params = { 'f': 'pjson', 'username': self.username, 'password': self.password, - 'referer': ARCGIS_URL + 'referer': self.token_referer } LOGGER.debug('Logging in') - with self.session.post(GENERATE_TOKEN_URL, data=params) as r: + with self.session.post(self.token_url, data=params) as r: self.token = r.json().get('token') # https://enterprise.arcgis.com/en/server/latest/administer/windows/about-arcgis-tokens.htm self.session.headers.update({ From 179c90ff31261c4dedeeda103a85e839ceef20f2 Mon Sep 17 00:00:00 2001 From: Alex <94073946+Alex-NRCan@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:21:05 -0400 Subject: [PATCH 68/74] Fixes a memoryfile issue with rasterio. When using f=json, it doesn't need to use the MemoryFile. (#1824) --- pygeoapi/provider/rasterio_.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pygeoapi/provider/rasterio_.py b/pygeoapi/provider/rasterio_.py index 69d2dbca8..261754c1d 100644 --- a/pygeoapi/provider/rasterio_.py +++ b/pygeoapi/provider/rasterio_.py @@ -240,16 +240,15 @@ def query(self, properties=[], subsets={}, bbox=None, bbox_crs=4326, out_meta['units'] = _data.units LOGGER.debug('Serializing data in memory') - with MemoryFile() as memfile: - with memfile.open(**out_meta) as dest: - dest.write(out_image) - - if format_ == 'json': - LOGGER.debug('Creating output in CoverageJSON') - out_meta['bands'] = args['indexes'] - return self.gen_covjson(out_meta, out_image) - - else: # return data in native format + if format_ == 'json': + LOGGER.debug('Creating output in CoverageJSON') + out_meta['bands'] = args['indexes'] + return self.gen_covjson(out_meta, out_image) + + else: # return data in native format + with MemoryFile() as memfile: + with memfile.open(**out_meta) as dest: + dest.write(out_image) LOGGER.debug('Returning data in native format') return memfile.read() From 3bdeefe4e778ff5a0fd55637fc82a57a00bd0326 Mon Sep 17 00:00:00 2001 From: Leo Ghignone Date: Wed, 16 Oct 2024 06:49:15 +1100 Subject: [PATCH 69/74] Properly support int variables of any width (#1829) --- pygeoapi/provider/rasterio_.py | 2 ++ pygeoapi/provider/xarray_.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pygeoapi/provider/rasterio_.py b/pygeoapi/provider/rasterio_.py index 261754c1d..3b0fbc2c7 100644 --- a/pygeoapi/provider/rasterio_.py +++ b/pygeoapi/provider/rasterio_.py @@ -80,6 +80,8 @@ def get_fields(self): dtype2 = dtype if dtype.startswith('float'): dtype2 = 'number' + elif dtype.startswith('int'): + dtype2 = 'integer' self._fields[i2] = { 'title': name, diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index ba835f033..9ed2726b1 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -112,6 +112,8 @@ def get_fields(self): dtype = value.dtype if dtype.name.startswith('float'): dtype = 'number' + elif dtype.name.startswith('int'): + dtype = 'integer' self._fields[key] = { 'type': dtype, From d2f38dea07dc9905c7abf6324cb928728d6225d2 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 3 Nov 2024 11:21:32 -0500 Subject: [PATCH 70/74] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index c94189a7c..c8b2a6b48 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,4 @@ # These are supported funding model platforms #github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -custom: ['https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=WT27AS28UFSNW&source=url'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] +custom: ['https://github.com/geopython/pygeoapi/wiki/Sponsorship'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From b6c38b66eec2cd66b954a238496269f5a4993dff Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 4 Nov 2024 05:37:13 -0500 Subject: [PATCH 71/74] fix HEAD requests for items (#1836) (#1838) --- pygeoapi/flask_app.py | 10 +++++----- pygeoapi/starlette_app.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 4e4f0097e..296107019 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -279,11 +279,7 @@ def collection_items(collection_id, item_id=None): """ if item_id is None: - if request.method == 'GET': # list items - return execute_from_flask(itemtypes_api.get_collection_items, - request, collection_id, - skip_valid_check=True) - elif request.method == 'POST': # filter or manage items + if request.method == 'POST': # filter or manage items if request.content_type is not None: if request.content_type == 'application/geo+json': return execute_from_flask( @@ -298,6 +294,10 @@ def collection_items(collection_id, item_id=None): return execute_from_flask( itemtypes_api.manage_collection_item, request, 'options', collection_id, skip_valid_check=True) + else: # GET: list items + return execute_from_flask(itemtypes_api.get_collection_items, + request, collection_id, + skip_valid_check=True) elif request.method == 'DELETE': return execute_from_flask(itemtypes_api.manage_collection_item, diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 00ed727d2..af8798a05 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -334,11 +334,7 @@ async def collection_items(request: Request, collection_id=None, item_id=None): if 'item_id' in request.path_params: item_id = request.path_params['item_id'] if item_id is None: - if request.method == 'GET': # list items - return await execute_from_starlette( - itemtypes_api.get_collection_items, request, collection_id, - skip_valid_check=True) - elif request.method == 'POST': # filter or manage items + if request.method == 'POST': # filter or manage items content_type = request.headers.get('content-type') if content_type is not None: if content_type == 'application/geo+json': @@ -357,6 +353,10 @@ async def collection_items(request: Request, collection_id=None, item_id=None): itemtypes_api.manage_collection_item, request, 'options', collection_id, skip_valid_check=True, ) + else: # GET: list items + return await execute_from_starlette( + itemtypes_api.get_collection_items, request, collection_id, + skip_valid_check=True) elif request.method == 'DELETE': return await execute_from_starlette( From e4beaf758ea43617adb80766010495536bafa371 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 4 Nov 2024 05:37:51 -0500 Subject: [PATCH 72/74] add service contact to OpenAPI (#1835) (#1839) --- pygeoapi/openapi.py | 52 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 3cf5c0b8f..fe390b742 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -134,6 +134,52 @@ def gen_response_object(description: str, media_type: str, return response +def gen_contact(cfg: dict) -> dict: + """ + Generates an OpenAPI contact object with OGC extensions + based on OGC API - Records contact + + :param cfg: `dict` of configuration + + :returns: `dict` of OpenAPI contact object + """ + + contact = { + 'name': cfg['metadata']['provider']['name'], + 'url': cfg['metadata']['provider']['url'], + 'email': cfg['metadata']['contact']['email'] + } + + contact['x-ogc-serviceContact'] = { + 'name': cfg['metadata']['contact']['name'], + 'position': cfg['metadata']['contact']['position'], + 'addresses': [{ + 'deliveryPoint': [cfg['metadata']['contact']['address']], + 'city': cfg['metadata']['contact']['city'], + 'administrativeArea': cfg['metadata']['contact']['stateorprovince'], # noqa + 'postalCode': cfg['metadata']['contact']['postalcode'], + 'country': cfg['metadata']['contact']['country'] + }], + 'phones': [{ + 'type': 'main', 'value': cfg['metadata']['contact']['phone'] + }, { + 'type': 'fax', 'value': cfg['metadata']['contact']['fax'] + }], + 'emails': [{ + 'value': cfg['metadata']['contact']['email'] + }], + 'contactInstructions': cfg['metadata']['contact']['instructions'], + 'links': [{ + 'type': 'text/html', + 'href': cfg['metadata']['contact']['url'] + }], + 'hoursOfService': cfg['metadata']['contact']['hours'], + 'roles': [cfg['metadata']['contact']['role']] + } + + return contact + + def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: """ Generates an OpenAPI 3.0 Document @@ -167,11 +213,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'x-keywords': l10n.translate(cfg['metadata']['identification']['keywords'], locale_), # noqa 'termsOfService': cfg['metadata']['identification']['terms_of_service'], - 'contact': { - 'name': cfg['metadata']['provider']['name'], - 'url': cfg['metadata']['provider']['url'], - 'email': cfg['metadata']['contact']['email'] - }, + 'contact': gen_contact(cfg), 'license': { 'name': cfg['metadata']['license']['name'], 'url': cfg['metadata']['license']['url'] From 09423fb4bedfbb8b72df609c65ae6b11ae3e6fee Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 7 Nov 2024 23:09:09 +0000 Subject: [PATCH 73/74] Lifted pin on sqlalchemy in order to be able to use v2+ (#1832) --- .github/workflows/main.yml | 1 - pygeoapi/provider/postgresql.py | 5 +++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4d98b4478..3805b9aa0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -94,7 +94,6 @@ jobs: pip3 install -r requirements-manager.txt pip3 install -r requirements-django.txt python3 setup.py install - pip3 install --upgrade "sqlalchemy<2" pip3 install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` #pip3 install --upgrade rasterio==1.1.8 - name: setup test data ⚙️ diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index d56bb7878..eb4b21454 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -62,7 +62,8 @@ import shapely from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, desc from sqlalchemy.engine import URL -from sqlalchemy.exc import InvalidRequestError, OperationalError +from sqlalchemy.exc import ConstraintColumnNotFoundError, \ + InvalidRequestError, OperationalError from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import Session, load_only from sqlalchemy.sql.expression import and_ @@ -515,7 +516,7 @@ def get_table_model( sqlalchemy_table_def = metadata.tables[f'{schema}.{table_name}'] try: sqlalchemy_table_def.append_constraint(PrimaryKeyConstraint(id_field)) - except KeyError: + except (ConstraintColumnNotFoundError, KeyError): raise ProviderQueryError( f"No such id_field column ({id_field}) on {schema}.{table_name}.") diff --git a/requirements.txt b/requirements.txt index 601f51b83..8ea9bd299 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,5 @@ PyYAML rasterio requests shapely -SQLAlchemy<2.0.0 +SQLAlchemy tinydb From 9e87184fbf19cb783f917ba7a56de62debf7a9da Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Fri, 8 Nov 2024 03:36:10 -0500 Subject: [PATCH 74/74] fix trivy error on vulnerability testing (#1843) --- .github/workflows/vulnerabilities.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vulnerabilities.yml b/.github/workflows/vulnerabilities.yml index 3252252ed..d8ac5199c 100644 --- a/.github/workflows/vulnerabilities.yml +++ b/.github/workflows/vulnerabilities.yml @@ -22,7 +22,7 @@ jobs: working-directory: . steps: - name: Checkout pygeoapi - uses: actions/checkout@v4 + uses: actions/checkout@master - name: Scan vulnerabilities with trivy uses: aquasecurity/trivy-action@master with: @@ -37,6 +37,9 @@ jobs: docker buildx build -t ${{ github.repository }}:${{ github.sha }} --platform linux/amd64 --no-cache -f Dockerfile . - name: Scan locally built Docker image for vulnerabilities with trivy uses: aquasecurity/trivy-action@master + env: + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1 with: scan-type: image exit-code: 1