From 97d3ee15012a57c9465c7c33edb86bb45794a729 Mon Sep 17 00:00:00 2001 From: "thomas.grandjean" Date: Wed, 25 Sep 2024 14:15:18 +0200 Subject: [PATCH] Update datasets with resolution config --- Dockerfile | 2 + argo-pipeline/api.py | 11 +++-- argo-pipeline/src/split_merge_tiles.py | 14 ++++--- .../pipeline/mapshaper_split_from_s3.py | 42 ++++++++++--------- cartiflette/pipeline/prepare_geodatasets.py | 4 +- docker/test.py | 6 ++- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9863757..73b6d1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,4 +31,6 @@ COPY docker/test.py . RUN curl https://install.python-poetry.org/ | python - RUN poetry install --only main --no-interaction +# TODO : is this necessary? This should throw an exception if datasets have not +# already been (manually) uploaded ? CMD ["python", "test.py"] \ No newline at end of file diff --git a/argo-pipeline/api.py b/argo-pipeline/api.py index 4156cac..859feaf 100644 --- a/argo-pipeline/api.py +++ b/argo-pipeline/api.py @@ -1,10 +1,12 @@ """A simple API to expose cartiflette files""" + import typing from fastapi import FastAPI, Response from fastapi.responses import FileResponse from cartiflette.api import download_from_cartiflette_inner -from cartiflette.config import PATH_WITHIN_BUCKET +from cartiflette.config import PATH_WITHIN_BUCKET, DATASETS_HIGH_RESOLUTION +from cartiflette.pipeline_constants import COG_TERRITOIRE # , IRIS app = FastAPI( title="API de récupération des fonds de carte avec cartiflette", @@ -42,9 +44,10 @@ def download_from_cartiflette_api( year=2022, crs=4326, simplification=simplification, - provider="IGN", - dataset_family="ADMINEXPRESS", - source="EXPRESS-COG-CARTO-TERRITOIRE", + provider="Cartiflette", + dataset_family="production", + # TODO : source can also be IRIS[DATASETS_HIGH_RESOLUTION] + source=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION], return_as_json=False, path_within_bucket=PATH_WITHIN_BUCKET, ) diff --git a/argo-pipeline/src/split_merge_tiles.py b/argo-pipeline/src/split_merge_tiles.py index 36e3850..b82ef5c 100644 --- a/argo-pipeline/src/split_merge_tiles.py +++ b/argo-pipeline/src/split_merge_tiles.py @@ -14,12 +14,14 @@ from s3fs import S3FileSystem -from cartiflette.config import BUCKET, PATH_WITHIN_BUCKET, FS -from cartiflette.pipeline import ( - mapshaperize_split_from_s3, - # mapshaperize_merge_split_from_s3, +from cartiflette.config import ( + BUCKET, + PATH_WITHIN_BUCKET, + FS, + DATASETS_HIGH_RESOLUTION, ) - +from cartiflette.pipeline import mapshaperize_split_from_s3 +from cartiflette.pipeline_constants import COG_TERRITOIRE logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -46,7 +48,7 @@ parser.add_argument( "-so", "--source", - default="EXPRESS-COG-CARTO-TERRITOIRE", + default=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION], help="Select upstream raw source to use for geometries", ) parser.add_argument( diff --git a/cartiflette/pipeline/mapshaper_split_from_s3.py b/cartiflette/pipeline/mapshaper_split_from_s3.py index d6bef51..feb0f70 100644 --- a/cartiflette/pipeline/mapshaper_split_from_s3.py +++ b/cartiflette/pipeline/mapshaper_split_from_s3.py @@ -144,23 +144,25 @@ def mapshaperize_split_from_s3( return {"success": len(success), "skipped": len(skipped)} -if __name__ == "__main__": - import logging - - logging.basicConfig(level=logging.INFO) - - mapshaperize_split_from_s3( - year=2023, - init_geometry_level="ARRONDISSEMENT_MUNICIPAL", - source="EXPRESS-COG-CARTO-TERRITOIRE", - simplification=40, - dissolve_by="DEPARTEMENT", - config_generation={ - "FRANCE_ENTIERE_DROM_RAPPROCHES": [ - {"format_output": "gpkg", "epsg": "4326"}, - {"format_output": "geojson", "epsg": "4326"}, - {"format_output": "gpkg", "epsg": "2154"}, - {"format_output": "geojson", "epsg": "2154"}, - ] - }, - ) +# if __name__ == "__main__": +# import logging +# from cartiflette.pipeline_constants import COG_TERRITOIRE +# from cartiflette.config import DATASETS_HIGH_RESOLUTION + +# logging.basicConfig(level=logging.INFO) + +# mapshaperize_split_from_s3( +# year=2023, +# init_geometry_level="ARRONDISSEMENT_MUNICIPAL", +# source=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION], +# simplification=40, +# dissolve_by="DEPARTEMENT", +# config_generation={ +# "FRANCE_ENTIERE_DROM_RAPPROCHES": [ +# {"format_output": "gpkg", "epsg": "4326"}, +# {"format_output": "geojson", "epsg": "4326"}, +# {"format_output": "gpkg", "epsg": "2154"}, +# {"format_output": "geojson", "epsg": "2154"}, +# ] +# }, +# ) diff --git a/cartiflette/pipeline/prepare_geodatasets.py b/cartiflette/pipeline/prepare_geodatasets.py index 1a3328b..c638db1 100644 --- a/cartiflette/pipeline/prepare_geodatasets.py +++ b/cartiflette/pipeline/prepare_geodatasets.py @@ -23,10 +23,12 @@ PATH_WITHIN_BUCKET, THREADS_DOWNLOAD, INTERMEDIATE_FORMAT, + DATASETS_HIGH_RESOLUTION, ) from cartiflette.pipeline_constants import ( PIPELINE_DOWNLOAD_ARGS, PIPELINE_SIMPLIFICATION_LEVELS, + COG_TERRITOIRE, ) from cartiflette.s3.geodataset import ( S3GeoDataset, @@ -187,7 +189,7 @@ def create_one_year_geodataset_batch( "path_within_bucket": path_within_bucket, "provider": "IGN", "dataset_family": "ADMINEXPRESS", - "source": "EXPRESS-COG-CARTO-TERRITOIRE", + "source": COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION], "borders": None, "crs": "*", "filter_by": "origin", diff --git a/docker/test.py b/docker/test.py index 14f95a6..d91df02 100644 --- a/docker/test.py +++ b/docker/test.py @@ -1,5 +1,7 @@ import subprocess from cartiflette import carti_download +from cartiflette.pipeline_constants import COG_TERRITOIRE +from cartiflette.config import DATASETS_HIGH_RESOLUTION ile_de_france = carti_download( values=["11"], @@ -7,8 +9,10 @@ borders="DEPARTEMENT", vectorfile_format="geojson", filter_by="REGION", - source="EXPRESS-COG-CARTO-TERRITOIRE", + source=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION], year=2022, + provider="Cartiflette", + dataset_family="production", ) ile_de_france.to_file("idf.json")