Update datasets with resolution config

InseeFrLab · Sep 25, 2024 · 97d3ee1 · 97d3ee1
1 parent 24f6e06
commit 97d3ee1
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 32 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -31,4 +31,6 @@ COPY docker/test.py .
 RUN curl https://install.python-poetry.org/ | python - 
 RUN poetry install --only main --no-interaction
 
+# TODO : is this necessary? This should throw an exception if datasets have not
+# already been (manually) uploaded ?
 CMD ["python", "test.py"]
diff --git a/argo-pipeline/api.py b/argo-pipeline/api.py
@@ -1,10 +1,12 @@
 """A simple API to expose cartiflette files"""
+
 import typing
 from fastapi import FastAPI, Response
 from fastapi.responses import FileResponse
 
 from cartiflette.api import download_from_cartiflette_inner
-from cartiflette.config import PATH_WITHIN_BUCKET
+from cartiflette.config import PATH_WITHIN_BUCKET, DATASETS_HIGH_RESOLUTION
+from cartiflette.pipeline_constants import COG_TERRITOIRE  # , IRIS
 
 app = FastAPI(
     title="API de récupération des fonds de carte avec <code>cartiflette</code>",
@@ -42,9 +44,10 @@ def download_from_cartiflette_api(
         year=2022,
         crs=4326,
         simplification=simplification,
-        provider="IGN",
-        dataset_family="ADMINEXPRESS",
-        source="EXPRESS-COG-CARTO-TERRITOIRE",
+        provider="Cartiflette",
+        dataset_family="production",
+        # TODO : source can also be IRIS[DATASETS_HIGH_RESOLUTION]
+        source=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION],
         return_as_json=False,
         path_within_bucket=PATH_WITHIN_BUCKET,
     )

diff --git a/argo-pipeline/src/split_merge_tiles.py b/argo-pipeline/src/split_merge_tiles.py
@@ -14,12 +14,14 @@
 
 from s3fs import S3FileSystem
 
-from cartiflette.config import BUCKET, PATH_WITHIN_BUCKET, FS
-from cartiflette.pipeline import (
-    mapshaperize_split_from_s3,
-    # mapshaperize_merge_split_from_s3,
+from cartiflette.config import (
+    BUCKET,
+    PATH_WITHIN_BUCKET,
+    FS,
+    DATASETS_HIGH_RESOLUTION,
 )
-
+from cartiflette.pipeline import mapshaperize_split_from_s3
+from cartiflette.pipeline_constants import COG_TERRITOIRE
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
@@ -46,7 +48,7 @@
 parser.add_argument(
     "-so",
     "--source",
-    default="EXPRESS-COG-CARTO-TERRITOIRE",
+    default=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION],
     help="Select upstream raw source to use for geometries",
 )
 parser.add_argument(

diff --git a/cartiflette/pipeline/mapshaper_split_from_s3.py b/cartiflette/pipeline/mapshaper_split_from_s3.py
@@ -144,23 +144,25 @@ def mapshaperize_split_from_s3(
     return {"success": len(success), "skipped": len(skipped)}
 
 
-if __name__ == "__main__":
-    import logging
-
-    logging.basicConfig(level=logging.INFO)
-
-    mapshaperize_split_from_s3(
-        year=2023,
-        init_geometry_level="ARRONDISSEMENT_MUNICIPAL",
-        source="EXPRESS-COG-CARTO-TERRITOIRE",
-        simplification=40,
-        dissolve_by="DEPARTEMENT",
-        config_generation={
-            "FRANCE_ENTIERE_DROM_RAPPROCHES": [
-                {"format_output": "gpkg", "epsg": "4326"},
-                {"format_output": "geojson", "epsg": "4326"},
-                {"format_output": "gpkg", "epsg": "2154"},
-                {"format_output": "geojson", "epsg": "2154"},
-            ]
-        },
-    )
+# if __name__ == "__main__":
+#     import logging
+#     from cartiflette.pipeline_constants import COG_TERRITOIRE
+#     from cartiflette.config import DATASETS_HIGH_RESOLUTION
+
+#     logging.basicConfig(level=logging.INFO)
+
+#     mapshaperize_split_from_s3(
+#         year=2023,
+#         init_geometry_level="ARRONDISSEMENT_MUNICIPAL",
+#         source=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION],
+#         simplification=40,
+#         dissolve_by="DEPARTEMENT",
+#         config_generation={
+#             "FRANCE_ENTIERE_DROM_RAPPROCHES": [
+#                 {"format_output": "gpkg", "epsg": "4326"},
+#                 {"format_output": "geojson", "epsg": "4326"},
+#                 {"format_output": "gpkg", "epsg": "2154"},
+#                 {"format_output": "geojson", "epsg": "2154"},
+#             ]
+#         },
+#     )
diff --git a/cartiflette/pipeline/prepare_geodatasets.py b/cartiflette/pipeline/prepare_geodatasets.py
@@ -23,10 +23,12 @@
     PATH_WITHIN_BUCKET,
     THREADS_DOWNLOAD,
     INTERMEDIATE_FORMAT,
+    DATASETS_HIGH_RESOLUTION,
 )
 from cartiflette.pipeline_constants import (
     PIPELINE_DOWNLOAD_ARGS,
     PIPELINE_SIMPLIFICATION_LEVELS,
+    COG_TERRITOIRE,
 )
 from cartiflette.s3.geodataset import (
     S3GeoDataset,
@@ -187,7 +189,7 @@ def create_one_year_geodataset_batch(
         "path_within_bucket": path_within_bucket,
         "provider": "IGN",
         "dataset_family": "ADMINEXPRESS",
-        "source": "EXPRESS-COG-CARTO-TERRITOIRE",
+        "source": COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION],
         "borders": None,
         "crs": "*",
         "filter_by": "origin",

diff --git a/docker/test.py b/docker/test.py
@@ -1,14 +1,18 @@
 import subprocess
 from cartiflette import carti_download
+from cartiflette.pipeline_constants import COG_TERRITOIRE
+from cartiflette.config import DATASETS_HIGH_RESOLUTION
 
 ile_de_france = carti_download(
     values=["11"],
     crs=4326,
     borders="DEPARTEMENT",
     vectorfile_format="geojson",
     filter_by="REGION",
-    source="EXPRESS-COG-CARTO-TERRITOIRE",
+    source=COG_TERRITOIRE[DATASETS_HIGH_RESOLUTION],
     year=2022,
+    provider="Cartiflette",
+    dataset_family="production",
 )
 
 ile_de_france.to_file("idf.json")