From 79735459e5a8f7b830868d2846b18ed77c54230d Mon Sep 17 00:00:00 2001
From: Jairo Matos Da Rocha <jairo@geocodelabs.com>
Date: Thu, 3 Oct 2024 14:12:18 -0300
Subject: [PATCH] file upload
---
app/api/task.py | 83 +++++++++++++++++++++-------------
app/oauth2.py | 1 -
app/utils/file.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++
pyproject.toml | 1 +
uv.lock | 46 +++++++++++++++++++
5 files changed, 212 insertions(+), 32 deletions(-)
create mode 100644 app/utils/file.py
diff --git a/app/api/task.py b/app/api/task.py
index cf1b71c..6462f02 100644
--- a/app/api/task.py
+++ b/app/api/task.py
@@ -1,56 +1,51 @@
+from pathlib import Path
+import shutil
+import tempfile
+from typing import List
from fastapi.responses import JSONResponse
import geopandas as gpd
+from pydantic import EmailStr
from app.models.oauth2 import UserInfo
from app.oauth2 import has_role
+from app.utils.file import check_geofiles
from worker import gee_get_index_pasture
-from app.models.payload import PayloadSaveGeojson
+from app.models.payload import PayloadSaveGeojson, User
import os
-from fastapi import APIRouter, Depends, HTTPException, Request, Query
+from fastapi import APIRouter, Depends, File, HTTPException, Request, Query, UploadFile
from app.config import logger
from celery.result import AsyncResult
router = APIRouter()
-@router.post("/savegeom" )
-async def savegeom(
+@router.post("/savegeom/geojson" )
+async def savegeom_geojson(
payload: PayloadSaveGeojson,
- request: Request,
crs: int = Query(4326, description="EPSG code for the geometry"),
user_data: UserInfo = Depends(has_role(['savegeom'])) # This should be a function that retrieves user data from the request.
):
-
- MAXHECTARES = os.environ.get('MAXHECTARES',40_000)
geojson = payload.dict().get('geojson',gpd.GeoDataFrame())
+ user = payload.dict().get('user')
logger.info(f"Received payload: {geojson}")
- try:
- gdf = gpd.GeoDataFrame.from_features(geojson, crs=crs)
- if gdf.empty or len(gdf) > 1:
- return HTTPException(status_code=400, detail="Empty GeoDataFrame or more than one feature.")
- if gdf.geometry.type[0] != "Polygon":
- return HTTPException(status_code=400, detail="Geometry must be a Polygon.")
- if gdf.to_crs(5880).area.iloc[0] / 10_000 > MAXHECTARES:
- return HTTPException(status_code=400, detail=f"Geometry area must be less than {MAXHECTARES} hectares.")
- logger.info('Geometry is valid')
- logger.info(user_data)
- try:
- dict_payload = {
- **payload.dict(),
- 'request_user': user_data
- }
- task = gee_get_index_pasture.delay(dict_payload)
- except Exception as e:
- logger.exception(f"Failed to create task: {e}")
- raise HTTPException(status_code=400, detail="Failed to create task.")
- except Exception as e:
- logger.error(f"{e}")
- raise HTTPException(status_code=400, detail=e)
+ gdf = gpd.GeoDataFrame.from_features(geojson, crs=crs)
+ return __savegeom__(gdf, user, user_data)
+
+@router.post("/savegeom/file" )
+async def savegeom_gdf(
+ name: str,
+ email:EmailStr,
+ files: List[UploadFile] = File(...),
+ user_data: UserInfo = Depends(has_role(['savegeom'])) # This should be a function that retrieves user data from the request.
+):
+ return __savegeom__(check_geofiles(files), {'name':name,'email':email}, user_data)
+
+
+
- return JSONResponse({"task_id": task.id})
@router.get("/status/{task_id}")
def get_status(task_id):
@@ -60,4 +55,30 @@ def get_status(task_id):
"task_status": task_result.status,
"task_result": task_result.result
}
- return JSONResponse(result)
\ No newline at end of file
+ return JSONResponse(result)
+
+
+def __checkgeom__(gdf: gpd.GeoDataFrame):
+ MAXHECTARES = os.environ.get('MAXHECTARES',40_000)
+ if gdf.empty:
+ raise HTTPException(status_code=400, detail="Empty GeoDataFrame.")
+ if len(gdf) > 1:
+ raise HTTPException(status_code=400, detail="GeoDataFrame must have only one geometry.")
+ if gdf.geometry.type[0] != "Polygon":
+ raise HTTPException(status_code=400, detail="Geometry must be a Polygon.")
+ if gdf.to_crs(5880).area.iloc[0] / 10_000 > MAXHECTARES:
+ raise HTTPException(status_code=400, detail=f"Geometry area must be less than {MAXHECTARES} hectares.")
+ logger.info('Geometry is valid')
+ return gdf.to_crs(4326).to_geo_dict()
+
+
+
+def __savegeom__(gdf: gpd.GeoDataFrame, user: User,user_data: UserInfo):
+ dict_payload = {
+ 'user':user,
+ 'geojson':__checkgeom__(gdf),
+ 'request_user': user_data
+ }
+ logger.info(f"Starting task with payload: {dict_payload}")
+ task = gee_get_index_pasture.delay(dict_payload)
+ return JSONResponse({"task_id": task.id})
\ No newline at end of file
diff --git a/app/oauth2.py b/app/oauth2.py
index 8ae368c..f522789 100644
--- a/app/oauth2.py
+++ b/app/oauth2.py
@@ -49,7 +49,6 @@ async def get_idp_public_key():
# Get the payload/token from keycloak
async def get_payload(token: str = Security(oauth2_scheme)) -> dict:
key= await get_idp_public_key()
- logger.info(f"token {token}")
try:
return keycloak_openid.decode_token(
diff --git a/app/utils/file.py b/app/utils/file.py
new file mode 100644
index 0000000..c00244c
--- /dev/null
+++ b/app/utils/file.py
@@ -0,0 +1,113 @@
+import shutil
+import tempfile
+from zipfile import ZipFile
+from pathlib import Path
+from fastapi import HTTPException
+import geopandas as gpd
+from app.config import logger
+
+def valid_file_geo(content_type):
+
+ if content_type in [
+ 'application/text',
+ 'application/xml',
+ 'text/plain',
+ 'application/geo+json',
+ 'application/geopackage+sqlite3',
+ 'application/octet-stream',
+ 'application/vnd.google-earth.kml+xml',
+ 'application/vnd.google-earth.kmz',
+ 'application/x-dbf',
+ 'application/x-esri-crs',
+ 'application/x-esri-shape',
+ 'application/zip'
+ ]:
+ return True
+ raise HTTPException(status_code=415, detail=f'Invalid file type: {content_type}')
+
+def valid_extension_shp(file):
+ if file.suffix in [
+ ".shp", # Geometria dos vetores
+ ".shx", # Índice de geometria
+ ".dbf", # Dados tabulares
+ ".prj", # Sistema de coordenadas e projeção
+ ".cpg", # Codificação de caracteres
+ ".sbn", # Índice espacial
+ ".sbx", # Arquivo auxiliar para índice espacial
+ ".xml", # Metadados em formato XML
+ ".qix", # Índice espacial (gerado por software)
+ ".aih", # Índice de atributos para .dbf
+ ".ain", # Arquivo auxiliar para índice de atributos
+ ".qmd" # Extensão adicional (se aplicável)
+ ]:
+ return True
+ raise HTTPException(status_code=415, detail=f'Invalid file type: {file.suffix}')
+
+
+def check_geofiles(files):
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ logger.info(f"Saving files to {tmpdirname}")
+ logger.info(files)
+ if len(files) == 1:
+ valid_file_geo(files[0].content_type)
+ file = files[0]
+ else:
+ extensions = []
+ for f in files:
+ valid_file_geo(f.content_type)
+ valid_extension_shp(Path(f.filename))
+ extensions.append(Path(f.filename).suffix)
+ with open(f'{tmpdirname}/{f.filename}', 'wb') as buffer:
+ shutil.copyfileobj(f.file, buffer)
+ minal_shape =set(['.shp', '.shx', '.dbf', '.prj']) - set(extensions)
+ if len(minal_shape) > 0:
+ raise HTTPException(status_code=400, detail=f'Missing files: {minal_shape}')
+ file = str(get_geofile(tmpdirname))
+ return read_file(file)
+
+
+def read_kml(file):
+ import fiona
+ try:
+ return gpd.read_file(file, driver='KML')
+ except Exception as e:
+ gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw'
+ return gpd.read_file(file, driver='KML')
+
+
+def read_kmz(file):
+ import fiona
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ kmz = ZipFile(file, 'r')
+ kmz.extract('doc.kml',tmpdirname)
+ try:
+ gdf = gpd.read_file(f'{tmpdirname}/doc.kml')
+ except Exception as e:
+ gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw'
+ gdf = gpd.read_file(f'{tmpdirname}/doc.kml')
+
+ return gdf
+
+def read_gpd(file):
+ return gpd.read_file(file)
+
+
+def get_geofile(dirname):
+ for file in Path(dirname).glob('*'):
+ if file.suffix in ['.shp']:
+ return file
+
+
+def read_file(file):
+ if isinstance(file, str):
+ gdf = gpd.read_file(file)
+ else:
+ match Path(file.filename).suffix.capitalize():
+ case '.kml':
+ gdf = read_kml(file.file)
+ case '.kmz':
+ gdf = read_kmz(file.file)
+ case _:
+ gdf = read_gpd(file.file)
+
+ return gdf
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index f381afa..b2f54ec 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,6 +11,7 @@ dependencies = [
"earthengine-api>=1.1.2",
"fastapi-keycloak-middleware>=1.1.0",
"fastapi>=0.115.0",
+ "fiona>=1.10.1",
"flower>=2.0.1",
"geemap>=0.34.5",
"geopandas>=1.0.1",
diff --git a/uv.lock b/uv.lock
index f2f003d..a1d3817 100644
--- a/uv.lock
+++ b/uv.lock
@@ -60,6 +60,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c7/80/9f608d13b4b3afcebd1dd13baf9551c95fc424d6390e4b1cfd7b1810cd06/async_property-0.2.2-py2.py3-none-any.whl", hash = "sha256:8924d792b5843994537f8ed411165700b27b2bd966cefc4daeefc1253442a9d7", size = 9546 },
]
+[[package]]
+name = "attrs"
+version = "24.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", size = 792678 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2", size = 63001 },
+]
+
[[package]]
name = "beautifulsoup4"
version = "4.12.3"
@@ -253,6 +262,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289 },
]
+[[package]]
+name = "cligj"
+version = "0.7.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ea/0d/837dbd5d8430fd0f01ed72c4cfb2f548180f4c68c635df84ce87956cff32/cligj-0.7.2.tar.gz", hash = "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27", size = 9803 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/73/86/43fa9f15c5b9fb6e82620428827cd3c284aa933431405d1bcf5231ae3d3e/cligj-0.7.2-py3-none-any.whl", hash = "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df", size = 7069 },
+]
+
[[package]]
name = "colorama"
version = "0.4.6"
@@ -477,6 +498,29 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ce/ad/500df28e424188c2d992fa9b754366f6564782712c72e9578df76dd7ee9b/fastapi_keycloak_middleware-1.1.0-py3-none-any.whl", hash = "sha256:cc5cba9b24dd297f8bb2a77207df66c903b05c411863cd4c6fc79152b9c7cde2", size = 22300 },
]
+[[package]]
+name = "fiona"
+version = "1.10.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs" },
+ { name = "certifi" },
+ { name = "click" },
+ { name = "click-plugins" },
+ { name = "cligj" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/51/e0/71b63839cc609e1d62cea2fc9774aa605ece7ea78af823ff7a8f1c560e72/fiona-1.10.1.tar.gz", hash = "sha256:b00ae357669460c6491caba29c2022ff0acfcbde86a95361ea8ff5cd14a86b68", size = 444606 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/73/ab/036c418d531afb74abe4ca9a8be487b863901fe7b42ddba1ba2fb0681d77/fiona-1.10.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7338b8c68beb7934bde4ec9f49eb5044e5e484b92d940bc3ec27defdb2b06c67", size = 16114589 },
+ { url = "https://files.pythonhosted.org/packages/ba/45/693c1cca53023aaf6e3adc11422080f5fa427484e7b85e48f19c40d6357f/fiona-1.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c77fcfd3cdb0d3c97237965f8c60d1696a64923deeeb2d0b9810286cbe25911", size = 14754603 },
+ { url = "https://files.pythonhosted.org/packages/dc/78/be204fb409b59876ef4658710a022794f16f779a3e9e7df654acc38b2104/fiona-1.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:537872cbc9bda7fcdf73851c91bc5338fca2b502c4c17049ccecaa13cde1f18f", size = 17223639 },
+ { url = "https://files.pythonhosted.org/packages/7e/0d/914fd3c4c32043c2c512fa5021e83b2348e1b7a79365d75a0a37cb545362/fiona-1.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:41cde2c52c614457e9094ea44b0d30483540789e62fe0fa758c2a2963e980817", size = 24464921 },
+ { url = "https://files.pythonhosted.org/packages/c5/e0/665ce969cab6339c19527318534236e5e4184ee03b38cd474497ebd22f4d/fiona-1.10.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:a00b05935c9900678b2ca660026b39efc4e4b916983915d595964eb381763ae7", size = 16106571 },
+ { url = "https://files.pythonhosted.org/packages/23/c8/150094fbc4220d22217f480cc67b6ee4c2f4324b4b58cd25527cd5905937/fiona-1.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f78b781d5bcbbeeddf1d52712f33458775dbb9fd1b2a39882c83618348dd730f", size = 14738178 },
+ { url = "https://files.pythonhosted.org/packages/20/83/63da54032c0c03d4921b854111e33d3a1dadec5d2b7e741fba6c8c6486a6/fiona-1.10.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29ceeb38e3cd30d91d68858d0817a1bb0c4f96340d334db4b16a99edb0902d35", size = 17221414 },
+ { url = "https://files.pythonhosted.org/packages/60/14/5ef47002ef19bd5cfbc7a74b21c30ef83f22beb80609314ce0328989ceda/fiona-1.10.1-cp313-cp313-win_amd64.whl", hash = "sha256:15751c90e29cee1e01fcfedf42ab85987e32f0b593cf98d88ed52199ef5ca623", size = 24461486 },
+]
+
[[package]]
name = "flower"
version = "2.0.1"
@@ -1027,6 +1071,7 @@ dependencies = [
{ name = "earthengine-api" },
{ name = "fastapi" },
{ name = "fastapi-keycloak-middleware" },
+ { name = "fiona" },
{ name = "flower" },
{ name = "geemap" },
{ name = "geopandas" },
@@ -1053,6 +1098,7 @@ requires-dist = [
{ name = "earthengine-api", specifier = ">=1.1.2" },
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "fastapi-keycloak-middleware", specifier = ">=1.1.0" },
+ { name = "fiona", specifier = ">=1.10.1" },
{ name = "flower", specifier = ">=2.0.1" },
{ name = "geemap", specifier = ">=0.34.5" },
{ name = "geopandas", specifier = ">=1.0.1" },