Skip to content

Commit

Permalink
Import - TOPO: formatage des données MAJIC pour commune et voie
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdouchin
mdouchin committed Jan 30, 2025
1 parent 7601616 commit 8a7b512
Show file tree
Hide file tree
Showing 16 changed files with 277 additions and 180 deletions.
9 changes: 7 additions & 2 deletions cadastre/cadastre_common_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,8 +262,9 @@ def postgisToSpatialite(sql: str, targetSrid: str = '2154') -> str:
{'in': r'alter table [^;]+drop constraint[^;]+;', 'out': ''},
# ~ {'in': r'^analyse [^;]+;', 'out': ''},
# replace
{'in': r'truncate (bati|fanr|lloc|nbat|pdll|prop)',
'out': r'drop table \1;create table \1 (tmp text)'},
{'in': r'truncate (bati|lloc|nbat|pdll|prop)',
'out': r'drop table if exists \1;create table \1 (tmp text)'},
{'in': r'truncate topo', 'out': r'delete from topo'},
{'in': r'truncate ', 'out': 'delete from '},
{'in': r'distinct on *\([a-z, ]+\)', 'out': 'distinct'},
{'in': r'serial', 'out': 'INTEGER PRIMARY KEY AUTOINCREMENT'},
Expand All @@ -276,6 +277,10 @@ def postgisToSpatialite(sql: str, targetSrid: str = '2154') -> str:
'out': r"date(substr(\2, 5, 4) || '-' || substr(\2, 3, 2) || '-' || substr(\2, 1, 2))"},
{'in': r"(to_date\()([^']+) *, *'DD/MM/YYYY' *\)",
'out': r"date(substr(\2, 7, 4) || '-' || substr(\2, 4, 2) || '-' || substr(\2, 1, 2))"},

{'in': r"(to_char\()(.+), 'DDD'\)",
'out': r"strftime('%j', \2)"},

{'in': r"(to_date\()([^']+) *, *'YYYYMMDD' *\)",
'out': r"date(substr(\2, 1, 4) || '-' || substr(\2, 5, 2) || '-' || substr(\2, 7, 2))"},
{'in': r"(to_char\()([^']+) *, *'dd/mm/YYYY' *\)",
Expand Down
236 changes: 139 additions & 97 deletions cadastre/cadastre_import.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,12 @@
from cadastre.definitions import (
IMPORT_MEMORY_ERROR_MESSAGE,
REGEX_BATI,
REGEX_FANTOIR,
REGEX_LOTLOCAL,
REGEX_NBATI,
REGEX_PDL,
REGEX_PROP,
URL_FANTOIR,
REGEX_TOPO,
URL_TOPO,
)
from cadastre.dialogs.dialog_common import CadastreCommon

Expand Down Expand Up @@ -118,9 +118,9 @@ def __init__(self, dialog):
'required': True
},
{
'key': '[FICHIER_FANTOIR]',
'regex': s.value("cadastre/regexFantoir", REGEX_FANTOIR, type=str),
'table': 'fanr',
'key': '[FICHIER_TOPO]',
'regex': s.value("cadastre/regexTopo", REGEX_TOPO, type=str),
'table': 'topo',
'required': True
},
{
Expand Down Expand Up @@ -165,7 +165,7 @@ def __init__(self, dialog):
if self.dialog.hasStructure:
self.hasConstraints = True

# Remove MAJIC from tables bati|fanr|lloc|nbat|pdll|prop
# Remove MAJIC from tables bati|topo|lloc|nbat|pdll|prop
self.removeMajicRawData = True

self.beginImport()
Expand Down Expand Up @@ -305,7 +305,7 @@ def importMajic(self):

# dict for parameters replacement
replaceDict = self.replaceDict.copy()
# mandatoryFilesKeys = ['[FICHIER_BATI]', '[FICHIER_FANTOIR]', '[FICHIER_NBATI]', '[FICHIER_PROP]']
# mandatoryFilesKeys = ['[FICHIER_BATI]', '[FICHIER_TOPO]', '[FICHIER_NBATI]', '[FICHIER_PROP]']
# missingMajicFiles = False

scriptList = []
Expand Down Expand Up @@ -350,6 +350,7 @@ def importMajic(self):
scriptList.append(importScript)

# Format data
replaceDict['DEPDIR'] = f'{self.dialog.edigeoDepartement}{self.dialog.edigeoDirection}'
scriptList.append(
{
'title': 'Mise en forme des données',
Expand Down Expand Up @@ -460,9 +461,8 @@ def get_available_majic_files(self) -> tuple[dict, dict]:
file_path = os.path.join(root, file_sub_path)
maj_list.append(file_path)

# Store dep_dir for this file
# avoid fantoir, as now it is given for the whole country
if table == 'fanr':
# avoid topo, since direction is not used in TOPO
if table == 'topo':
continue

# Get dep_dir : first line with content
Expand Down Expand Up @@ -490,17 +490,21 @@ def check_missing_majic_files(self, majic_files_found: dict) -> bool:
"<b>Des fichiers MAJIC importants sont manquants</b> :<br/>"
" <b>{}</b> <br/><br/>"
"Vérifier le chemin des fichiers MAJIC :<br/>"
"<b>{}</b> <br/>"
"<b>{}</b> <br/><br/>"
"ainsi que les mots recherchés pour chaque type de fichier configurés dans les options du plugin Cadastre :<br/>"
"<b>{}</b><br/><br/>"
"<b>NB:</b> Vous pouvez télécharger les fichiers FANTOIR à cette adresse :<br/>"
"<b>{}</b><br/><br/><br/>"
"<b>NB:</b> Vous pouvez télécharger les fichiers TOPO à cette adresse :<br/>"
"<a href='{}'>{}</a><br/>"
).format(
', '.join(missing_files),
', <br/>'.join(missing_files),
self.dialog.majicSourceDir,
', <br/>'.join([f"* {a['key'].strip('[]')}: {a['regex'].upper()}" for a in self.majicSourceFileNames]),
URL_FANTOIR,
URL_FANTOIR,
', <br/>'.join([
f"* {a['key'].strip('[]')}: {a['regex'].upper()}"
for a in self.majicSourceFileNames
if a['table'] in missing_files
]),
URL_TOPO,
URL_TOPO,
)
missing_majic_ignore = QMessageBox.question(
self.dialog,
Expand Down Expand Up @@ -691,9 +695,12 @@ def import_majic_into_database(self) -> bool:
self.totalSteps += len(majic_files_found[table])
processed_files_count += len(majic_files_found[table])
for file_path in majic_files_found[table]:
self.qc.updateLog(f'<b>{table}</b>')
self.qc.updateLog(file_path)

import_file = self.import_majic_file_into_database(table, file_path, dep_dir)
if table == 'topo':
import_file = self.import_file_with_ogr(file_path, 'topo')
else:
import_file = self.import_majic_file_into_database(table, file_path, dep_dir)
if not import_file:
continue

Expand Down Expand Up @@ -1187,6 +1194,12 @@ def replaceParametersInScript(self, scriptPath, replaceDict):
self.qc.updateLog(msg)
return msg

except KeyError as e:
msg = "<b>Erreur lors du paramétrage des scripts d'import: %s</b>" % e
self.go = False
self.qc.updateLog(msg)
return msg

finally:
QApplication.restoreOverrideCursor()

Expand Down Expand Up @@ -1408,7 +1421,7 @@ def importAllEdigeoToDatabase(self):
self.step = 0
self.totalSteps = len(thfList)
for thf in thfList:
self.importEdigeoThfToDatabase(thf)
self.import_file_with_ogr(thf, 'thf')
self.updateProgressBar()
if not self.go:
break
Expand Down Expand Up @@ -1442,110 +1455,139 @@ def importAllEdigeoToDatabase(self):
self.totalSteps = initialTotalSteps
QApplication.restoreOverrideCursor()

def importEdigeoThfToDatabase(self, filename):
def import_file_with_ogr(self, file_path: str, file_type: str):
"""
Import one edigeo THF files into database
Import file into the database.
It can either be an EDIGEO THF file or a TOPO file.
source : db_manager/dlg_import_vector.py
"""
if self.go:
# Get options
if not self.go:
return None

# SRID configurations
if file_type == 'thf':
targetSridOption = '-t_srs'
if self.sourceSridFull == self.targetSridFull:
targetSridOption = '-a_srs'

# Build ogr2ogr command
conn_name = self.dialog.connectionName
settings = QSettings()
settings.beginGroup(f"/{self.db.dbplugin().connectionSettingsKey()}/{conn_name}")
# Build ogr2ogr command
conn_name = self.dialog.connectionName
settings = QSettings()
settings.beginGroup(f"/{self.db.dbplugin().connectionSettingsKey()}/{conn_name}")

# normalising file path
filename = os.path.normpath(filename)
if self.dialog.dbType == 'postgis':
if not settings.contains("database"): # non-existent entry?
raise Exception(self.tr('There is no defined database connection "%s".') % conn_name)
settingsList = ["service", "host", "port", "database", "username", "password"]
service, host, port, database, username, password = (settings.value(x) for x in settingsList)

if service:
pg_access = 'PG:service={} active_schema={}'.format(
service,
self.dialog.schema
)
else:
# qgis can connect to postgis DB without a specified host param connection, but ogr2ogr cannot
if not host:
host = "localhost"

pg_access = 'PG:host={} port={} dbname={} active_schema={} user={} password={}'.format(
host,
port,
database,
self.dialog.schema,
username,
password
)
cmdArgs = [
'',
# normalising file path
file_path = os.path.normpath(file_path)
if self.dialog.dbType == 'postgis':
if not settings.contains("database"): # non-existent entry?
raise Exception(self.tr('There is no defined database connection "%s".') % conn_name)
settingsList = ["service", "host", "port", "database", "username", "password"]
service, host, port, database, username, password = (settings.value(x) for x in settingsList)

if service:
pg_access = 'PG:service={} active_schema={}'.format(
service,
self.dialog.schema
)
else:
# qgis can connect to postgis DB without a specified host param connection, but ogr2ogr cannot
if not host:
host = "localhost"

pg_access = 'PG:host={} port={} dbname={} active_schema={} user={} password={}'.format(
host,
port,
database,
self.dialog.schema,
username,
password
)
cmdArgs = [
'',
]
if file_type == 'thf':
cmdArgs += [
'-s_srs', self.sourceSridFull,
targetSridOption, self.targetSridFull,
'-append',
'-f', 'PostgreSQL',
pg_access,
filename,
]
cmdArgs += [
'-append',
'-f', 'PostgreSQL',
pg_access,
file_path,
'-lco', 'PG_USE_COPY=YES',
'-gt', '50000',
'--config', 'PG_USE_COPY', 'YES',
]
if file_type == 'thf':
cmdArgs += [
'-lco', 'GEOMETRY_NAME=geom',
'-lco', 'PG_USE_COPY=YES',
'-nlt', 'GEOMETRY',
'-gt', '50000',
'--config', 'OGR_EDIGEO_CREATE_LABEL_LAYERS', 'NO',
'--config', 'PG_USE_COPY', 'YES',
]
# -c client_encoding=latin1
if file_type == 'topo':
cmdArgs += [
'-nln', 'topo',
]
# -c client_encoding=latin1

if self.dialog.dbType == 'spatialite':
if not settings.contains("sqlitepath"): # non-existent entry?
self.go = False
raise Exception('there is no defined database connection "%s".' % conn_name)
if self.dialog.dbType == 'spatialite':
if not settings.contains("sqlitepath"): # non-existent entry?
self.go = False
raise Exception('there is no defined database connection "%s".' % conn_name)

database = settings.value("sqlitepath")
database = settings.value("sqlitepath")

cmdArgs = [
'',
cmdArgs = [
'',
]
if file_type == 'thf':
cmdArgs += [
'-s_srs', self.sourceSridFull,
targetSridOption, self.targetSridFull,
'-append',
'-f', 'SQLite',
database,
filename,
]
cmdArgs += [
'-append',
'-f', 'SQLite',
database,
file_path,
'-gt', '50000',
'--config', 'OGR_SQLITE_SYNCHRONOUS', 'OFF',
'--config', 'OGR_SQLITE_CACHE', '512'
]
if file_type == 'thf':
cmdArgs += [
'-lco', 'GEOMETRY_NAME=geom',
'-nlt', 'GEOMETRY',
'-dsco', 'SPATIALITE=YES',
'-gt', '50000',
'--config', 'OGR_EDIGEO_CREATE_LABEL_LAYERS', 'NO',
'--config', 'OGR_SQLITE_SYNCHRONOUS', 'OFF',
'--config', 'OGR_SQLITE_CACHE', '512'
]
if file_type == 'topo':
cmdArgs += [
'-nln', 'topo',
]

# self.qc.updateLog( ' '.join(cmdArgs))
# Run only if ogr2ogr found
if self.go:
# Workaround to get ogr2ogr error messages via stdout
# as ogr2ogr.py does not return exceptions nor error messages
# but only prints the error before returning False
stdout = sys.stdout
try:
sys.stdout = file = io.StringIO()
self.go = ogr2ogr(cmdArgs)
printedString = file.getvalue()
finally:
sys.stdout = stdout
# self.qc.updateLog( ' '.join(cmdArgs))
# Run only if ogr2ogr found
if self.go:
# Workaround to get ogr2ogr error messages via stdout
# as ogr2ogr.py does not return exceptions nor error messages
# but only prints the error before returning False
stdout = sys.stdout
try:
sys.stdout = file = io.StringIO()
self.go = ogr2ogr(cmdArgs)
printedString = file.getvalue()
finally:
sys.stdout = stdout

if not self.go:
self.qc.updateLog(
"<b>Erreur - L'import des données via OGR2OGR a échoué:</b>\n\n{}\n\n{}".format(
printedString,
cmdArgs
)
if not self.go:
self.qc.updateLog(
"<b>Erreur - L'import des données via OGR2OGR a échoué:</b>\n\n{}\n\n{}".format(
printedString,
cmdArgs
)
)

return None

Expand Down
6 changes: 3 additions & 3 deletions cadastre/definitions.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
URL_FANTOIR = (
"https://drive.opendata.craig.fr/s/opendata?path=%2Fadresse%2Ffantoir"
URL_TOPO = (
"https://drive.opendata.craig.fr/s/opendata?path=%2Fadresse%2Ftopo"
)

URL_DOCUMENTATION = "https://docs.3liz.org/QgisCadastrePlugin/"

REGEX_BATI = "BATI"
REGEX_FANTOIR = "FANTOIR|FANR"
REGEX_LOTLOCAL = "LLOC|D166"
REGEX_NBATI = "NBAT"
REGEX_PDL = "PDL"
REGEX_PROP = "PROP"
REGEX_TOPO = "TOPO"

IMPORT_MEMORY_ERROR_MESSAGE = "<b>ERREUR : Mémoire</b></br>"
"Veuillez recommencer l'import en baissant la valeur du "
Expand Down
Loading

0 comments on commit 8a7b512

Please sign in to comment.