Import - TOPO: formatage des données MAJIC pour commune et voie

cadastre/cadastre_common_base.py

@@ -262,7 +262,7 @@ def postgisToSpatialite(sql: str, targetSrid: str = '2154') -> str:
         {'in': r'alter table [^;]+drop constraint[^;]+;', 'out': ''},
         # ~ {'in': r'^analyse [^;]+;', 'out': ''},
         # replace
-        {'in': r'truncate (bati|fanr|lloc|nbat|pdll|prop)',
+        {'in': r'truncate (bati|topo|lloc|nbat|pdll|prop)',
          'out': r'drop table \1;create table \1 (tmp text)'},
         {'in': r'truncate ', 'out': 'delete from '},
         {'in': r'distinct on *\([a-z, ]+\)', 'out': 'distinct'},

cadastre/cadastre_import.py

@@ -43,12 +43,12 @@
 from cadastre.definitions import (
     IMPORT_MEMORY_ERROR_MESSAGE,
     REGEX_BATI,
-    REGEX_FANTOIR,
     REGEX_LOTLOCAL,
     REGEX_NBATI,
     REGEX_PDL,
     REGEX_PROP,
-    URL_FANTOIR,
+    REGEX_TOPO,
+    URL_TOPO,
 )
 from cadastre.dialogs.dialog_common import CadastreCommon
 
@@ -118,9 +118,9 @@ def __init__(self, dialog):
                 'required': True
             },
             {
-                'key': '[FICHIER_FANTOIR]',
-                'regex': s.value("cadastre/regexFantoir", REGEX_FANTOIR, type=str),
-                'table': 'fanr',
+                'key': '[FICHIER_TOPO]',
+                'regex': s.value("cadastre/regexTopo", REGEX_TOPO, type=str),
+                'table': 'topo',
                 'required': True
             },
             {
@@ -165,7 +165,7 @@ def __init__(self, dialog):
         if self.dialog.hasStructure:
             self.hasConstraints = True
 
-        # Remove MAJIC from tables bati|fanr|lloc|nbat|pdll|prop
+        # Remove MAJIC from tables bati|topo|lloc|nbat|pdll|prop
         self.removeMajicRawData = True
 
         self.beginImport()
@@ -305,7 +305,7 @@ def importMajic(self):
 
         # dict for parameters replacement
         replaceDict = self.replaceDict.copy()
-        # mandatoryFilesKeys = ['[FICHIER_BATI]', '[FICHIER_FANTOIR]', '[FICHIER_NBATI]', '[FICHIER_PROP]']
+        # mandatoryFilesKeys = ['[FICHIER_BATI]', '[FICHIER_TOPO]', '[FICHIER_NBATI]', '[FICHIER_PROP]']
         # missingMajicFiles = False
 
         scriptList = []
@@ -350,6 +350,7 @@ def importMajic(self):
         scriptList.append(importScript)
 
         # Format data
+        replaceDict['DEPDIR'] = f'{self.dialog.edigeoDepartement}{self.dialog.edigeoDirection}'
         scriptList.append(
             {
                 'title': 'Mise en forme des données',
@@ -461,8 +462,8 @@ def get_available_majic_files(self) -> tuple[dict, dict]:
                         maj_list.append(file_path)
 
                         # Store dep_dir for this file
-                        # avoid fantoir, as now it is given for the whole country
-                        if table == 'fanr':
+                        # avoid topo, as now it is given for the whole country
+                        if table == 'topo':
                             continue
 
                         # Get dep_dir : first line with content
@@ -490,17 +491,21 @@ def check_missing_majic_files(self, majic_files_found: dict) -> bool:
                 "<b>Des fichiers MAJIC importants sont manquants</b> :<br/>"
                 " <b>{}</b> <br/><br/>"
                 "Vérifier le chemin des fichiers MAJIC :<br/>"
-                "<b>{}</b> <br/>"
+                "<b>{}</b> <br/><br/>"
                 "ainsi que les mots recherchés pour chaque type de fichier configurés dans les options du plugin Cadastre :<br/>"
-                "<b>{}</b><br/><br/>"
-                "<b>NB:</b> Vous pouvez télécharger les fichiers FANTOIR à cette adresse :<br/>"
+                "<b>{}</b><br/><br/><br/>"
+                "<b>NB:</b> Vous pouvez télécharger les fichiers TOPO à cette adresse :<br/>"
                 "<a href='{}'>{}</a><br/>"
             ).format(
-                ', '.join(missing_files),
+                ', <br/>'.join(missing_files),
                 self.dialog.majicSourceDir,
-                ', <br/>'.join([f"* {a['key'].strip('[]')}: {a['regex'].upper()}" for a in self.majicSourceFileNames]),
-                URL_FANTOIR,
-                URL_FANTOIR,
+                ', <br/>'.join([
+                    f"* {a['key'].strip('[]')}: {a['regex'].upper()}"
+                    for a in self.majicSourceFileNames
+                    if a['table'] in missing_files
+                ]),
+                URL_TOPO,
+                URL_TOPO,
             )
             missing_majic_ignore = QMessageBox.question(
                 self.dialog,
@@ -1187,6 +1192,12 @@ def replaceParametersInScript(self, scriptPath, replaceDict):
                 self.qc.updateLog(msg)
                 return msg
 
+            except KeyError as e:
+                msg = "<b>Erreur lors du paramétrage des scripts d'import: %s</b>" % e
+                self.go = False
+                self.qc.updateLog(msg)
+                return msg
+
             finally:
                 QApplication.restoreOverrideCursor()
 

cadastre/definitions.py

@@ -1,15 +1,15 @@
-URL_FANTOIR = (
-    "https://drive.opendata.craig.fr/s/opendata?path=%2Fadresse%2Ffantoir"
+URL_TOPO = (
+    "https://drive.opendata.craig.fr/s/opendata?path=%2Fadresse%2Ftopo"
 )
 
 URL_DOCUMENTATION = "https://docs.3liz.org/QgisCadastrePlugin/"
 
 REGEX_BATI = "BATI"
-REGEX_FANTOIR = "FANTOIR|FANR"
 REGEX_LOTLOCAL = "LLOC|D166"
 REGEX_NBATI = "NBAT"
 REGEX_PDL = "PDL"
 REGEX_PROP = "PROP"
+REGEX_TOPO = "TOPO"
 
 IMPORT_MEMORY_ERROR_MESSAGE = "<b>ERREUR : Mémoire</b></br>"
 "Veuillez recommencer l'import en baissant la valeur du "

cadastre/dialogs/options_dialog.py

@@ -14,11 +14,11 @@
 
 from cadastre.definitions import (
     REGEX_BATI,
-    REGEX_FANTOIR,
     REGEX_LOTLOCAL,
     REGEX_NBATI,
     REGEX_PDL,
     REGEX_PROP,
+    REGEX_TOPO,
 )
 from cadastre.tools import set_window_title
 
@@ -120,9 +120,9 @@ def getValuesFromSettings(self):
         regexBati = s.value("cadastre/regexBati", REGEX_BATI, type=str)
         if regexBati:
             self.inMajicBati.setText(regexBati)
-        regexFantoir = s.value("cadastre/regexFantoir", REGEX_FANTOIR, type=str)
-        if regexFantoir:
-            self.inMajicFantoir.setText(regexFantoir)
+        regexTopo = s.value("cadastre/regexTopo", REGEX_TOPO, type=str)
+        if regexTopo:
+            self.inMajicTopo.setText(regexTopo)
         regexLotLocal = s.value("cadastre/regexLotLocal", REGEX_LOTLOCAL, type=str)
         if regexLotLocal:
             self.inMajicLotlocal.setText(regexLotLocal)
@@ -192,7 +192,7 @@ def onAccept(self):
         # Save Majic file names
         s = QgsSettings()
         s.setValue("cadastre/regexBati", self.inMajicBati.text().strip(' \t\n\r'))
-        s.setValue("cadastre/regexFantoir", self.inMajicFantoir.text().strip(' \t\n\r'))
+        s.setValue("cadastre/regexTopo", self.inMajicTopo.text().strip(' \t\n\r'))
         s.setValue("cadastre/regexLotLocal", self.inMajicLotlocal.text().strip(' \t\n\r'))
         s.setValue("cadastre/regexNbati", self.inMajicNbati.text().strip(' \t\n\r'))
         s.setValue("cadastre/regexPdl", self.inMajicPdl.text().strip(' \t\n\r'))

cadastre/dialogs/search_dialog.py

@@ -409,7 +409,7 @@ def checkMajicContent(self):
 
         if not self.hasMajicDataParcelle or not self.hasMajicDataVoie:
             self.qc.updateLog(
-                "<b>Pas de données MAJIC non bâties et/ou fantoir</b> -> désactivation de la recherche d'adresse")
+                "<b>Pas de données MAJIC non bâties et/ou TOPO</b> -> désactivation de la recherche d'adresse")
         if not self.hasMajicDataProp:
             self.qc.updateLog(
                 "<b>Pas de données MAJIC propriétaires</b> -> désactivation de la recherche de propriétaires")

cadastre/forms/cadastre_option_form.ui

@@ -169,16 +169,16 @@
              </widget>
             </item>
             <item row="7" column="2">
-             <widget class="QLineEdit" name="inMajicFantoir">
+             <widget class="QLineEdit" name="inMajicTopo">
               <property name="text">
-               <string>FANTOIR|FANR</string>
+               <string>TOPO</string>
               </property>
              </widget>
             </item>
             <item row="7" column="0">
              <widget class="QLabel" name="label_2">
               <property name="text">
-               <string>FANTOIR</string>
+               <string>TOPO</string>
               </property>
              </widget>
             </item>
@@ -310,7 +310,7 @@
   <tabstop>inMajicPdl</tabstop>
   <tabstop>inMajicProp</tabstop>
   <tabstop>lineEdit</tabstop>
-  <tabstop>inMajicFantoir</tabstop>
+  <tabstop>inMajicTopo</tabstop>
   <tabstop>inComposerTemplateFile</tabstop>
   <tabstop>btComposerTemplateFile</tabstop>
   <tabstop>inTempDir</tabstop>

cadastre/scripts/plugin/2024/majic3_formatage_donnees.sql

@@ -966,34 +966,43 @@ FROM ${PREFIXE}lloc;
 -- Traitement: commune
 INSERT INTO ${PREFIXE}commune
 (
- commune, geo_commune, annee, ccodep, ccodir, ccocom, clerivili, libcom, typcom, ruract, carvoi, indpop, poprel, poppart, popfict, annul, dteannul, dtecreart, codvoi,
+ commune, geo_commune, annee, ccodep, ccodir, ccocom, clerivili, libcom, typcom,
+ ruract, carvoi, indpop, poprel, poppart, popfict, annul, dteannul, dtecreart, codvoi,
  typvoi, indldnbat, motclas, lot
 )
 SELECT
-  REPLACE(SUBSTRING(tmp,1,6),' ', '0') AS commune,
-  REPLACE(SUBSTRING(tmp,1,6),' ', '0') AS geo_commune,
-  '${ANNEE}',
-  SUBSTRING(tmp,1,2) AS ccodep,
-  SUBSTRING(tmp,3,1) AS ccodir,
-  SUBSTRING(tmp,4,3) AS ccocom,
-  SUBSTRING(tmp,11,1) AS clerivili,
-  SUBSTRING(tmp,12,30) AS libcom,
-  CASE WHEN trim(SUBSTRING(tmp,43,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,43,1)) END AS typcom,
-  SUBSTRING(tmp,46,1) AS ruract,
-  SUBSTRING(tmp,49,1) AS carvoi,
-  SUBSTRING(tmp,50,1) AS indpop,
-  CASE WHEN trim(SUBSTRING(tmp,53,7))='' THEN NULL ELSE to_number(trim(SUBSTRING(tmp,53,7)),'0000000') END AS poprel,
-  to_number(SUBSTRING(tmp,60,7),'9999999') AS poppart,
-  to_number(SUBSTRING(tmp,67,7),'0000000') AS popfict,
-  SUBSTRING(tmp,74,1) AS annul,
-  SUBSTRING(tmp,75,7) AS dteannul,
-  SUBSTRING(tmp,82,7) AS dtecreart,
-  SUBSTRING(tmp,104,5) AS codvoi,
-  SUBSTRING(tmp,109,1) AS typvoi,
-  SUBSTRING(tmp,110,1) AS indldnbat,
-  SUBSTRING(tmp,113,8) AS motclas,
+  '${DEPDIR}' || SUBSTRING("code topo", 10, 3) AS commune,
+  '${DEPDIR}' || SUBSTRING("code topo", 10, 3) AS geo_commune,
+  '${ANNEE}' AS annee,
+  SUBSTRING("code topo", 8, 2) AS ccodep,
+  SUBSTRING('${DEPDIR}', 3, 1) AS ccodir,
+  SUBSTRING("code topo", 10, 3) AS ccocom,
+  NULL AS clerivili,
+  trim("libelle") AS libcom,
+  nullif("type commune actuel (r ou n)", '') AS typcom,
+  nullif("rur actuel", '') AS ruract,
+  NULL AS carvoi,
+  NULL AS indpop,
+  NULL AS poprel,
+  NULL AS poppart,
+  NULL AS popfict,
+  nullif("annulation", '') AS annul,
+  CASE
+    WHEN "date annulation" != '00000000'
+      THEN substr("date annulation", 1, 4) || lpad(extract('doy' FROM to_date("date annulation", 'YYYYMMDD'))::text, 3, '0')
+    ELSE '0000000'
+  END AS dteannul,
+  CASE
+    WHEN "date creation de article" != '00000000'
+      THEN substr("date creation de article", 1, 4) || lpad(extract('doy' FROM to_date("date creation de article", 'YYYYMMDD'))::text, 3, '0')
+    ELSE '0000000'
+  END AS dtecreart,
+  NULL AS codvoi,
+  NULL AS typvoi,
+  NULL AS indldnbat,
+  NULL AS motclas,
   '${LOT}' as lot
-FROM ${PREFIXE}fanr WHERE SUBSTRING(tmp,4,3)  != ' ' AND trim(SUBSTRING(tmp,7,4))='';
+FROM ${PREFIXE}topo WHERE substr("code topo", 17, 2) = '13';
 
 -- Traitement: voie
 INSERT INTO ${PREFIXE}voie
@@ -1003,33 +1012,75 @@ INSERT INTO ${PREFIXE}voie
  commune, lot
 )
 SELECT
-  REPLACE(SUBSTRING(tmp,1,6)||SUBSTRING(tmp,104,5)||SUBSTRING(tmp,7,4),' ', '0') AS voie,
+  REPLACE('${DEPDIR}' || SUBSTRING("code topo", 10, 3) || '00000' || SUBSTRING("code topo", 13, 4) , ' ', '0')  AS voie,
   '${ANNEE}',
-  SUBSTRING(tmp,1,2) AS ccodep,
-  SUBSTRING(tmp,3,1) AS ccodir,
-  SUBSTRING(tmp,4,3) AS ccocom,
-  CASE WHEN trim(SUBSTRING(tmp,7,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,7,1)) END AS natvoiriv,
-  SUBSTRING(tmp,7,4) AS ccoriv,
-  SUBSTRING(tmp,11,1) AS clerivili,
-  TRIM(SUBSTRING(tmp,12,4)) AS natvoi,
-  SUBSTRING(tmp,16,26) AS libvoi,
-  CASE WHEN trim(SUBSTRING(tmp,43,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,43,1)) END AS typcom,
-  SUBSTRING(tmp,46,1) AS ruract,
-  CASE WHEN trim(SUBSTRING(tmp,49,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,49,1)) END AS carvoi,
-  SUBSTRING(tmp,50,1) AS indpop,
-  SUBSTRING(tmp,53,7) AS poprel,
-  to_number(SUBSTRING(tmp,60,7),'0000000') AS poppart,
-  to_number(SUBSTRING(tmp,67,7),'0000000') AS popfict,
-  CASE WHEN trim(SUBSTRING(tmp,74,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,74,1)) END AS annul,
-  SUBSTRING(tmp,75,7) AS dteannul,
-  SUBSTRING(tmp,82,7) AS dtecreart,
-  SUBSTRING(tmp,104,5) AS codvoi,
-  CASE WHEN trim(SUBSTRING(tmp,109,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,109,1)) END AS typvoi,
-  CASE WHEN trim(SUBSTRING(tmp,110,1))='' THEN NULL ELSE trim(SUBSTRING(tmp,110,1)) END AS indldnbat,
-  SUBSTRING(tmp,113,8) AS motclas,
-  REPLACE(SUBSTRING(tmp,1,6),' ', '0') AS commune,
+  SUBSTRING("code topo", 8, 2) AS ccodep,
+  SUBSTRING('${DEPDIR}', 3, 1) AS ccodir,
+  SUBSTRING("code topo", 10, 3) AS ccocom,
+  SUBSTRING("code topo", 13, 1) AS natvoiriv,
+  SUBSTRING("code topo", 13, 4) AS ccoriv,
+  NULL AS clerivili,
+  replace(libelle, (regexp_replace(libelle, ab.reg, '')), '') AS natvoi,
+  trim(regexp_replace(libelle, ab.reg, '')) AS libvoi,
+  nullif("type commune actuel (r ou n)", '') AS typcom,
+  nullif("rur actuel", '') AS ruract,
+  nullif("caractere voie", '') AS carvoi,
+  NULL AS indpop,
+  NULL AS poprel,
+  NULL AS poppart,
+  NULL AS popfict,
+  nullif("annulation", '') AS annul,
+  CASE
+    WHEN "date annulation" != '00000000'
+      THEN substr("date annulation", 1, 4) || lpad(extract('doy' FROM to_date("date annulation", 'YYYYMMDD'))::text, 3, '0')
+    ELSE '0000000'
+  END AS dteannul,
+  CASE
+    WHEN "date creation de article" != '00000000'
+      THEN substr("date creation de article", 1, 4) || lpad(extract('doy' FROM to_date("date creation de article", 'YYYYMMDD'))::text, 3, '0')
+    ELSE '0000000'
+  END AS dtecreart,
+  NULL AS codvoi,
+  "type voie" AS typvoi,
+  1 AS indldnbat,
+  trim("mot classant") AS motclas,
+  '${DEPDIR}' || SUBSTRING("code topo", 10, 3) AS commune,
   '${LOT}' as lot
-FROM ${PREFIXE}fanr WHERE trim(SUBSTRING(tmp,4,3))  != '' AND trim(SUBSTRING(tmp,7,4))  != '';
+FROM ${PREFIXE}topo,
+LATERAL (
+    SELECT concat(
+        '^(',
+        'ACH |AER |AERG|AGL |AIRE|ALL |ANGL|ARC |ART |AUT |AV  |',
+        'BASE|BD  |BER |BORD|BRE |BRG |BRTL|BSN |',
+        'CAE |CALL|CAMI|CAMP|CAN |CAR |CARE|CASR|CC  |CD  |',
+        'CF  |CHA |CHE |CHEM|CHL |CHP |CHT |CHV |CITE|CIVE|',
+        'CLOS|CLR |COIN|COL |COR |CORO|COTE|COUR|CPG |CR  |',
+        'CRS |CRX |CTR |CTRE|',
+        'DARS|DEVI|DIG |DOM |DRA |DSC |',
+        'ECA |ECL |EMBR|EMP |ENC |ENV |ESC |ESP |ESPA|ETNG|',
+        'FD  |FG  |FON |FOR |FORT|FOS |FRM |',
+        'GAL |GARE|GBD |GPL |GR  |GREV|',
+        'HAB |HAM |HIP |HLE |HLG |HLM |HTR |',
+        'ILE |ILOT|IMP |',
+        'JARD|JTE |',
+        'LAC |LEVE|LICE|LIGN|LOT |',
+        'MAIL|MAIS|MAR |MARE|MAS |MNE |MRN |MTE |',
+        'NTE |',
+        'PAE |PARC|PAS |PASS|PCH |PCHE|PHAR|PIST|PKG |PL  |',
+        'PLA |PLAG|PLAN|PLCI|PLE |PLN |PLT |PNT |PONT|PORQ|',
+        'PORT|POST|POT |PROM|PRT |PRV |PTA |PTE |PTR |PTTE|',
+        'QUA |QUAI|',
+        'RAC |REM |RES |RIVE|RLE |ROC |RPE |RPT |RTD |RTE |',
+        'RUE |RUET|RUIS|RULT|RVE |',
+        'SAS |SEN |SQ  |STDE|',
+        'TER |TOUR|TPL |TRA |TRAB|TRN |TRT |TSSE|TUN |',
+        'VAL |VALL|VC  |VCHE|VEN |VGE |VIA |VIAD|VIL |VLA |',
+        'VOIE|VOIR|VOUT|VOY |VTE |',
+        'ZA  |ZAC |ZAD |ZI  |ZONE|ZUP |',
+        'N   |D   |V   ', ')'
+    ) AS reg
+) AS ab
+WHERE substr("code topo", 17, 2) = '14'
 
 -- purge des doublons : voie
 CREATE INDEX idxan_voie ON voie (annee);