Final split between legacy compose and BAN-ID data processing

lib/api/consumers/export-to-exploitation-db-consumer.js

@@ -36,16 +36,9 @@ const ADDRESS_BBOX_BUFFER = 50
 
 // Collections names
 const EXPLOITATION_DB_COLLECTION_NAMES = {
-  legacy: {
-    district: 'communes_temp',
-    commonToponym: 'voies_temp',
-    address: 'numeros_temp'
-  },
-  banID: {
-    district: 'district',
-    commonToponym: 'common_toponym',
-    address: 'address'
-  }
+  district: 'communes',
+  commonToponym: 'voies',
+  address: 'numeros'
 }
 
 // QUERIES & POSTGIS FUNCTIONS
@@ -132,7 +125,6 @@ export default async function exportToExploitationDB({data}) {
     // Clean collections
     // Delete all data related to the district (legacy and banID)
     await deleteAllLegacyDataRelatedToCOG(cog)
-    await deleteAllDataRelatedToDistrict(districtID)
 
     // CommonToponym
     // Count the total number of common toponyms and pages to process
@@ -160,12 +152,10 @@ export default async function exportToExploitationDB({data}) {
       })
       // Format the data and calculate the fantoir code, tiles and postal code
       const pageDataWithExtraDataCalculation = pageData.map(commonToponym => calculateExtraDataForCommonToponym(commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap))
-      const formatedPageData = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponym(commonToponym))
       const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(commonToponym => formatCommonToponymDataForLegacy(commonToponym, district, pseudoCodeVoieGenerator, commonToponymIDlegacyCommonToponymIDMap))
 
       // Insert the data in the collection (legacy and banID)
-      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false})
-      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.commonToponym).insertMany(formatedPageData, {ordered: false})
+      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.commonToponym).insertMany(formatedPageDataForLegacy, {ordered: false})
     }
 
     const commonToponymsExportPromises = []
@@ -200,12 +190,10 @@ export default async function exportToExploitationDB({data}) {
 
       // Format the data and calculate the fantoir code, tiles and postal code
       const pageDataWithExtraDataCalculation = pageData.map(address => calculateExtraDataForAddress(address, cog, commonToponymIDFantoirCodeMap))
-      const formatedPageData = pageDataWithExtraDataCalculation.map(address => formatAddress(address))
       const formatedPageDataForLegacy = pageDataWithExtraDataCalculation.map(address => formatAddressDataForLegacy(address, district, commonToponymIDlegacyCommonToponymIDMap))
 
       // Insert the data in the collection (legacy and banID)
-      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.address).insertMany(formatedPageDataForLegacy, {ordered: false})
-      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.address).insertMany(formatedPageData, {ordered: false})
+      await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.address).insertMany(formatedPageDataForLegacy, {ordered: false})
     }
 
     const addressesExportPromises = []
@@ -218,10 +206,7 @@ export default async function exportToExploitationDB({data}) {
     // District
     // For Legacy collections
     const districtFormatedForLegacy = await formatDistrictDataForLegacy(district, totalCommonToponymRecords, totalAddressRecords, transaction)
-    await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.district).insertOne(districtFormatedForLegacy)
-
-    // For BanID collections
-    await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.district).insertOne(district)
+    await mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.district).insertOne(districtFormatedForLegacy)
 
     // Pseudo code voie generator saving data
     await pseudoCodeVoieGenerator.save()
@@ -238,37 +223,15 @@ export default async function exportToExploitationDB({data}) {
 // Helpers
 
 // Helpers for exploitation DB
-const deleteAllDataRelatedToDistrict = async districtID => {
-  await Promise.all([
-    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.district).deleteOne({districtID}),
-    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.commonToponym).deleteMany({districtID}),
-    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.banID.address).deleteMany({districtID})
-  ])
-}
 
 const deleteAllLegacyDataRelatedToCOG = async cog => {
   await Promise.all([
-    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.district).deleteOne({codeCommune: cog}),
-    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.commonToponym).deleteMany({codeCommune: cog}),
-    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.legacy.address).deleteMany({codeCommune: cog}),
+    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.district).deleteMany({codeCommune: cog}),
+    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.commonToponym).deleteMany({codeCommune: cog}),
+    mongo.db.collection(EXPLOITATION_DB_COLLECTION_NAMES.address).deleteMany({codeCommune: cog}),
   ])
 }
 
-// Helpers for formatting data
-export const formatCommonToponym = commonToponym => {
-  // To-do : define the format for the common toponym
-  // For now, we remove data calculation used for the legacy format (centroid, addressCount, certifiedAddressCount, bbox)
-  const {centroid, addressCount, certifiedAddressCount, addressBbox, bbox, ...rest} = commonToponym
-  return rest
-}
-
-const formatAddress = address => {
-  // To-do : define the format for the address
-  // For now, we remove data calculation used for the legacy format (bbox)
-  const {bbox, ...rest} = address
-  return rest
-}
-
 // Helpers for calculation
 export const calculateExtraDataForCommonToponym = (commonToponym, cog, fantoirFinder, commonToponymIDFantoirCodeMap) => {
   // Calculate the fantoir code for each common toponym

lib/api/consumers/format-to-legacy-helpers.js

@@ -83,7 +83,8 @@ export const formatDistrictDataForLegacy = async (district, totalCommonToponymRe
     ...(addressAnalysis ? {analyseAdressage: addressAnalysis} : {}),
     idRevision: meta?.bal?.idRevision,
     dateRevision: meta?.bal?.dateRevision,
-    composedAt: new Date()
+    composedAt: new Date(),
+    withBanId: true
   }
 }
 

lib/api/legacy-routes.cjs

@@ -246,31 +246,27 @@ app.get(
 app.post(
   '/ban/communes/:codeCommune/compose',
   ensureIsAdmin,
-  analyticsMiddleware.composeCommune,
   w(async (req, res) => {
     const {codeCommune} = req.commune
     const {force} = req.body
 
     try {
       // Async Call to ID-Fix
-      if (API_IDFIX_URL) {
-        fetch(`${API_IDFIX_URL}/compute-from-cog/${codeCommune}`, {
-          method: 'GET',
-          headers: {
-            'content-Type': 'application/json',
-            Authorization: `Token ${API_IDFIX_TOKEN}`,
-          },
+      fetch(`${API_IDFIX_URL}/compute-from-cog/${codeCommune}?force=${force}`, {
+        method: 'GET',
+        headers: {
+          'content-Type': 'application/json',
+          Authorization: `Token ${API_IDFIX_TOKEN}`,
+        },
+      })
+        .then(response => response.json())
+        .then(idFixResponse => {
+          console.log(idFixResponse)
+        })
+        .catch(error => {
+          console.log(`[${currentDate()}][ERROR]`, error)
         })
-          .then(response => response.json())
-          .then(idFixResponse => {
-            console.log(idFixResponse)
-          })
-          .catch(error => {
-            console.log(`[${currentDate()}][ERROR]`, error)
-          })
-      }
 
-      await Commune.askComposition(codeCommune, {force})
       const commune = await getCommune(codeCommune)
       return res.send(commune)
     } catch (error) {
@@ -290,6 +286,35 @@ app.post(
   })
 )
 
+app.post(
+  '/api/legacy-compose/:codeCommune/',
+  ensureIsAdmin,
+  analyticsMiddleware.composeCommune,
+  w(async (req, res) => {
+    const {codeCommune} = req.commune
+    const {force, ignoreIdConfig} = req.body
+
+    try {
+      // Async Call to ID-Fix
+      await Commune.askComposition(codeCommune, {force}, ignoreIdConfig)
+      return res.send(`Legacy compose asked for cog: ${codeCommune}`)
+    } catch (error) {
+      const errMsg = `---
+      ${new Date().toUTCString()}
+      codeCommune : ${codeCommune}
+      erreur : ${error}
+      ---
+      `.replace(/^(\s)*/gm, '')
+      fs.appendFile('errorCompose.log', errMsg, error => {
+        if (error) {
+          throw error
+        }
+      })
+      return res.sendStatus(500)
+    }
+  })
+)
+
 app.get(
   '/api/communes-summary',
   analyticsMiddleware.communesSummary,

lib/compose/bal-garbage-collector/__mocks__/api-id-fix-mock.js

@@ -0,0 +1,3 @@
+export const idFixComputeFromCogs = async codeCommune => (codeCommune)
+
+export default idFixComputeFromCogs

lib/compose/bal-garbage-collector/index.js

@@ -1,20 +1,28 @@
-import {askComposition} from '../../models/commune.cjs'
+import {idFixComputeFromCogs} from '../../util/api-id-fix.cjs'
 import {getCurrentRevisons} from '../../util/api-depot.cjs'
 import {getCommuneRevisionID} from './models.js'
 
 export const balGarbageCollector = async () => {
-  const [revisionBAL, revisionBAN] = await Promise.all([
-    getCurrentRevisons(),
-    getCommuneRevisionID()
-  ])
+  try {
+    console.info('BAL garbage collector start')
+    const [revisionBAL, revisionBAN] = await Promise.all([
+      getCurrentRevisons(),
+      getCommuneRevisionID()
+    ])
 
-  const idRevisionBAL = revisionBAL.map(({_id, codeCommune}) => [codeCommune, _id])
-  const idRevisionBAN = Object.fromEntries(revisionBAN.map(({idRevision, codeCommune}) => [codeCommune, idRevision]))
+    const idRevisionBAL = revisionBAL.map(({_id, codeCommune}) => [codeCommune, _id])
+    const idRevisionBAN = Object.fromEntries(revisionBAN.map(({idRevision, codeCommune}) => [codeCommune, idRevision]))
 
-  const codesCommunes = idRevisionBAL.filter(([codeCommune, idRevision]) => idRevisionBAN[codeCommune] !== idRevision)
-
-  const communeComposed = await Promise.all(codesCommunes.map(([codeCommune]) => askComposition(codeCommune)))
-  return communeComposed
+    const codesCommunes = idRevisionBAL.filter(([codeCommune, idRevision]) => idRevisionBAN[codeCommune] !== idRevision)
+      .map(([codeCommune]) => codeCommune)
+    console.info(`Re-syncronizing district cogs : ${codesCommunes.join(',')}`)
+    // Seding the list of cogs to the id-fix API not waiting for the response
+    idFixComputeFromCogs(codesCommunes)
+    console.info('BAL garbage collector end')
+    return codesCommunes
+  } catch (error) {
+    console.error('BAL garbage collector error', error)
+  }
 }
 
 export default balGarbageCollector

lib/compose/bal-garbage-collector/index.spec.js

@@ -3,7 +3,7 @@ import {bddCommuneMock} from './__mocks__/revision-data-mock.js'
 
 jest.unstable_mockModule('../../util/api-depot.cjs', async () => import('./__mocks__/api-depot-mock.js'))
 jest.unstable_mockModule('./models.js', async () => import('./__mocks__/models-mock.js'))
-jest.unstable_mockModule('../../models/commune.cjs', async () => import('./__mocks__/ask-composition-mock.js'))
+jest.unstable_mockModule('../../util/api-id-fix.cjs', async () => import('./__mocks__/api-id-fix-mock.js'))
 
 const {balGarbageCollector} = await import('./index.js')
 

lib/compose/cli.cjs

@@ -1,17 +1,21 @@
 #!/usr/bin/env node
 require('dotenv').config()
+const argv = require('minimist')(process.argv.slice(2))
 const mongo = require('../util/mongo.cjs')
 const {getAskedComposition} = require('../models/commune.cjs')
 const {runInParallel} = require('../util/parallel.cjs')
 
 async function main() {
   await mongo.connect()
-  const [commune] = process.argv.slice(2)
+  const commune = argv._[0] ? String(argv._[0]) : undefined // Convert to string if not undefined
   const communes = commune ? [commune] : await getAskedComposition()
 
+  // Check if --ignoreIdConfig flag is provided
+  const ignoreIdConfig = argv.ignoreIdConfig || false
+
   await runInParallel(
     require.resolve('./worker.cjs'),
-    communes.map(codeCommune => ({codeCommune})),
+    communes.map(codeCommune => ({codeCommune, ignoreIdConfig})),
     {maxWorkerMemory: 3072, maxRetries: 5}
   )
 

lib/compose/index.cjs

@@ -49,9 +49,15 @@ async function getBalData(codeCommune, revision) {
   return prepareBalData(adresses, {codeCommune})
 }
 
-async function composeCommune(codeCommune) {
+async function composeCommune(codeCommune, ignoreIdConfig) {
   const communeCOG = getCommuneCOG(codeCommune)
   const commune = await getCommune(codeCommune)
+
+  if (!ignoreIdConfig && commune?.withBanId) {
+    console.info(`La commune ${codeCommune} est gérée avec Ban ID => composition ignorée`)
+    return false
+  }
+
   const compositionOptions = commune?.compositionOptions || {}
 
   if (!communeCOG) {
@@ -62,7 +68,7 @@ async function composeCommune(codeCommune) {
 
   if (!compositionOptions.force && currentRevision && commune?.idRevision === currentRevision?._id) {
     console.log(`${codeCommune} | révision source inchangée => composition ignorée`)
-    return
+    return false
   }
 
   const isBAL = Boolean(currentRevision)
@@ -71,7 +77,7 @@ async function composeCommune(codeCommune) {
   // Ne sera plus nécessaire quand l'API de dépôt contiendra 100% des BAL contenues dans la BAN
   if (!isBAL && commune?.typeComposition === 'bal') {
     console.log(`${codeCommune} | passage de 'bal' à 'assemblage' interdit => composition ignorée`)
-    return
+    return false
   }
 
   const balData = isBAL && await getBalData(codeCommune, currentRevision)
@@ -136,7 +142,8 @@ async function composeCommune(codeCommune) {
     nbLieuxDits: lieuxDits.length,
     typeComposition: isBAL ? 'bal' : 'assemblage',
     idRevision: currentRevision?._id,
-    dateRevision: currentRevision?.publishedAt
+    dateRevision: currentRevision?.publishedAt,
+    withBanId: false
   }
 
   if (codeCommune in locauxAdressesIndex) {
@@ -155,6 +162,7 @@ async function composeCommune(codeCommune) {
   await pseudoCodeVoieGenerator.save()
 
   await saveCommuneData(codeCommune, {commune: communeRecord, voies: voiesToPersist, numeros})
+  return true
 }
 
 module.exports = composeCommune

lib/compose/worker.cjs

@@ -4,11 +4,11 @@ const composeCommune = require('./index.cjs')
 
 async function main(options) {
   await mongo.connect()
-  const {codeCommune} = options
+  const {codeCommune, ignoreIdConfig} = options
 
   console.time(`commune ${codeCommune}`)
 
-  await composeCommune(codeCommune)
+  await composeCommune(codeCommune, ignoreIdConfig)
   await finishComposition(codeCommune)
 
   console.timeEnd(`commune ${codeCommune}`)

lib/jobs/compose-commune.cjs

@@ -2,7 +2,7 @@ const {isEqual, parseISO} = require('date-fns')
 const {getCommune, finishComposition} = require('../models/commune.cjs')
 const composeCommune = require('../compose/index.cjs')
 
-async function handle({data: {codeCommune, compositionAskedAt}}) {
+async function handle({data: {codeCommune, compositionAskedAt, ignoreIdConfig}}) {
   const communeEntry = await getCommune(codeCommune)
 
   if (!communeEntry.compositionAskedAt || !isEqual(communeEntry.compositionAskedAt, parseISO(compositionAskedAt))) {
@@ -11,8 +11,10 @@ async function handle({data: {codeCommune, compositionAskedAt}}) {
 
   console.log(`Composition des adresses de la commune ${codeCommune}`)
 
-  await composeCommune(codeCommune)
+  await composeCommune(codeCommune, ignoreIdConfig)
   await finishComposition(codeCommune)
+
+  console.log(`Composition des adresses de la commune ${codeCommune} terminée`)
 }
 
 module.exports = handle

lib/models/commune.cjs

@@ -4,7 +4,7 @@ const {getCommuneActuelle, getRegion, getDepartement, getCommune: getCommuneCOG}
 const compositionQueue = require('../util/queue.cjs')('compose-commune')
 const {prepareAdresse, prepareToponyme} = require('../formatters/geojson.cjs')
 
-async function askComposition(codeCommune, options = {}) {
+async function askComposition(codeCommune, options = {}, ignoreIdConfig) {
   const communeActuelle = getCommuneActuelle(codeCommune)
 
   if (!communeActuelle) {
@@ -17,7 +17,7 @@ async function askComposition(codeCommune, options = {}) {
     {$set: {compositionAskedAt: now, compositionOptions: options}},
     {upsert: true}
   )
-  await compositionQueue.add({codeCommune: communeActuelle.code, compositionAskedAt: now}, {removeOnComplete: true})
+  await compositionQueue.add({codeCommune: communeActuelle.code, compositionAskedAt: now, ignoreIdConfig}, {removeOnComplete: true})
 }
 
 async function askCompositionAll() {