Import des départements et AOMs 2023 (#3625)

* make import departements script work

* same changes for departements than for communes

* change of link for PECI

* start explaining aom script

* remove useless columns from aoms

* still non working tests

* use more conventional up / down syntax

* tweak runtime for dev

* it did work finally and was a problem with my test database

* replace import aom with new files

* first part of AOM import is working

* continue work

* continue work

* AOM script goes to the end but generates mistakes

* change migration

* working full script (except commune principale)

* deal with commas in surface

* working main commune functino

* remove structure

* clean things

* remove files

* mix format mix credo

* refactor import aom script

* fix tests

* remove useless method in migration

* reverse runtime db timeout for dev

* Move import aoms to mix task

* correct some names

* make main commune script more efficient

* replace created in 2022 by created after 2021

* Add transaction timeout

* mix format

* Format number of AOMs

---------

Co-authored-by: Antoine Augusti <[email protected]>
Co-authored-by: Antoine Augusti <[email protected]>

apps/transport/lib/db/aom.ex

@@ -7,24 +7,23 @@ defmodule DB.AOM do
   """
   use Ecto.Schema
   use TypedEctoSchema
-  alias DB.{Dataset, Region, Repo}
+  alias DB.{Dataset, Departement, Region, Repo}
   alias Geo.MultiPolygon
 
   typed_schema "aom" do
+    # composition_res_id matches the id_reseau attribute from the Cerema dataset it’s the official ID of the AOM
     field(:composition_res_id, :integer)
     field(:insee_commune_principale, :string)
-    field(:departement, :string)
     field(:siren, :string)
     field(:nom, :string)
     field(:forme_juridique, :string)
     field(:nombre_communes, :integer)
-    field(:population_municipale, :integer)
-    field(:population_totale, :integer)
+    field(:population, :integer)
     field(:surface, :string)
-    field(:commentaire, :string)
     field(:geom, Geo.PostGIS.Geometry) :: MultiPolygon.t()
 
     belongs_to(:region, Region)
+    belongs_to(:departement_object, Departement, foreign_key: :departement, references: :insee, type: :string)
     has_many(:datasets, Dataset)
 
     many_to_many(:legal_owners_dataset, Dataset, join_through: "dataset_aom_legal_owner")
@@ -34,5 +33,5 @@ defmodule DB.AOM do
   def get(insee_commune_principale: nil), do: nil
   def get(insee_commune_principale: insee), do: Repo.get_by(AOM, insee_commune_principale: insee)
 
-  def created_in_2022?(%__MODULE__{composition_res_id: composition_res_id}), do: composition_res_id >= 1_000
+  def created_after_2021?(%__MODULE__{composition_res_id: composition_res_id}), do: composition_res_id >= 1_000
 end

apps/transport/lib/db/commune.ex

@@ -16,6 +16,7 @@ defmodule DB.Commune do
     field(:siren, :string)
     field(:arrondissement_insee, :string)
 
+    # In theory a commune has only one AOM, the reference is done through the composition_res_id attribute
     belongs_to(:aom_res, AOM, references: :composition_res_id)
     belongs_to(:region, Region)
     belongs_to(:departement, Departement, foreign_key: :departement_insee, references: :insee, type: :string)