diff --git a/apps/transport/lib/db/aom.ex b/apps/transport/lib/db/aom.ex index 7429f4f5c2..f2e2a86d2b 100644 --- a/apps/transport/lib/db/aom.ex +++ b/apps/transport/lib/db/aom.ex @@ -7,24 +7,23 @@ defmodule DB.AOM do """ use Ecto.Schema use TypedEctoSchema - alias DB.{Dataset, Region, Repo} + alias DB.{Dataset, Departement, Region, Repo} alias Geo.MultiPolygon typed_schema "aom" do + # composition_res_id matches the id_reseau attribute from the Cerema dataset it’s the official ID of the AOM field(:composition_res_id, :integer) field(:insee_commune_principale, :string) - field(:departement, :string) field(:siren, :string) field(:nom, :string) field(:forme_juridique, :string) field(:nombre_communes, :integer) - field(:population_municipale, :integer) - field(:population_totale, :integer) + field(:population, :integer) field(:surface, :string) - field(:commentaire, :string) field(:geom, Geo.PostGIS.Geometry) :: MultiPolygon.t() belongs_to(:region, Region) + belongs_to(:departement_object, Departement, foreign_key: :departement, references: :insee, type: :string) has_many(:datasets, Dataset) many_to_many(:legal_owners_dataset, Dataset, join_through: "dataset_aom_legal_owner") @@ -34,5 +33,5 @@ defmodule DB.AOM do def get(insee_commune_principale: nil), do: nil def get(insee_commune_principale: insee), do: Repo.get_by(AOM, insee_commune_principale: insee) - def created_in_2022?(%__MODULE__{composition_res_id: composition_res_id}), do: composition_res_id >= 1_000 + def created_after_2021?(%__MODULE__{composition_res_id: composition_res_id}), do: composition_res_id >= 1_000 end diff --git a/apps/transport/lib/db/commune.ex b/apps/transport/lib/db/commune.ex index 749771756e..a4321ed448 100644 --- a/apps/transport/lib/db/commune.ex +++ b/apps/transport/lib/db/commune.ex @@ -16,6 +16,7 @@ defmodule DB.Commune do field(:siren, :string) field(:arrondissement_insee, :string) + # In theory a commune has only one AOM, the reference is done through the composition_res_id attribute belongs_to(:aom_res, AOM, references: :composition_res_id) belongs_to(:region, Region) belongs_to(:departement, Departement, foreign_key: :departement_insee, references: :insee, type: :string) diff --git a/apps/transport/lib/mix/tasks/transport/import_aoms.ex b/apps/transport/lib/mix/tasks/transport/import_aoms.ex new file mode 100644 index 0000000000..ee3e79b9fa --- /dev/null +++ b/apps/transport/lib/mix/tasks/transport/import_aoms.ex @@ -0,0 +1,371 @@ +defmodule Mix.Tasks.Transport.ImportAoms do + @moduledoc """ + Import the AOM files and updates the database. + + The AOM files are custom made from an Excel file from the Cerema + https://www.data.gouv.fr/fr/datasets/liste-et-composition-des-autorites-organisatrices-de-la-mobilite-aom/ + https://www.cerema.fr/fr/actualites/liste-composition-autorites-organisatrices-mobilite-au-1er-4 + + and pushed as community resources on data.gouv.fr. + There are 2 files: + - one with the description of each AOM + - one with the list of cities that are part of each AOM + + This is a one shot import task, run when the AOM have changed, at least every year. + + The import can be launched through mix transport.import_aoms + """ + + @shortdoc "Refreshes the database table `aom` with the latest data" + use Mix.Task + import Ecto.{Query} + alias DB.{AOM, Commune, Region, Repo} + require Logger + + # The 2 community resources stable urls + # To create the following file: just rename columns and export as CSV, no content modification needed + @aom_file "https://gist.githubusercontent.com/vdegove/42d134c59b286525ff412876be3b6547/raw/d631b46c9096c148d854fbd5e9710987efb22999/base-rt-2023-diffusion-v2-aoms.csv" + # To create the following file: just delete useless columns and export as CSV, no content modification needed + @aom_insee_file "https://gist.githubusercontent.com/vdegove/42d134c59b286525ff412876be3b6547/raw/d631b46c9096c148d854fbd5e9710987efb22999/base-rt-2023-diffusion-v2-communes.csv" + + # We don’t add collectivité d’outremer de Saint-Martin + @ignored_aom_ids ["312"] + + @spec to_int(binary()) :: number() | nil + def to_int(""), do: nil + def to_int("#N/D"), do: nil + def to_int("#ERROR!"), do: nil + + def to_int(str) do + str + |> String.replace(" ", "") + # also replace non breaking spaces + |> String.replace("\u00A0", "") + |> String.to_integer() + end + + @spec changeset(map()) :: {integer(), Ecto.Changeset.t()} + def changeset(line) do + aom = line |> existing_or_new_aom() |> Repo.preload(:region) + + nom = normalize_nom(String.trim(line["Nom"])) + new_region = Repo.get_by(Region, nom: normalize_region(line["Région"])) + + if !is_nil(aom.region) and !is_nil(new_region) and aom.region != new_region do + Logger.info("aom #{nom} || previous region #{aom.region.nom} --- #{new_region.nom}") + end + + external_id = to_int(line["Id réseau"]) + + {external_id, + Ecto.Changeset.change(aom, %{ + composition_res_id: external_id, + departement: extract_departement_insee(line["Département"]), + siren: line["N° SIREN"] |> String.trim(), + nom: nom, + forme_juridique: normalize_forme(line["Forme juridique"]), + # This is inconsistent with the real number of communes for some lines + nombre_communes: to_int(line["Nombre de communes"]), + population: to_int(line["Population"]), + # Database stores a string, we could use a float + surface: line["Surface (km²)"] |> String.trim() |> String.replace(",", "."), + region: new_region + })} + end + + @spec normalize_region(binary()) :: binary() + defp normalize_region("Grand-Est"), do: "Grand Est" + defp normalize_region("Provence-Alpes-Côte-d'Azur"), do: "Région Sud — Provence-Alpes-Côte d’Azur" + defp normalize_region("Provence-Alpes-Côte d'Azur"), do: "Région Sud — Provence-Alpes-Côte d’Azur" + defp normalize_region("Nouvelle Aquitaine"), do: "Nouvelle-Aquitaine" + defp normalize_region("Auvergne-Rhône Alpes"), do: "Auvergne-Rhône-Alpes" + defp normalize_region("Nouvelle Calédonie"), do: "Nouvelle-Calédonie" + defp normalize_region(region), do: region + + @spec normalize_forme(binary()) :: binary() + defp normalize_forme("CA"), do: "Communauté d'agglomération" + defp normalize_forme("CU"), do: "Communauté urbaine" + defp normalize_forme("CC"), do: "Communauté de communes" + defp normalize_forme("METRO"), do: "Métropole" + defp normalize_forme("PETR"), do: "Pôle d'équilibre territorial et rural" + defp normalize_forme(f), do: f + + @spec normalize_nom(binary()) :: binary() + defp normalize_nom("SIVOTU (nouvelle dénomination le 24/02/2010:AGGLOBUS)"), do: "Agglobus" + defp normalize_nom("ILE D'YEU"), do: "L'Île-d'Yeu" + defp normalize_nom(n), do: n + + # Oups + defp extract_departement_insee("977 - Collectivité d’outre-mer de Nouvelle Calédonie"), do: "988" + defp extract_departement_insee(insee_and_name), do: insee_and_name |> String.split(" - ") |> hd() |> String.trim() + + def run(_params) do + Logger.info("Starting AOM import") + Mix.Task.run("app.start") + + old_aoms = + AOM + |> Repo.all() + |> Map.new(fn aom -> {aom.composition_res_id, aom} end) + + # get all the aom to import, outside of the transaction to reduce the time in the transaction + # this already builds the changeset + # Mapset of {composition_res_id, changeset} + aoms_to_add = get_aom_to_import() |> Enum.map(&changeset/1) |> MapSet.new() + + display_changes(old_aoms, aoms_to_add) + + {:ok, _} = + Repo.transaction( + fn -> + disable_trigger() + # we load all aoms + import_aoms(aoms_to_add) + # Some datasets should change AOM + migrate_datasets_to_new_aoms() + delete_old_aoms(aoms_to_add, old_aoms) + # we load the join on cities + import_insee_aom() + enable_trigger() + end, + timeout: 1_000_000 + ) + + # we can then compute the aom geometries (the union of each cities geometries) + compute_geom() + set_main_commune() + + :ok + end + + defp get_aom_to_import do + Logger.info("Importing Cerema file…") + + {:ok, %HTTPoison.Response{status_code: 200, body: body}} = + HTTPoison.get(@aom_file, [], hackney: [follow_redirect: true]) + + {:ok, stream} = StringIO.open(body) + + stream + |> IO.binstream(:line) + |> CSV.decode(separator: ?,, headers: true, validate_row_length: true) + |> Enum.map(fn {:ok, line} -> line end) + |> Enum.reject(fn line -> line["Id réseau"] in (["", nil] ++ @ignored_aom_ids) end) + end + + defp existing_or_new_aom(line) do + AOM + |> Repo.get_by(composition_res_id: to_int(line["Id réseau"])) + |> case do + nil -> + %AOM{} + + aom -> + aom + end + end + + defp import_aoms(aoms_to_add) do + Logger.info("importing AOMs…") + aoms_to_add |> Enum.each(fn {_id, aom} -> Repo.insert_or_update!(aom) end) + end + + defp delete_old_aoms(aom_added, old_aoms) do + Logger.info("deleting removed aom") + + composition_res_id_added = + aom_added + |> Enum.map(fn {id, _changeset} -> id end) + |> MapSet.new() + + old_aoms + |> Enum.each(fn {composition_res_id, old_aom} -> + unless MapSet.member?(composition_res_id_added, composition_res_id) do + Logger.info("trying to delete old aom: #{old_aom.id} - #{old_aom.nom}") + + # Note: if the delete is impossible, you need to find what still depend on this aom, + # and change the link to a newer aom + Repo.delete!(old_aom) + end + end) + end + + defp import_insee_aom do + Logger.info("Linking aoms to cities") + + {:ok, %HTTPoison.Response{status_code: 200, body: body}} = + HTTPoison.get(@aom_insee_file, [], hackney: [follow_redirect: true]) + + {:ok, stream} = StringIO.open(body) + + stream + |> IO.binstream(:line) + |> CSV.decode(separator: ?,, headers: true, validate_row_length: true) + |> Enum.map(fn {:ok, line} -> {line["N° INSEE"], line["Id réseau"]} end) + |> Enum.reject(fn {_insee, id_reseau} -> id_reseau == "" || id_reseau == "-" end) + |> Enum.flat_map(fn {insee, id_reseau} -> + # To reduce the number of UPDATE in the DB, we first check which city needs to be updated + Commune + |> where([c], c.insee == ^insee and (c.aom_res_id != ^id_reseau or is_nil(c.aom_res_id))) + |> select([c], c.id) + |> Repo.all() + |> Enum.map(fn c -> {c, id_reseau} end) + end) + |> Enum.reduce(%{}, fn {commune, aom}, commune_by_aom -> + # Then we group those city by AO, to only do one UPDATE query for several cities + commune_by_aom + |> Map.update(aom, [commune], fn list_communes -> [commune | list_communes] end) + end) + |> Enum.map(fn {aom, list_communes} -> + Commune + |> where([c], c.id in ^list_communes) + |> Repo.update_all(set: [aom_res_id: aom]) + end) + end + + defp compute_geom do + Logger.info("computing AOM geometries") + + Repo.update_all( + from(a in AOM, + update: [ + set: [ + geom: + fragment( + """ + ( + SELECT + ST_UNION(commune.geom) + FROM commune + WHERE commune.aom_res_id = ? + ) + """, + a.composition_res_id + ) + ] + ] + ), + [], + timeout: 1_000_000 + ) + end + + def set_main_commune do + Logger.info("set main commune") + + max_for_each_aom = + from(c in DB.Commune, + where: not is_nil(c.aom_res_id), + group_by: c.aom_res_id, + select: %{aom_res_id: c.aom_res_id, max_population: max(c.population)} + ) + + main_communes = + from(c in DB.Commune, + where: not is_nil(c.aom_res_id), + join: max_for_each_aom in subquery(max_for_each_aom), + on: c.aom_res_id == max_for_each_aom.aom_res_id and c.population == max_for_each_aom.max_population, + select: [c.aom_res_id, c.insee] + ) + + main_communes = + main_communes + |> DB.Repo.all() + |> MapSet.new(fn [aom_res_id, insee] -> {aom_res_id, insee} end) + + {:ok, _} = + Repo.transaction( + fn -> + disable_trigger() + + main_communes + |> Enum.each(fn {aom_res_id, insee} -> + AOM + |> Repo.get_by!(composition_res_id: aom_res_id) + |> Ecto.Changeset.change(%{insee_commune_principale: insee}) + |> Repo.update() + end) + + enable_trigger() + end, + timeout: 1_000_000 + ) + end + + defp disable_trigger do + Repo.query!("ALTER TABLE aom DISABLE TRIGGER refresh_places_aom_trigger;") + Repo.query!("ALTER TABLE commune DISABLE TRIGGER refresh_places_commune_trigger;") + end + + defp enable_trigger do + Repo.query!("ALTER TABLE aom ENABLE TRIGGER refresh_places_aom_trigger;") + Repo.query!("ALTER TABLE commune ENABLE TRIGGER refresh_places_commune_trigger;") + Repo.query!("REFRESH MATERIALIZED VIEW places;") + end + + defp migrate_datasets_to_new_aoms do + queries = """ + -- This could be mostly automatized, you just have to look for a commune of the old AOM and see where it was migrated. + -- + -- 2023 + -- [info] Datasets still associated with deleted AOM as territory : + -- %{230 => [[230, 275, 401]], 449 => [[449, 1509, 653]]} + -- [info] Datasets still associated with deleted AOM as legal owner: + -- %{230 => [[230, 275, 401], [230, 275, 338]], + -- 440 => [[440, 1475, 732]], + -- 449 => [[449, 1509, 653], [449, 1509, 787]], + -- 558 => [[558, 1469, 732]], + -- 677 => [[677, 1478, 732]]} + -- CC du Pays d'Issoudun (id : 230, res_id: 275) to Région Centre-Val de Loire (CC du Pays d'Issoudun) (res_id: 13608) + -- Migrates this dataset as both territory and legal owner https://transport.data.gouv.fr/datasets/issoudun-offre-theorique-mobilite-reseau-urbain + -- This one as legal owner https://transport.data.gouv.fr/datasets/arrets-itineraires-et-horaires-theoriques-des-reseaux-de-transport-des-membres-de-jvmalin + update dataset set aom_id = (select id from aom where composition_res_id = 13608) where aom_id = 230; + update dataset_aom_legal_owner set aom_id = (select id from aom where composition_res_id = 13608) where aom_id = 230; + -- CC Arve et Salève (id : 440, res_id: 1475) to SM4CC (res_id: 417) + -- CC Faucigny-Glières (id: 558, res_id :1509 to SM4CC (res_id: 417) + -- CC du Pays Rochois (id: 677, res_id: 1478 to SM4CC (res_id: 417) + -- There is a fourth CC in SM4CC, CC des Quatre Rivières (haute savoie) + -- Removes aggregate legal owner here https://transport.data.gouv.fr/datasets/agregat-oura but keeps SM4CC + delete from dataset_aom_legal_owner where aom_id in (440, 558, 677); + -- L'Île-d'Yeu (id: 449, res_id: 1509) to L’Île-d’Yeu (res_id: 310); + update dataset set aom_id = (select id from aom where composition_res_id = 310) where aom_id = 449; + update dataset_aom_legal_owner set aom_id = (select id from aom where composition_res_id = 310) where aom_id = 449; + """ + + queries |> String.split(";") |> Enum.each(&Repo.query!/1) + end + + defp display_changes(old_aoms, aoms_to_add) do + mapset_first_elem_diff = fn a, b -> + a |> MapSet.new(&elem(&1, 0)) |> MapSet.difference(b |> MapSet.new(&elem(&1, 0))) + end + + new_aoms = mapset_first_elem_diff.(aoms_to_add, old_aoms) + removed_aoms = mapset_first_elem_diff.(old_aoms, aoms_to_add) + Logger.info("#{new_aoms |> Enum.count()} new AOMs. reseau_id codes: #{Enum.join(new_aoms, ", ")}") + Logger.info("#{removed_aoms |> Enum.count()} removed AOMs. reseau_id codes: #{Enum.join(removed_aoms, ", ")}") + + # Some Ecto fun: two ways of joining through assoc, see https://hexdocs.pm/ecto/associations.html + deleted_aom_datasets = + DB.Dataset + |> join(:left, [d], aom in assoc(d, :aom)) + |> where([d, aom], aom.composition_res_id in ^(removed_aoms |> MapSet.to_list())) + |> select([d, aom], [aom.id, aom.composition_res_id, d.id]) + |> DB.Repo.all() + |> Enum.group_by(&hd(&1)) + + Logger.info("Datasets still associated with deleted AOM as territory : #{inspect(deleted_aom_datasets)}") + + deleted_legal_owners_query = + from(d in DB.Dataset, + # This magically works with the many_to_many + join: aom in assoc(d, :legal_owners_aom), + where: aom.composition_res_id in ^(removed_aoms |> MapSet.to_list()), + select: [aom.id, aom.composition_res_id, d.id] + ) + + deleted_legal_owners = deleted_legal_owners_query |> DB.Repo.all() |> Enum.group_by(&hd(&1)) + + Logger.info("Datasets still associated with deleted AOM as legal owner: #{inspect(deleted_legal_owners)}") + end +end diff --git a/apps/transport/lib/mix/tasks/transport/import_departements.ex b/apps/transport/lib/mix/tasks/transport/import_departements.ex index 6bd2b387f4..4fb76a4da7 100644 --- a/apps/transport/lib/mix/tasks/transport/import_departements.ex +++ b/apps/transport/lib/mix/tasks/transport/import_departements.ex @@ -8,9 +8,9 @@ defmodule Mix.Tasks.Transport.ImportDepartements do alias DB.{Departement, Repo} require Logger - @departements_geojson_url "http://etalab-datasets.geo.data.gouv.fr/contours-administratifs/2022/geojson/departements-100m.geojson" + @departements_geojson_url "http://etalab-datasets.geo.data.gouv.fr/contours-administratifs/2023/geojson/departements-100m.geojson" # See https://github.com/etalab/decoupage-administratif - @departements_url "https://unpkg.com/@etalab/decoupage-administratif@2.2.1/data/departements.json" + @departements_url "https://unpkg.com/@etalab/decoupage-administratif@3.1.1/data/departements.json" def insert_or_update_departement( %{ @@ -22,23 +22,34 @@ defmodule Mix.Tasks.Transport.ImportDepartements do }, geojsons ) do - insee - |> get_or_create_departement() - |> Changeset.change(%{ - insee: insee, - region_insee: region, - chef_lieu: chef_lieu, - nom: nom, - zone: zone, - geom: build_geometry(geojsons, insee) - }) - |> Repo.insert_or_update!() + changeset = + insee + |> get_or_create_departement() + |> Changeset.change(%{ + insee: insee, + region_insee: region, + chef_lieu: chef_lieu, + nom: nom, + zone: zone, + geom: build_geometry(geojsons, insee) + }) + + changeset_change_keys = changeset.changes |> Map.keys() + + unless Enum.empty?(changeset_change_keys -- [:geom, :population]) do + Logger.info("Important changes for INSEE #{insee}. #{readable_changeset(changeset)}") + end + + changeset |> Repo.insert_or_update!() + changeset_change_keys end defp geojson_by_insee do - @departements_geojson_url - |> HTTPoison.get!(timeout: 15_000, recv_timeout: 15_000) - |> Map.fetch!(:body) + %{status: 200, body: body} = + Req.get!(@departements_geojson_url, connect_options: [timeout: 15_000], receive_timeout: 15_000) + + body + # Req doesn’t decode GeoJSON body automatically as it does for JSON |> Jason.decode!() |> Map.fetch!("features") |> Enum.into(%{}, fn record -> {record["properties"]["code"], record["geometry"]} end) @@ -64,13 +75,19 @@ defmodule Mix.Tasks.Transport.ImportDepartements do end defp load_etalab_departements do - @departements_url - |> HTTPoison.get!(timeout: 15_000, recv_timeout: 15_000) - |> Map.fetch!(:body) - |> Jason.decode!() + %{status: 200, body: body} = + Req.get!(@departements_url, connect_options: [timeout: 15_000], receive_timeout: 15_000) + + body |> Enum.filter(&(&1["zone"] in ["metro", "drom"] or &1["nom"] == "Nouvelle-Calédonie")) end + defp readable_changeset(%Ecto.Changeset{changes: changes, data: data}) do + changes + |> Map.keys() + |> Enum.map_join(" ; ", fn key -> "#{key}: #{Map.get(data, key)} => #{Map.get(changes, key)}" end) + end + def run(_params) do Logger.info("Importing departements") @@ -92,10 +109,13 @@ defmodule Mix.Tasks.Transport.ImportDepartements do Logger.info("#{nb_new} new departements") Logger.info("#{nb_removed} departements should be removed") + Logger.info("Deleting removed communes…") Departement |> where([c], c.insee in ^removed_departements) |> Repo.delete_all() - etalab_departements |> Enum.each(&insert_or_update_departement(&1, geojsons)) - + Logger.info("Updating departments (including potentially incorrect geometry)…") + changelist = etalab_departements |> Enum.map(&insert_or_update_departement(&1, geojsons)) + Logger.info("Finished. Count of changes: #{inspect(changelist |> List.flatten() |> Enum.frequencies())}") + Logger.info("Ensure valid geometries and rectify if needed.") ensure_valid_geometries() end diff --git a/apps/transport/lib/mix/tasks/transport/import_epci.ex b/apps/transport/lib/mix/tasks/transport/import_epci.ex index 83ccd6dd02..abcfddffb3 100644 --- a/apps/transport/lib/mix/tasks/transport/import_epci.ex +++ b/apps/transport/lib/mix/tasks/transport/import_epci.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Transport.ImportEPCI do alias DB.{EPCI, Repo} require Logger - @epci_file "https://unpkg.com/@etalab/decoupage-administratif@2.0.0/data/epci.json" + @epci_file "https://unpkg.com/@etalab/decoupage-administratif@3.1.1/data/epci.json" def run(params) do Logger.info("importing epci") diff --git a/apps/transport/lib/transport/import_aom.ex b/apps/transport/lib/transport/import_aom.ex deleted file mode 100644 index d95f1386f2..0000000000 --- a/apps/transport/lib/transport/import_aom.ex +++ /dev/null @@ -1,291 +0,0 @@ -defmodule Transport.ImportAOMs do - @moduledoc """ - Import the AOM files and updates the database. - - The AOM files are custom made from an Excel file from the Cerema - https://www.data.gouv.fr/fr/datasets/liste-et-composition-des-autorites-organisatrices-de-la-mobilite-aom/ - https://www.cerema.fr/fr/actualites/liste-composition-autorites-organisatrices-mobilite-au-1er-4 - - and pushed as community resources on data.gouv.fr. - There are 2 files: - - one with the description of each AOM - - one with the list of cities that are part of each AOM - - This is a one shot import task, run when the AOM have changed, at least every year. - - The import can be launched from the site backoffice. - """ - - import Ecto.{Query} - alias DB.{AOM, Commune, Region, Repo} - require Logger - - # The 2 community resources stable urls - @aom_file "https://gist.githubusercontent.com/AntoineAugusti/8daac155f4d12b32ccd4e0a75bb964c7/raw/aoms.csv" - @aom_insee_file "https://gist.githubusercontent.com/AntoineAugusti/8daac155f4d12b32ccd4e0a75bb964c7/raw/aoms_insee.csv" - @ignored_aoms ["Saint-Martin"] - - @spec to_int(binary()) :: number() | nil - def to_int(""), do: nil - def to_int("#N/D"), do: nil - def to_int("#ERROR!"), do: nil - - def to_int(str) do - str - |> String.replace(" ", "") - # also replace non breaking spaces - |> String.replace("\u00A0", "") - |> String.to_integer() - end - - @spec changeset(AOM.t(), map()) :: {integer(), Ecto.Changeset.t()} - def changeset(aom, line) do - insee = - (Repo.get_by(Commune, siren: line["N°SIREN Commune principale"]) || - Repo.get_by(Commune, insee: line["N°SIREN Commune principale"])).insee - - nom = String.trim(line["Nom de l’AOM"]) - - new_region = Repo.get_by(Region, nom: normalize_region(line["Régions"])) - - if !is_nil(aom.region) and !is_nil(new_region) and aom.region != new_region do - Logger.info("aom #{nom} || previous region #{aom.region.nom} --- #{new_region.nom}") - end - - external_id = to_int(line["Id réseau"]) - - {external_id, - Ecto.Changeset.change(aom, %{ - composition_res_id: external_id, - insee_commune_principale: insee, - departement: line["Dep"] |> String.trim(), - siren: line["N° SIREN"] |> String.trim(), - nom: nom, - forme_juridique: normalize_forme(line["Forme juridique"]), - nombre_communes: to_int(line["Nombre de communes du RT"]), - population_municipale: to_int(line["Population municipale 2018"]), - population_totale: to_int(line["Population totale 2018"]), - surface: line["Surface (km²)"] |> String.trim(), - commentaire: line["Commentaire"] |> String.trim(), - region: new_region - })} - end - - @spec normalize_region(binary()) :: binary() - defp normalize_region("Grand-Est"), do: "Grand Est" - defp normalize_region("Provence-Alpes-Côte-d'Azur"), do: "Région Sud — Provence-Alpes-Côte d’Azur" - defp normalize_region("Provence-Alpes-Côte d'Azur"), do: "Région Sud — Provence-Alpes-Côte d’Azur" - defp normalize_region("Nouvelle Aquitaine"), do: "Nouvelle-Aquitaine" - defp normalize_region("Auvergne-Rhône Alpes"), do: "Auvergne-Rhône-Alpes" - defp normalize_region("Nouvelle Calédonie"), do: "Nouvelle-Calédonie" - defp normalize_region(region), do: region - - @spec normalize_forme(binary()) :: binary() - defp normalize_forme("CA"), do: "Communauté d'agglomération" - defp normalize_forme("CU"), do: "Communauté urbaine" - defp normalize_forme("CC"), do: "Communauté de communes" - defp normalize_forme("SIVU"), do: "Syndicat intercommunal à vocation unique" - defp normalize_forme("METRO"), do: "Métropole" - defp normalize_forme("SMF"), do: "Syndicat mixte fermé" - defp normalize_forme("SMO"), do: "Syndicat mixte ouvert" - defp normalize_forme("EPA"), do: "Établissement public administratif" - defp normalize_forme("EPL"), do: "Établissement public local" - defp normalize_forme("PETR"), do: "Pôle d'équilibre territorial et rural" - defp normalize_forme(f), do: f - - def run do - aoms = - AOM - |> Repo.all() - |> Enum.map(fn aom -> {aom.composition_res_id, aom} end) - |> Map.new() - - # get all the aom to import, outside of the transaction to reduce the time in the transaction - aom_to_add = get_aom_to_import() - - {:ok, _} = - Repo.transaction( - fn -> - disable_trigger() - # we load all aoms - import_aoms(aom_to_add) - - migrate_aoms() - delete_old_aoms(aom_to_add, aoms) - - # we load the join on cities - import_insee_aom() - enable_trigger() - end, - timeout: 1_000_000 - ) - - # we can then compute the aom geometries (the union of each cities geometries) - compute_geom() - - :ok - end - - defp get_aom_to_import do - Logger.info("importing aoms") - - {:ok, %HTTPoison.Response{status_code: 200, body: body}} = - HTTPoison.get(@aom_file, [], hackney: [follow_redirect: true]) - - {:ok, stream} = StringIO.open(body) - - stream - |> IO.binstream(:line) - |> CSV.decode(separator: ?,, headers: true, validate_row_length: true) - |> Enum.reject(fn {:ok, line} -> line["Id réseau"] in ["", nil] or line["Nom de l’AOM"] in @ignored_aoms end) - |> Enum.map(fn {:ok, line} -> - AOM - |> Repo.get_by(composition_res_id: to_int(line["Id réseau"])) - |> case do - nil -> - %AOM{} - - aom -> - aom - end - |> Repo.preload(:region) - |> changeset(line) - end) - |> MapSet.new() - end - - defp import_aoms(aom_to_add) do - aom_to_add |> Enum.each(fn {_id, aom} -> Repo.insert_or_update!(aom) end) - end - - defp delete_old_aoms(aom_added, old_aoms) do - Logger.info("deleting removed aom") - - composition_res_id_added = - aom_added - |> Enum.map(fn {id, _changeset} -> id end) - |> MapSet.new() - - old_aoms - |> Enum.each(fn {composition_res_id, old_aom} -> - unless MapSet.member?(composition_res_id_added, composition_res_id) do - Logger.info("trying to delete old aom: #{old_aom.id} - #{old_aom.nom}") - - # Note: if the delete is impossible, you need to find what still depend on this aom, - # and change the link to a newer aom - Repo.delete!(old_aom) - end - end) - end - - defp import_insee_aom do - Logger.info("Linking aoms to cities") - - {:ok, %HTTPoison.Response{status_code: 200, body: body}} = - HTTPoison.get(@aom_insee_file, [], hackney: [follow_redirect: true]) - - {:ok, stream} = StringIO.open(body) - - aom_ids = AOM |> select([a], {a.siren, a.composition_res_id}) |> Repo.all() |> Enum.into(%{}) - - stream - |> IO.binstream(:line) - |> CSV.decode(separator: ?,, headers: true, validate_row_length: true) - |> Enum.map(fn {:ok, line} -> {line["siren_aom"], line["insee"]} end) - |> Enum.reject(fn {aom_siren, insee} -> aom_siren == "" || insee == "" || aom_siren not in Map.keys(aom_ids) end) - |> Enum.map(fn {aom_siren, insee} -> {aom_ids[aom_siren], insee} end) - |> Enum.flat_map(fn {aom, insee} -> - # To reduce the number of UPDATE in the DB, we first check which city needs to be updated - Commune - |> where([c], c.insee == ^insee and (c.aom_res_id != ^aom or is_nil(c.aom_res_id))) - |> select([c], c.id) - |> Repo.all() - |> Enum.map(fn c -> {aom, c} end) - end) - |> Enum.reduce(%{}, fn {aom, commune}, commune_by_aom -> - # Then we group those city by AO, to only do one UPDATE query for several cities - commune_by_aom - |> Map.update(aom, [commune], fn list_communes -> [commune | list_communes] end) - end) - |> Enum.map(fn {aom, list_communes} -> - Commune - |> where([c], c.id in ^list_communes) - |> Repo.update_all(set: [aom_res_id: aom]) - end) - end - - defp compute_geom do - Logger.info("computing AOM geometries") - - Repo.update_all( - from(a in AOM, - update: [ - set: [ - geom: - fragment( - """ - ( - SELECT - ST_UNION(commune.geom) - FROM commune - WHERE commune.aom_res_id = ? - ) - """, - a.composition_res_id - ) - ] - ] - ), - [], - timeout: 1_000_000 - ) - end - - defp disable_trigger do - Repo.query!("ALTER TABLE aom DISABLE TRIGGER refresh_places_aom_trigger;") - Repo.query!("ALTER TABLE commune DISABLE TRIGGER refresh_places_commune_trigger;") - end - - defp enable_trigger do - Repo.query!("ALTER TABLE aom ENABLE TRIGGER refresh_places_aom_trigger;") - Repo.query!("ALTER TABLE commune ENABLE TRIGGER refresh_places_commune_trigger;") - Repo.query!("REFRESH MATERIALIZED VIEW places;") - end - - defp migrate_aoms do - queries = """ - -- Sainte-Menehould to CC de l'Argonne Champenoise - update dataset set aom_id = (select id from aom where composition_res_id = 1163) where aom_id = 121; - -- Vierzon to région CVL - update dataset set aom_id = null, region_id = (select id from region where nom = 'Centre-Val de Loire') where aom_id = 126; - -- Sablé-sur-Sarthe to Communauté de communes du Pays Sabolien - update dataset set aom_id = (select id from aom where composition_res_id = 1290) where aom_id = 137; - -- Langres to PETR du Pays de Langres - update dataset set aom_id = (select id from aom where composition_res_id = 1172) where aom_id = 149; - -- Mayenne to CC Mayenne Communauté - update dataset set aom_id = (select id from aom where composition_res_id = 1277) where aom_id = 173; - -- Douarnenez to CC Douarnenez Communauté - update dataset set aom_id = (select id from aom where composition_res_id = 1375) where aom_id = 175; - -- Obernai to CC du Pays de Sainte-Odile - update dataset set aom_id = (select id from aom where composition_res_id = 1235) where aom_id = 210; - -- Nogent-le-Rotrou to CVL region - update dataset set aom_id = null, region_id = (select id from region where nom = 'Centre-Val de Loire') where aom_id = 215; - -- Mende, Figeac to Occitanie region - update dataset set aom_id = null, region_id = (select id from region where nom = 'Occitanie') where aom_id in (222, 223); - -- Tignes to Auvergne-Rhône-Alpes region - update dataset set aom_id = null, region_id = (select id from region where nom = 'Auvergne-Rhône-Alpes') where aom_id = 242; - -- Bernay to CC Intercom Bernay Terres de Normandie - update dataset set aom_id = (select id from aom where composition_res_id = 1108) where aom_id = 245; - -- Sud Estuaire to CC du Sud Estuaire - update dataset set aom_id = (select id from aom where composition_res_id = 1268) where aom_id = 246; - -- Oloron Sainte-Marie to CC du Haut Béarn - update dataset set aom_id = (select id from aom where composition_res_id = 1434) where aom_id = 248; - -- Granville to CC de Granville, Terre et Mer - update dataset set aom_id = (select id from aom where composition_res_id = 1114) where aom_id = 305; - -- Neufchâteau to CC de l'Ouest Vosgien - update dataset set aom_id = (select id from aom where composition_res_id = 1254) where aom_id = 304; - """ - - queries |> String.split(";") |> Enum.each(&Repo.query!/1) - end -end diff --git a/apps/transport/lib/transport/stats_handler.ex b/apps/transport/lib/transport/stats_handler.ex index 3f06d3d0c5..796840224b 100644 --- a/apps/transport/lib/transport/stats_handler.ex +++ b/apps/transport/lib/transport/stats_handler.ex @@ -41,9 +41,9 @@ defmodule Transport.StatsHandler do on: (d.id == legal_owners_dataset.id or d.aom_id == a.id) and d.is_active, as: :dataset ) - |> group_by([a], [a.id, a.population_totale, a.region_id]) + |> group_by([a], [a.id, a.population, a.region_id]) |> select([a, dataset: d], %{ - population: a.population_totale, + population: a.population, region_id: a.region_id, nb_datasets: count(d.id) }) diff --git a/apps/transport/lib/transport_web/api/controllers/stats_controller.ex b/apps/transport/lib/transport_web/api/controllers/stats_controller.ex index 21a7d909bc..639a4dd72b 100644 --- a/apps/transport/lib/transport_web/api/controllers/stats_controller.ex +++ b/apps/transport/lib/transport_web/api/controllers/stats_controller.ex @@ -75,8 +75,8 @@ defmodule TransportWeb.API.StatsController do defp filter_neg(val) when val < 0, do: nil defp filter_neg(val) when val >= 0, do: val - def new_aom_without_datasets?(%{created_in_2022: true, nb_datasets: 0}), do: true - def new_aom_without_datasets?(%{created_in_2022: true, dataset_types: %{pt: 0}}), do: true + def new_aom_without_datasets?(%{created_after_2021: true, nb_datasets: 0}), do: true + def new_aom_without_datasets?(%{created_after_2021: true, dataset_types: %{pt: 0}}), do: true def new_aom_without_datasets?(_), do: false @spec features(Ecto.Query.t()) :: [map()] @@ -292,7 +292,7 @@ defmodule TransportWeb.API.StatsController do |> select([aom, aggregates_by_aom: d], %{ geometry: aom.geom, id: aom.id, - created_in_2022: aom.composition_res_id >= 1_000, + created_after_2021: aom.composition_res_id >= 1_000, insee_commune_principale: aom.insee_commune_principale, nb_datasets: fragment("select count(id) from dataset where aom_id = ? and is_active", aom.id), dataset_formats: %{ @@ -400,7 +400,7 @@ defmodule TransportWeb.API.StatsController do %{ geometry: aom.geom, id: aom.id, - created_in_2022: aom.composition_res_id >= 1_000, + created_after_2021: aom.composition_res_id >= 1_000, insee_commune_principale: aom.insee_commune_principale, nom: aom.nom, forme_juridique: aom.forme_juridique, diff --git a/apps/transport/lib/transport_web/controllers/aoms.ex b/apps/transport/lib/transport_web/controllers/aoms.ex index 210650de3d..fe09134d2f 100644 --- a/apps/transport/lib/transport_web/controllers/aoms.ex +++ b/apps/transport/lib/transport_web/controllers/aoms.ex @@ -11,10 +11,10 @@ defmodule TransportWeb.AOMSController do :in_aggregate, :up_to_date, :has_realtime, - :population_municipale, :nom_commune, :insee_commune_principale, - :nombre_communes + :nombre_communes, + :population ] @spec index(Plug.Conn.t(), map()) :: Plug.Conn.t() @@ -38,7 +38,7 @@ defmodule TransportWeb.AOMSController do published: not Enum.empty?(all_datasets), in_aggregate: not Enum.empty?(aggregated_datasets), up_to_date: datasets_up_to_date, - population_municipale: aom.population_municipale, + population: aom.population, nom_commune: nom_commune, insee_commune_principale: aom.insee_commune_principale, nombre_communes: aom.nombre_communes, diff --git a/apps/transport/lib/transport_web/controllers/backoffice/page_controller.ex b/apps/transport/lib/transport_web/controllers/backoffice/page_controller.ex index d04d2ab65e..17e83fc57f 100644 --- a/apps/transport/lib/transport_web/controllers/backoffice/page_controller.ex +++ b/apps/transport/lib/transport_web/controllers/backoffice/page_controller.ex @@ -212,19 +212,6 @@ defmodule TransportWeb.Backoffice.PageController do |> Enum.sort_by(fn {{_reason, %DateTime{} = dt}, _emails} -> dt end, {:desc, DateTime}) end - def import_all_aoms(%Plug.Conn{} = conn, _params) do - conn = - try do - Transport.ImportAOMs.run() - conn |> put_flash(:info, "AOMs successfully imported") - rescue - e -> - conn |> put_flash(:error, "AOMs import failed. #{inspect(e)}") - end - - conn |> redirect(to: backoffice_page_path(conn, :index)) - end - def dataset_with_resource_under_90_availability do query = """ with down_ranges as (select *, tsrange(ru.start, ru.end) as down_range, tsrange(now()::timestamp - interval '30 day', now()::timestamp) as compute_range from resource_unavailability ru), diff --git a/apps/transport/lib/transport_web/controllers/page_controller.ex b/apps/transport/lib/transport_web/controllers/page_controller.ex index bca6f2fca9..5e751b7b0c 100644 --- a/apps/transport/lib/transport_web/controllers/page_controller.ex +++ b/apps/transport/lib/transport_web/controllers/page_controller.ex @@ -199,9 +199,9 @@ defmodule TransportWeb.PageController do defp count_aoms_with_dataset, do: Repo.aggregate(aoms_with_dataset(), :count, :id) - defp population_with_dataset, do: Repo.aggregate(aoms_with_dataset(), :sum, :population_totale) || 0 + defp population_with_dataset, do: Repo.aggregate(aoms_with_dataset(), :sum, :population) || 0 - defp population_totale, do: Repo.aggregate(AOM, :sum, :population_totale) + defp population_totale, do: Repo.aggregate(AOM, :sum, :population) defp percent_population, do: percent(population_with_dataset(), population_totale()) diff --git a/apps/transport/lib/transport_web/templates/aoms/index.html.heex b/apps/transport/lib/transport_web/templates/aoms/index.html.heex index 9169c7171f..5b9bbcc00f 100644 --- a/apps/transport/lib/transport_web/templates/aoms/index.html.heex +++ b/apps/transport/lib/transport_web/templates/aoms/index.html.heex @@ -26,7 +26,7 @@ <%= format_bool(aom.in_aggregate) %> <%= format_bool(aom.up_to_date) %> <%= format_bool(aom.has_realtime) %> - <%= aom.population_municipale %> + <%= aom.population %> <%= aom.nom_commune %> (<%= aom.insee_commune_principale %>) <%= aom.nombre_communes %> diff --git a/apps/transport/lib/transport_web/templates/backoffice/page/index.html.heex b/apps/transport/lib/transport_web/templates/backoffice/page/index.html.heex index e5909d9890..2ce8b82f4e 100644 --- a/apps/transport/lib/transport_web/templates/backoffice/page/index.html.heex +++ b/apps/transport/lib/transport_web/templates/backoffice/page/index.html.heex @@ -53,13 +53,6 @@ <%= submit(dgettext("backoffice", "Import all datasets from data.gouv")) %> <% end %> - - <%= dgettext("backoffice", "Update all AOMs") %> -
<%= dgettext("page-index", "Scheduled public transit") %>

- <%= @count_aoms_with_dataset %> + <%= format_number(@count_aoms_with_dataset) %>
<%= dgettext("page-index", "Territories covered") |> raw() %>
- <%= dgettext("page-index", "on") %> <%= @count_aoms %> + <%= dgettext("page-index", "on") %> <%= format_number(@count_aoms) %>
diff --git a/apps/transport/priv/gettext/backoffice.pot b/apps/transport/priv/gettext/backoffice.pot index d0ae7334f3..fa2ecf0bfb 100644 --- a/apps/transport/priv/gettext/backoffice.pot +++ b/apps/transport/priv/gettext/backoffice.pot @@ -165,10 +165,6 @@ msgstr "" msgid "Proxy configuration" msgstr "" -#, elixir-autogen, elixir-format -msgid "Update all AOMs" -msgstr "" - #, elixir-autogen, elixir-format msgid "Jobs" msgstr "" diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/backoffice.po b/apps/transport/priv/gettext/en/LC_MESSAGES/backoffice.po index c2c9f2205f..b62cbcb734 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/backoffice.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/backoffice.po @@ -165,10 +165,6 @@ msgstr "" msgid "Proxy configuration" msgstr "" -#, elixir-autogen, elixir-format -msgid "Update all AOMs" -msgstr "" - #, elixir-autogen, elixir-format msgid "Jobs" msgstr "" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/backoffice.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/backoffice.po index 03f116a4ac..eb98d94683 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/backoffice.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/backoffice.po @@ -165,10 +165,6 @@ msgstr "Montrer seulement les jeux de données" msgid "Proxy configuration" msgstr "Configuration du proxy" -#, elixir-autogen, elixir-format -msgid "Update all AOMs" -msgstr "Mettre à jour toutes les AOMs" - #, elixir-autogen, elixir-format msgid "Jobs" msgstr "Jobs" diff --git a/apps/transport/priv/repo/migrations/20231110140739_remove_columns_from_aom.exs b/apps/transport/priv/repo/migrations/20231110140739_remove_columns_from_aom.exs new file mode 100644 index 0000000000..f823b731d9 --- /dev/null +++ b/apps/transport/priv/repo/migrations/20231110140739_remove_columns_from_aom.exs @@ -0,0 +1,104 @@ +defmodule DB.Repo.Migrations.RemoveColumnsFromAOM do + use Ecto.Migration + + def change do + execute("UPDATE aom SET departement = '0' || departement WHERE length(departement) = 1;", "") + execute("UPDATE aom SET departement = '988' WHERE departement = '98';", "") + rename(table(:aom), :population_totale, to: :population) + + alter table(:aom) do + # CEREMA only provides one population column now + remove(:population_municipale, :integer) + # This column was empty in database + remove(:commentaire, :string) + # This column was empty in database, now we use the region relation + remove(:region_name, :string) + modify(:departement, references(:departement, column: :insee, type: :string), from: :string) + end + + execute(&execute_up/0, &execute_down/0) + + # Force update + execute("UPDATE dataset SET id = id", "") + end + + defp execute_up do + repo().query!(sql_dataset_update_function("population"), [], log: :info) + end + + def execute_down do + repo().query!(sql_dataset_update_function("population_totale"), [], log: :info) + end + + def sql_dataset_update_function(aom_population_attribute) do + """ + CREATE OR REPLACE FUNCTION dataset_search_update() RETURNS trigger as $$ + DECLARE + nom text; + region_nom region.nom%TYPE; + population dataset.population%TYPE; + BEGIN + + NEW.search_vector = setweight(to_tsvector('custom_french', coalesce(NEW.custom_title, '')), 'B') || + setweight(to_tsvector('custom_french', array_to_string(NEW.tags, ',')), 'C') || + setweight(to_tsvector('custom_french', coalesce(NEW.description, '')), 'D'); + + IF NEW.aom_id IS NOT NULL THEN + SELECT aom.nom, region.nom, aom.#{aom_population_attribute} INTO nom, region_nom, population + FROM aom + JOIN region ON region.id = aom.region_id + WHERE aom.id = NEW.aom_id; + + NEW.search_vector = NEW.search_vector || + setweight(to_tsvector('custom_french', coalesce(nom, '')), 'A') || + setweight(to_tsvector('custom_french', coalesce(region_nom, '')), 'B'); + NEW.population = population; + + SELECT string_agg(commune.nom, ' ') INTO nom + FROM commune + JOIN aom ON aom.composition_res_id = commune.aom_res_id + WHERE aom.id = NEW.aom_id; + + NEW.search_vector = NEW.search_vector || + setweight(to_tsvector('custom_french', coalesce(nom, '')), 'B'); + + ELSIF NEW.region_id IS NOT NULL THEN + SELECT region.nom, SUM(aom.#{aom_population_attribute}) INTO nom, population + FROM region + JOIN aom ON aom.region_id = region.id + WHERE region.id = NEW.region_id + GROUP BY region.nom; + + NEW.search_vector = NEW.search_vector || + setweight(to_tsvector('custom_french', coalesce(nom, '')), 'A'); + NEW.population = population; + + ELSE + SELECT coalesce(sum(c.population),0) INTO population FROM dataset_communes dc + LEFT JOIN commune c ON c.id = dc.commune_id WHERE dc.dataset_id = NEW.id; + + NEW.population = population; + END IF; + + IF EXISTS ( + select dataset_id + from dataset_aom_legal_owner + where dataset_id = NEW.id + group by dataset_id + having count(aom_id) >= 2 + ) THEN + SELECT string_agg(a.nom, ' ') INTO nom + from aom a + left join dataset_aom_legal_owner d on d.aom_id = a.id + where d.dataset_id = NEW.id OR a.id = NEW.aom_id; + + NEW.search_vector = NEW.search_vector || + setweight(to_tsvector('custom_french', coalesce(nom, '')), 'A'); + END IF; + + RETURN NEW; + END + $$ LANGUAGE plpgsql; + """ + end +end diff --git a/apps/transport/test/support/database_case.ex b/apps/transport/test/support/database_case.ex index 244e873861..97dc27c01a 100644 --- a/apps/transport/test/support/database_case.ex +++ b/apps/transport/test/support/database_case.ex @@ -39,7 +39,7 @@ defmodule TransportWeb.DatabaseCase do insee_commune_principale: "53130", nom: "Laval", region: Repo.get_by(Region, nom: "Pays de la Loire"), - population_totale: 42, + population: 42, composition_res_id: 1 }) @@ -47,7 +47,7 @@ defmodule TransportWeb.DatabaseCase do insee_commune_principale: "38185", nom: "Grenoble", region: Repo.get_by(Region, nom: "Auvergne-Rhône-Alpes"), - population_totale: 43, + population: 43, composition_res_id: 2 }) @@ -55,7 +55,7 @@ defmodule TransportWeb.DatabaseCase do insee_commune_principale: "36044", nom: "Châteauroux", region: Repo.get_by(Region, nom: "Auvergne-Rhône-Alpes"), - population_totale: 44, + population: 44, composition_res_id: 3 }) @@ -84,7 +84,7 @@ defmodule TransportWeb.DatabaseCase do insee_commune_principale: "75056", nom: "Île-de-France Mobilités", region: Repo.get_by(Region, nom: "Île-de-France"), - population_totale: 45, + population: 45, composition_res_id: 4 }) diff --git a/apps/transport/test/support/factory.ex b/apps/transport/test/support/factory.ex index 71787cddc2..170aef2e04 100644 --- a/apps/transport/test/support/factory.ex +++ b/apps/transport/test/support/factory.ex @@ -8,6 +8,27 @@ defmodule DB.Factory do # Ecto records + def departement_factory do + %DB.Departement{ + insee: "38", + nom: "Isère", + geom: %Geo.Polygon{ + coordinates: [ + [ + {55.0, 3.0}, + {60.0, 3.0}, + {60.0, 5.0}, + {55.0, 5.0}, + {55.0, 3.0} + ] + ], + srid: 4326, + properties: %{} + }, + zone: "metro" + } + end + def region_factory do %DB.Region{ nom: sequence("region_nom") diff --git a/apps/transport/test/transport/stats_handler_test.exs b/apps/transport/test/transport/stats_handler_test.exs index 46a302f0de..0f7ca27f87 100644 --- a/apps/transport/test/transport/stats_handler_test.exs +++ b/apps/transport/test/transport/stats_handler_test.exs @@ -40,7 +40,7 @@ defmodule Transport.StatsHandlerTest do end test "climate_resilience_bill_count" do - aom = insert(:aom, population_totale: 1_000) + aom = insert(:aom, population: 1_000) insert(:dataset, aom: aom, is_active: false, custom_tags: ["loi-climat-resilience"], type: "public-transit") insert(:dataset, aom: aom, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit") insert(:dataset, aom: aom, is_active: true, custom_tags: ["loi-climat-resilience"], type: "public-transit") @@ -145,9 +145,9 @@ defmodule Transport.StatsHandlerTest do end test "uses legal owners to assign datasets to AOMs" do - aom1 = insert(:aom, population_totale: 1_000_000) - aom2 = insert(:aom, population_totale: 1_000_000) - insert(:aom, population_totale: 1_000_000) + aom1 = insert(:aom, population: 1_000_000) + aom2 = insert(:aom, population: 1_000_000) + insert(:aom, population: 1_000_000) insert(:dataset, type: "public-transit", is_active: true, legal_owners_aom: [aom2], aom: aom1) assert %{nb_aoms_with_data: 2, nb_aoms: 3, population_couverte: 2, population_totale: 3} = compute_stats() diff --git a/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs b/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs index 312e2b3653..2a63c182e1 100644 --- a/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/api/aom_controller_test.exs @@ -22,9 +22,11 @@ defmodule TransportWeb.API.AomControllerTest do properties: %{} } + insert(:departement) + insert(:aom, geom: geom, - departement: "75", + departement: "38", forme_juridique: "Communauté de communes", insee_commune_principale: "38185", siren: "247400690" @@ -36,7 +38,7 @@ defmodule TransportWeb.API.AomControllerTest do assert_response_schema(json, "AOMResponse", TransportWeb.API.Spec.spec()) assert json == %{ - "departement" => "75", + "departement" => "38", "forme_juridique" => "Communauté de communes", "insee_commune_principale" => "38185", "nom" => "Grenoble", diff --git a/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs b/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs index a65c290b1d..7a681989d6 100644 --- a/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/api/stats_controller_test.exs @@ -178,7 +178,7 @@ defmodule TransportWeb.API.StatsControllerTest do geom: "SRID=4326;LINESTRING(1 1,2 2)" |> Geo.WKT.decode!() ) - assert DB.AOM.created_in_2022?(aom) + assert DB.AOM.created_after_2021?(aom) assert [] == TransportWeb.API.StatsController.quality_features_query() |> TransportWeb.API.StatsController.features() @@ -186,7 +186,7 @@ defmodule TransportWeb.API.StatsControllerTest do # If created before 2022, it is present even without a dataset aom = aom |> Ecto.Changeset.change(%{composition_res_id: 500}) |> DB.Repo.update!() - refute DB.AOM.created_in_2022?(aom) + refute DB.AOM.created_after_2021?(aom) assert [%{"properties" => %{"dataset_count" => 0, "nom" => ^aom_nom}}] = TransportWeb.API.StatsController.quality_features_query() |> TransportWeb.API.StatsController.features() @@ -195,7 +195,7 @@ defmodule TransportWeb.API.StatsControllerTest do aom = aom |> Ecto.Changeset.change(%{composition_res_id: 1_200}) |> DB.Repo.update!() insert(:dataset, is_active: true, aom: aom, type: "public-transit") - assert DB.AOM.created_in_2022?(aom) + assert DB.AOM.created_after_2021?(aom) assert [%{"properties" => %{"dataset_types" => %{pt: 1}, "nom" => ^aom_nom}}] = TransportWeb.API.StatsController.quality_features_query() |> TransportWeb.API.StatsController.features() diff --git a/apps/transport/test/transport_web/controllers/dataset_search_test.exs b/apps/transport/test/transport_web/controllers/dataset_search_test.exs index 57fa6080ee..ae433cd6b3 100644 --- a/apps/transport/test/transport_web/controllers/dataset_search_test.exs +++ b/apps/transport/test/transport_web/controllers/dataset_search_test.exs @@ -258,8 +258,8 @@ defmodule TransportWeb.DatasetSearchControllerTest do end test "when searching for a region, use the population to sort" do - small_aom = insert(:aom, region: region = insert(:region), population_totale: 100) - big_aom = insert(:aom, region: region, population_totale: 200) + small_aom = insert(:aom, region: region = insert(:region), population: 100) + big_aom = insert(:aom, region: region, population: 200) # regional dataset: first result expected region_dataset = insert(:dataset, region_id: region.id, is_active: true, population: 0)