-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into phoenix_1.7
- Loading branch information
Showing
9 changed files
with
169 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,19 +1,34 @@ | ||
| defmodule DB.EPCI do | ||
| @moduledoc """ | ||
| EPCI schema | ||
| EPCI schema. | ||
| Link the EPCI to some Communes. | ||
| The EPCI are loaded once and for all by the task transport/lib/transport/import_epci.ex | ||
| The EPCI are loaded by the task transport/lib/transport/import_epci.ex. | ||
| The EPCI imported are only "à fiscalité propre". This excludes Etablissements Publics Territoriaux. | ||
| This allows to have a 1 to 1 relation between a commune and an EPCI. | ||
| """ | ||
| use Ecto.Schema | ||
| use TypedEctoSchema | ||
| import Ecto.Changeset | ||
|
|
||
| typed_schema "epci" do | ||
| field(:code, :string) | ||
| field(:insee, :string) | ||
| field(:nom, :string) | ||
| field(:type, :string) | ||
| field(:mode_financement, :string) | ||
| field(:geom, Geo.PostGIS.Geometry) :: Geo.MultiPolygon.t() | ||
| has_many(:communes, DB.Commune, foreign_key: :epci_insee) | ||
| end | ||
|
|
||
| # for the moment we don't need a link relational link to the Commune table, | ||
| # so we only store an array of insee code | ||
| field(:communes_insee, {:array, :string}, default: []) | ||
| def changeset(epci, attrs) do | ||
| epci | ||
| |> cast(attrs, [:insee, :nom, :geom, :type, :mode_financement]) | ||
| |> validate_required([:insee, :nom, :geom, :type, :mode_financement]) | ||
| |> validate_inclusion(:type, allowed_types()) | ||
| |> validate_inclusion(:mode_financement, allowed_mode_financement()) | ||
| end | ||
|
|
||
| defp allowed_types, | ||
| do: ["Communauté d'agglomération", "Communauté urbaine", "Communauté de communes", "Métropole", "Métropole de Lyon"] | ||
|
|
||
| defp allowed_mode_financement, do: ["Fiscalité professionnelle unique", "Fiscalité additionnelle"] | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,47 +1,45 @@ | ||
| defmodule Mix.Tasks.Transport.ImportEPCI do | ||
| @moduledoc """ | ||
| Import the EPCI file to get the relation between the cities and the EPCI | ||
| Run: mix transport.importEPCI | ||
| """ | ||
|
|
||
| use Mix.Task | ||
| import Ecto.Query | ||
| alias Ecto.Changeset | ||
| alias DB.{EPCI, Repo} | ||
| alias DB.{Commune, EPCI, Repo} | ||
| require Logger | ||
|
|
||
| @epci_file "https://unpkg.com/@etalab/[email protected]/data/epci.json" | ||
| @epci_geojson_url "http://etalab-datasets.geo.data.gouv.fr/contours-administratifs/2023/geojson/epci-100m.geojson" | ||
|
|
||
| def run(params) do | ||
| Logger.info("importing epci") | ||
| def run(_params) do | ||
| Logger.info("Importing EPCIs") | ||
|
|
||
| if params[:no_start] do | ||
| HTTPoison.start() | ||
| else | ||
| Mix.Task.run("app.start", []) | ||
| end | ||
| Mix.Task.run("app.start") | ||
|
|
||
| with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <- HTTPoison.get(@epci_file), | ||
| {:ok, json} <- Jason.decode(body) do | ||
| json |> Enum.each(&insert_epci/1) | ||
|
|
||
| # Remove EPCIs that have been removed | ||
| epci_codes = Enum.map(json, & &1["code"]) | ||
| EPCI |> where([e], e.code not in ^epci_codes) |> Repo.delete_all() | ||
|
|
||
| nb_epci = Repo.aggregate(EPCI, :count, :id) | ||
| Logger.info("#{nb_epci} are now in database") | ||
| :ok | ||
| else | ||
| e -> | ||
| Logger.warning("impossible to fetch epci file, error #{inspect(e)}") | ||
| :error | ||
| end | ||
| %{status: 200, body: json} = Req.get!(@epci_file, connect_options: [timeout: 15_000], receive_timeout: 15_000) | ||
| check_communes_list(json) | ||
| geojsons = geojson_by_insee() | ||
|
|
||
| json |> Enum.each(&insert_epci(&1, geojsons)) | ||
| json |> Enum.each(&update_communes_epci/1) | ||
|
|
||
| # Remove EPCIs that have been removed | ||
| epci_codes = json |> Enum.map(& &1["code"]) | ||
| EPCI |> where([e], e.insee not in ^epci_codes) |> Repo.delete_all() | ||
|
|
||
| nb_epci = Repo.aggregate(EPCI, :count, :id) | ||
| Logger.info("#{nb_epci} are now in database") | ||
| Logger.info("Ensure valid geometries and rectify if needed.") | ||
| ensure_valid_geometries() | ||
| :ok | ||
| end | ||
|
|
||
| @spec get_or_create_epci(binary()) :: EPCI.t() | ||
| defp get_or_create_epci(code) do | ||
| EPCI | ||
| |> Repo.get_by(code: code) | ||
| |> Repo.get_by(insee: code) | ||
| |> case do | ||
| nil -> | ||
| %EPCI{} | ||
|
|
@@ -51,21 +49,82 @@ defmodule Mix.Tasks.Transport.ImportEPCI do | |
| end | ||
| end | ||
|
|
||
| @spec insert_epci(map()) :: any() | ||
| defp insert_epci(%{"code" => code, "nom" => nom, "membres" => m}) do | ||
| @spec insert_epci(map(), map()) :: any() | ||
| defp insert_epci(%{"code" => code, "nom" => nom, "type" => type, "modeFinancement" => mode_financement}, geojsons) do | ||
| code | ||
| |> get_or_create_epci() | ||
| |> Changeset.change(%{ | ||
| code: code, | ||
| |> EPCI.changeset(%{ | ||
| insee: code, | ||
| nom: nom, | ||
| communes_insee: get_insees(m) | ||
| type: normalize_type(type), | ||
| mode_financement: normalize_mode_financement(mode_financement), | ||
| geom: build_geometry(geojsons, code) | ||
| }) | ||
| |> Repo.insert_or_update() | ||
| end | ||
|
|
||
| defp check_communes_list(body) do | ||
| all_communes = | ||
| body | ||
| |> Enum.map(fn epci -> | ||
| epci["membres"] |> Enum.map(& &1["code"]) | ||
| end) | ||
| |> List.flatten() | ||
|
|
||
| duplicate_communes = all_communes -- Enum.uniq(all_communes) | ||
|
|
||
| if duplicate_communes != [] do | ||
| raise "One or multiple communes belong to different EPCIs. List: #{duplicate_communes}" | ||
| end | ||
| end | ||
|
|
||
| defp update_communes_epci(%{"code" => code, "membres" => m}) do | ||
| communes_arr = get_insees(m) | ||
| communes = Repo.all(from(c in Commune, where: c.insee in ^communes_arr)) | ||
|
|
||
| communes | ||
| |> Enum.each(fn commune -> | ||
| commune | ||
| |> Changeset.change(epci_insee: code) | ||
| |> Repo.update() | ||
| end) | ||
|
|
||
| :ok | ||
| end | ||
|
|
||
| @spec get_insees([map()]) :: [binary()] | ||
| defp get_insees(members) do | ||
| members | ||
| |> Enum.map(fn m -> m["code"] end) | ||
| end | ||
|
|
||
| defp geojson_by_insee do | ||
| %{status: 200, body: body} = | ||
| Req.get!(@epci_geojson_url, connect_options: [timeout: 15_000], receive_timeout: 15_000) | ||
|
|
||
| body | ||
| # Req doesn’t decode GeoJSON body automatically as it does for JSON | ||
| |> Jason.decode!() | ||
| |> Map.fetch!("features") | ||
| |> Map.new(fn record -> {record["properties"]["code"], record["geometry"]} end) | ||
| end | ||
|
|
||
| defp build_geometry(geojsons, insee) do | ||
| {:ok, geom} = Geo.PostGIS.Geometry.cast(Map.fetch!(geojsons, insee)) | ||
| %{geom | srid: 4326} | ||
| end | ||
|
|
||
| defp ensure_valid_geometries, | ||
| do: Repo.query!("UPDATE epci SET geom = ST_MakeValid(geom) WHERE NOT ST_IsValid(geom);") | ||
|
|
||
| @spec normalize_type(binary()) :: binary() | ||
| defp normalize_type("CA"), do: "Communauté d'agglomération" | ||
| defp normalize_type("CU"), do: "Communauté urbaine" | ||
| defp normalize_type("CC"), do: "Communauté de communes" | ||
| defp normalize_type("METRO"), do: "Métropole" | ||
| defp normalize_type("MET69"), do: "Métropole de Lyon" | ||
|
|
||
| @spec normalize_mode_financement(binary()) :: binary() | ||
| defp normalize_mode_financement("FPU"), do: "Fiscalité professionnelle unique" | ||
| defp normalize_mode_financement("FA"), do: "Fiscalité additionnelle" | ||
| end | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
39 changes: 39 additions & 0 deletions
39
apps/transport/priv/repo/migrations/20231222145809_improve_epci.exs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| defmodule DB.Repo.Migrations.ImproveEPCI do | ||
| use Ecto.Migration | ||
|
|
||
| def change do | ||
| rename(table(:epci), :code, to: :insee) | ||
| create(unique_index(:epci, [:insee])) | ||
|
|
||
| alter table(:commune) do | ||
| add(:epci_insee, references(:epci, column: :insee, type: :string, on_delete: :nilify_all), null: true) | ||
| end | ||
|
|
||
| create(index(:commune, [:epci_insee])) | ||
|
|
||
| # Migrate data from epci communes_insee array column to epci_insee column in commune table | ||
| execute( | ||
| """ | ||
| UPDATE commune | ||
| SET epci_insee = epci.insee | ||
| FROM epci | ||
| WHERE commune.insee = ANY(epci.communes_insee) | ||
| """, | ||
| """ | ||
| UPDATE epci | ||
| SET communes_insee = ARRAY( | ||
| SELECT insee | ||
| FROM commune | ||
| WHERE commune.epci_insee = epci.insee | ||
| ) | ||
| """ | ||
| ) | ||
|
|
||
| alter table(:epci) do | ||
| add(:type, :string) | ||
| add(:mode_financement, :string) | ||
| add(:geom, :geometry) | ||
| remove(:communes_insee, {:array, :string}, default: []) | ||
| end | ||
| end | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters