diff --git a/apps/transport/lib/registry/engine.ex b/apps/transport/lib/registry/engine.ex index a677c36c48..b7a8b446b0 100644 --- a/apps/transport/lib/registry/engine.ex +++ b/apps/transport/lib/registry/engine.ex @@ -5,6 +5,7 @@ defmodule Transport.Registry.Engine do alias Transport.Registry.GTFS alias Transport.Registry.NeTEx + alias Transport.Registry.Model.DataSource alias Transport.Registry.Model.Stop alias Transport.Registry.Result @@ -45,15 +46,17 @@ defmodule Transport.Registry.Engine do end def prepare_extractor(%DB.Resource{} = resource) do + data_source_id = "PAN:resource:#{resource.id}" + case resource.format do - "GTFS" -> {:ok, {GTFS, resource.url}} - "NeTEx" -> {:ok, {NeTEx, resource.url}} + "GTFS" -> {:ok, {GTFS, data_source_id, resource.url}} + "NeTEx" -> {:ok, {NeTEx, data_source_id, resource.url}} _ -> {:error, "Unsupported format"} end end - def download({extractor, url}) do - Logger.debug("download #{extractor} #{url}") + def download({extractor, data_source_id, url}) do + Logger.debug("download #{extractor} #{data_source_id} #{url}") tmp_path = System.tmp_dir!() |> Path.join("#{Ecto.UUID.generate()}.dat") safe_error = fn msg -> @@ -75,7 +78,7 @@ defmodule Transport.Registry.Engine do {:ok, %{status: status}} -> cond do status >= 200 && status < 300 -> - {:ok, {extractor, tmp_path}} + {:ok, {extractor, data_source_id, tmp_path}} status > 400 -> safe_error.("Error #{status} while downloading the resource from #{url}") @@ -86,10 +89,10 @@ defmodule Transport.Registry.Engine do end end - @spec extract_from_archive({module(), Path.t()}) :: Result.t([Stop.t()]) - def extract_from_archive({extractor, file}) do - Logger.debug("extract_from_archive #{extractor} #{file}") - extractor.extract_from_archive(file) + @spec extract_from_archive({module(), DataSource.data_source_id(), Path.t()}) :: Result.t([Stop.t()]) + def extract_from_archive({extractor, data_source_id, file}) do + Logger.debug("extract_from_archive #{extractor} #{data_source_id} #{file}") + extractor.extract_from_archive(data_source_id, file) end def dump_to_csv(enumerable, output_file) do diff --git a/apps/transport/lib/registry/extractor.ex b/apps/transport/lib/registry/extractor.ex index 0f5db7d9ec..fa5f0da5a5 100644 --- a/apps/transport/lib/registry/extractor.ex +++ b/apps/transport/lib/registry/extractor.ex @@ -5,8 +5,10 @@ defmodule Transport.Registry.Extractor do require Logger + alias Transport.Registry.Model.DataSource alias Transport.Registry.Model.Stop alias Transport.Registry.Result - @callback extract_from_archive(path :: Path.t()) :: Result.t([Stop.t()]) + @callback extract_from_archive(data_source_id :: DataSource.data_source_id(), path :: Path.t()) :: + Result.t([Stop.t()]) end diff --git a/apps/transport/lib/registry/gtfs.ex b/apps/transport/lib/registry/gtfs.ex index 80c875e0ee..7f558651ff 100644 --- a/apps/transport/lib/registry/gtfs.ex +++ b/apps/transport/lib/registry/gtfs.ex @@ -15,7 +15,7 @@ defmodule Transport.Registry.GTFS do @doc """ Extract stops from GTFS ressource. """ - def extract_from_archive(archive) do + def extract_from_archive(data_source_id, archive) do case file_stream(archive) do {:error, error} -> Logger.error(error) @@ -26,13 +26,13 @@ defmodule Transport.Registry.GTFS do content |> Utils.to_stream_of_maps() - |> Stream.flat_map(&handle_stop/1) + |> Stream.flat_map(&handle_stop(data_source_id, &1)) |> Enum.to_list() |> Result.ok() end end - defp handle_stop(record) do + defp handle_stop(data_source_id, record) do latitude = Utils.fetch_position(record, "stop_lat") longitude = Utils.fetch_position(record, "stop_lon") @@ -44,7 +44,9 @@ defmodule Transport.Registry.GTFS do latitude: latitude, longitude: longitude, projection: :utm_wgs84, - stop_type: record |> Utils.csv_get_with_default("location_type", "0") |> to_stop_type() + stop_type: record |> Utils.csv_get_with_default("location_type", "0") |> to_stop_type(), + data_source_format: :gtfs, + data_source_id: data_source_id } ] else diff --git a/apps/transport/lib/registry/netex.ex b/apps/transport/lib/registry/netex.ex index 248c4619f7..aa8dcf321d 100644 --- a/apps/transport/lib/registry/netex.ex +++ b/apps/transport/lib/registry/netex.ex @@ -13,10 +13,7 @@ defmodule Transport.Registry.NeTEx do @doc """ Extract stops from a NeTEx archive. """ - def extract_from_archive(archive) do - # FIXME: propagate some context - data_source_id = nil - + def extract_from_archive(data_source_id, archive) do archive |> Transport.NeTEx.read_all_stop_places() |> Enum.flat_map(&process_stop_places(data_source_id, &1))