From 01d673df174337587cdd36b3232150a376fdb7be Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Thu, 16 Nov 2023 10:28:05 +0100 Subject: [PATCH] =?UTF-8?q?ConsolidateBNLCJob=20:=20r=C3=A9pare=20d=C3=A9c?= =?UTF-8?q?odage=20fichier=20BNLC=20(#3598)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ConsolidateBNLCJob : répare décodage fichier BNLC * Tests: add id_lieu in other files to prove it's ignored --- .../transport/lib/jobs/consolidate_bnlc_job.ex | 11 +++++------ .../jobs/consolidate_bnlc_job_test.exs | 18 +++++++++--------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/apps/transport/lib/jobs/consolidate_bnlc_job.ex b/apps/transport/lib/jobs/consolidate_bnlc_job.ex index ad918adb9f..29e798e41b 100644 --- a/apps/transport/lib/jobs/consolidate_bnlc_job.ex +++ b/apps/transport/lib/jobs/consolidate_bnlc_job.ex @@ -247,11 +247,6 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do |> Stream.take(1) |> Enum.to_list() |> hd() - # In the 0.2.0 schema version the `id_lieu` column was present. - # https://schema.data.gouv.fr/etalab/schema-lieux-covoiturage/0.2.8/documentation.html - # Starting with 0.3.0 `id_lieu` should not be present in the files - # we consolidate as we add it ourselves with `add_id_lieu_column/1` - |> Enum.reject(&(&1 == "id_lieu")) end @doc """ @@ -263,7 +258,11 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do def consolidate_resources(resources_details) do file = File.open!(@bnlc_path, [:write, :utf8]) bnlc_headers = bnlc_csv_headers() - final_headers = ["id_lieu"] ++ bnlc_headers + # In the 0.2.8 schema version the `id_lieu` column was present. + # https://schema.data.gouv.fr/etalab/schema-lieux-covoiturage/0.2.8/documentation.html + # Starting with 0.3.0 `id_lieu` should not be present in the files + # we consolidate as we add it ourselves with `add_id_lieu_column/1` + final_headers = ["id_lieu"] ++ Enum.reject(bnlc_headers, &(&1 == "id_lieu")) %HTTPoison.Response{body: body, status_code: 200} = @bnlc_github_url |> http_client().get!() diff --git a/apps/transport/test/transport/jobs/consolidate_bnlc_job_test.exs b/apps/transport/test/transport/jobs/consolidate_bnlc_job_test.exs index a41caa9260..39070f2ce5 100644 --- a/apps/transport/test/transport/jobs/consolidate_bnlc_job_test.exs +++ b/apps/transport/test/transport/jobs/consolidate_bnlc_job_test.exs @@ -299,7 +299,7 @@ defmodule Transport.Test.Transport.Jobs.ConsolidateBNLCJobTest do end ) - assert ["foo", "bar", "baz"] == ConsolidateBNLCJob.bnlc_csv_headers() + assert ["foo", "bar", "baz", "id_lieu"] == ConsolidateBNLCJob.bnlc_csv_headers() end test "consolidate_resources" do @@ -339,9 +339,9 @@ defmodule Transport.Test.Transport.Jobs.ConsolidateBNLCJobTest do Transport.HTTPoison.Mock |> expect(:get, fn ^other_url, [], [follow_redirect: true] -> body = """ - "foo";"bar";"baz";"insee";"id_local";"extra_col" - "a";"b";"c";"21231";"3";"its_a_trap" - "d";"e";"f";"21231";"4";"should_be_ignored" + "foo";"bar";"baz";"insee";"id_local";"extra_col";"id_lieu" + "a";"b";"c";"21231";"3";"its_a_trap";"not_falling_for_this" + "d";"e";"f";"21231";"4";"should_be_ignored";"cant_mess_with_me" """ {:ok, %HTTPoison.Response{status_code: 200, body: body}} @@ -356,9 +356,9 @@ defmodule Transport.Test.Transport.Jobs.ConsolidateBNLCJobTest do 2, fn "https://raw.githubusercontent.com/etalab/transport-base-nationale-covoiturage/main/bnlc-.csv" -> body = """ - "foo","bar","baz","insee","id_local" - I,Love,CSV,21231,5 - Very,Much,So,21231,6 + "foo","bar","baz","id_lieu","insee","id_local" + I,Love,CSV,3,21231,5 + Very,Much,So,4,21231,6 """ %HTTPoison.Response{status_code: 200, body: body} @@ -519,8 +519,8 @@ defmodule Transport.Test.Transport.Jobs.ConsolidateBNLCJobTest do Transport.HTTPoison.Mock |> expect(:get, fn ^bar_url, [], [follow_redirect: true] -> body = """ - foo,bar,baz,insee,id_local,extra_col - 1,2,3,21231,3,is_ignored + foo,bar,baz,insee,id_local,extra_col,id_lieu + 1,2,3,21231,3,is_ignored,is_generated_again """ {:ok, %HTTPoison.Response{status_code: 200, body: body}}