Merge branch 'master' into search-spike

.github/workflows/ops_tests.yml

@@ -0,0 +1,14 @@
+name: CI ops tests
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Run ops tests
+    steps:
+      - uses: actions/checkout@v4
+      - uses: erlef/setup-beam@v1
+        with:
+          version-file: .tool-versions
+          version-type: strict
+      - run: elixir ops_tests/ops_tests.exs

apps/transport/lib/jobs/import_dataset_monthly_metrics_job.ex

@@ -69,16 +69,18 @@ defmodule Transport.Jobs.ImportDatasetMonthlyMetricsJob do
          dataset_datagouv_id
        ) do
     Enum.each([{:views, monthly_visit}, {:downloads, monthly_download_resource}], fn {metric_name, count} ->
+      count = count || 0
+
       %DB.DatasetMonthlyMetric{}
       |> DB.DatasetMonthlyMetric.changeset(%{
         dataset_datagouv_id: dataset_datagouv_id,
         year_month: metric_month,
         metric_name: metric_name,
-        count: count || 0
+        count: count
       })
       |> DB.Repo.insert!(
         conflict_target: [:dataset_datagouv_id, :year_month, :metric_name],
-        on_conflict: {:replace, [:count]}
+        on_conflict: [set: [count: count, updated_at: DateTime.utc_now()]]
       )
     end)
   end

apps/transport/lib/jobs/resource_history_job.ex

@@ -288,7 +288,7 @@ defmodule Transport.Jobs.ResourceHistoryJob do
         {:error, "Got a non 200 status: #{status}"}
 
       {:error, %HTTPoison.Error{reason: reason}} ->
-        {:error, "Got an error: #{reason}"}
+        {:error, "Got an error: #{inspect(reason)}"}
     end
   end
 

apps/transport/lib/transport/gtfs_rt.ex

@@ -26,7 +26,7 @@ defmodule Transport.GTFSRT do
         {:error, "Got a non 200 HTTP status code: #{status_code}"}
 
       {:error, %HTTPoison.Error{reason: reason}} ->
-        {:error, "Got an HTTP error: #{reason}"}
+        {:error, "Got an HTTP error: #{inspect(reason)}"}
     end
   end
 

apps/transport/lib/transport_web/live/siri_querier_live.ex

@@ -106,7 +106,7 @@ defmodule TransportWeb.Live.SIRIQuerierLive do
           })
 
         {:error, %HTTPoison.Error{reason: reason}} ->
-          socket |> assign(siri_response_error: reason, siri_response_status_code: nil)
+          socket |> assign(siri_response_error: inspect(reason), siri_response_status_code: nil)
       end
 
     {:noreply, socket}

apps/transport/test/transport/gtfs_rt_test.exs

@@ -39,9 +39,15 @@ defmodule Transport.GTFSRTTest do
     end
 
     test "HTTPoison error" do
-      setup_http_response(@url, {:error, %HTTPoison.Error{reason: "SSL problemz"}})
+      reason =
+        {:tls_alert,
+         {:certificate_expired,
+          ~c"TLS client: In state certify at ssl_handshake.erl:2065 generated CLIENT ALERT: Fatal - Certificate Expired\n"}}
 
-      assert {:error, "Got an HTTP error: SSL problemz"} == GTFSRT.decode_remote_feed(@url)
+      setup_http_response(@url, {:error, %HTTPoison.Error{reason: reason}})
+
+      {:error, reason} = GTFSRT.decode_remote_feed(@url)
+      assert reason =~ ~r"^Got an HTTP error:"
     end
   end
 

apps/transport/test/transport/jobs/import_dataset_monthly_metrics_job_test.exs

@@ -103,7 +103,9 @@ defmodule Transport.Test.Transport.Jobs.ImportDatasetMonthlyMetricsTestJob do
                  dataset_datagouv_id: ^datagouv_id,
                  year_month: "2023-12",
                  metric_name: :views,
-                 count: 1337
+                 count: 1337,
+                 inserted_at: inserted_at,
+                 updated_at: updated_at
                },
                # Has been inserted
                %DB.DatasetMonthlyMetric{
@@ -113,6 +115,9 @@ defmodule Transport.Test.Transport.Jobs.ImportDatasetMonthlyMetricsTestJob do
                  count: 43
                }
              ] = DB.Repo.all(DB.DatasetMonthlyMetric)
+
+      # `updated_at` has been updated to reflect that this row has changed
+      assert DateTime.after?(updated_at, inserted_at)
     end
   end
 

ops_tests/ops_tests.exs

@@ -3,27 +3,21 @@
 ExUnit.start()
 
 Mix.install([
-  {:req, "~> 0.2.1"}
+  {:req, "~> 0.4.8"},
+  {:dns, "~> 2.4.0"}
 ])
 
 defmodule Transport.OpsTests do
-  use ExUnit.Case
+  use ExUnit.Case, async: true
 
-  def get_header!(headers, header) do
-    {_header, value} =
-      headers
-      |> Enum.find(fn {k, _} -> k == header end)
-
-    value
-  end
-
-  def assert_redirect(from: url, to: target_url) do
-    %{status: 301, headers: headers} =
-      Req.build(:get, url)
-      |> Req.run!()
-
-    assert get_header!(headers, "location") == target_url
-  end
+  # See https://developers.clever-cloud.com/doc/administrate/domain-names/#your-application-runs-in-the-europeparis-par-zone
+  @domain_name "transport.data.gouv.fr"
+  @clever_cloud_ip_addresses [
+    {46, 252, 181, 103},
+    {46, 252, 181, 104},
+    {185, 42, 117, 108},
+    {185, 42, 117, 109}
+  ]
 
   test "correct DOMAIN_NAME for prod-worker" do
     assert_redirect(
@@ -38,4 +32,74 @@ defmodule Transport.OpsTests do
       to: "https://workers.prochainement.transport.data.gouv.fr/"
     )
   end
+
+  test "redirects from www to non-www" do
+    assert_redirect(from: "https://www.#{@domain_name}", to: "https://#{@domain_name}/")
+  end
+
+  describe "Check DNS records" do
+    test "main A/CNAME records" do
+      {:ok, ips} = DNS.resolve(@domain_name, :a)
+      assert MapSet.new(ips) == MapSet.new(@clever_cloud_ip_addresses)
+
+      # CNAMEs to Clever Cloud
+      [
+        "prochainement",
+        "proxy",
+        "proxy.prochainement",
+        "validation",
+        "workers",
+        "workers.prochainement",
+        "www"
+      ]
+      |> Enum.each(fn subdomain ->
+        record = "#{subdomain}.#{@domain_name}"
+        assert {:ok, [~c"domain.par.clever-cloud.com"]} == DNS.resolve(record, :cname), "Wrong DNS record for #{record}"
+      end)
+
+      # Satellite websites
+      assert {:ok, [~c"transport-blog.netlify.app"]} == DNS.resolve("blog.#{@domain_name}", :cname)
+      assert {:ok, [~c"transport-contribuer.netlify.app"]} == DNS.resolve("contribuer.#{@domain_name}", :cname)
+      assert {:ok, [~c"hosting.gitbook.com"]} == DNS.resolve("doc.#{@domain_name}", :cname)
+      assert {:ok, [~c"stats.uptimerobot.com"]} == DNS.resolve("status.#{@domain_name}", :cname)
+    end
+
+    test "MX records" do
+      {:ok, records} = DNS.resolve(@domain_name, :mx)
+      assert MapSet.new([{10, ~c"mx1.alwaysdata.com"}, {20, ~c"mx2.alwaysdata.com"}]) == MapSet.new(records)
+      assert {:ok, [{100, ~c"mx.sendgrid.net"}]} = DNS.resolve("front-mail.#{@domain_name}", :mx)
+    end
+
+    test "SPF, DKIM and DMARC" do
+      # SPF
+      {:ok, records} = DNS.resolve(@domain_name, :txt)
+
+      assert Enum.member?(records, [
+               ~c"v=spf1 include:spf.mailjet.com include:_spf.alwaysdata.com include:_spf.scw-tem.cloud include:servers.mcsv.net -all"
+             ])
+
+      assert {:ok, [[~c"v=spf1 include:sendgrid.net ~all"]]} = DNS.resolve("front-mail.#{@domain_name}", :txt)
+
+      # DKIM
+      assert {:ok, _} = DNS.resolve("37d278a7-e548-4029-a58d-111bdcf23d46._domainkey.#{@domain_name}", :txt)
+      assert {:ok, _} = DNS.resolve("default._domainkey.#{@domain_name}", :txt)
+      assert {:ok, _} = DNS.resolve("fnt._domainkey.#{@domain_name}", :txt)
+      assert {:ok, _} = DNS.resolve("mailjet._domainkey.#{@domain_name}", :txt)
+      assert {:ok, [~c"dkim2.mcsv.net"]} == DNS.resolve("k2._domainkey.#{@domain_name}", :cname)
+      assert {:ok, [~c"dkim3.mcsv.net"]} == DNS.resolve("k3._domainkey.#{@domain_name}", :cname)
+
+      # DMARC
+      assert {:ok, [[~c"v=DMARC1;p=quarantine;"]]} == DNS.resolve("_dmarc.#{@domain_name}", :txt)
+    end
+  end
+
+  def get_header!(headers, header) do
+    {_header, [value]} = Enum.find(headers, fn {k, _} -> k == header end)
+    value
+  end
+
+  def assert_redirect(from: url, to: target_url) do
+    %Req.Response{status: 301, headers: headers} = Req.get!(url, redirect: false)
+    assert get_header!(headers, "location") == target_url
+  end
 end