diff --git a/.github/workflows/ops_tests.yml b/.github/workflows/ops_tests.yml new file mode 100644 index 0000000000..a0af9d8663 --- /dev/null +++ b/.github/workflows/ops_tests.yml @@ -0,0 +1,14 @@ +name: CI ops tests +on: push + +jobs: + test: + runs-on: ubuntu-latest + name: Run ops tests + steps: + - uses: actions/checkout@v4 + - uses: erlef/setup-beam@v1 + with: + version-file: .tool-versions + version-type: strict + - run: elixir ops_tests/ops_tests.exs diff --git a/apps/transport/lib/jobs/import_dataset_monthly_metrics_job.ex b/apps/transport/lib/jobs/import_dataset_monthly_metrics_job.ex index 73dc1c4a8e..ab8b100787 100644 --- a/apps/transport/lib/jobs/import_dataset_monthly_metrics_job.ex +++ b/apps/transport/lib/jobs/import_dataset_monthly_metrics_job.ex @@ -69,16 +69,18 @@ defmodule Transport.Jobs.ImportDatasetMonthlyMetricsJob do dataset_datagouv_id ) do Enum.each([{:views, monthly_visit}, {:downloads, monthly_download_resource}], fn {metric_name, count} -> + count = count || 0 + %DB.DatasetMonthlyMetric{} |> DB.DatasetMonthlyMetric.changeset(%{ dataset_datagouv_id: dataset_datagouv_id, year_month: metric_month, metric_name: metric_name, - count: count || 0 + count: count }) |> DB.Repo.insert!( conflict_target: [:dataset_datagouv_id, :year_month, :metric_name], - on_conflict: {:replace, [:count]} + on_conflict: [set: [count: count, updated_at: DateTime.utc_now()]] ) end) end diff --git a/apps/transport/test/transport/jobs/import_dataset_monthly_metrics_job_test.exs b/apps/transport/test/transport/jobs/import_dataset_monthly_metrics_job_test.exs index 20114f6dd7..44e7fc5c34 100644 --- a/apps/transport/test/transport/jobs/import_dataset_monthly_metrics_job_test.exs +++ b/apps/transport/test/transport/jobs/import_dataset_monthly_metrics_job_test.exs @@ -103,7 +103,9 @@ defmodule Transport.Test.Transport.Jobs.ImportDatasetMonthlyMetricsTestJob do dataset_datagouv_id: ^datagouv_id, year_month: "2023-12", metric_name: :views, - count: 1337 + count: 1337, + inserted_at: inserted_at, + updated_at: updated_at }, # Has been inserted %DB.DatasetMonthlyMetric{ @@ -113,6 +115,9 @@ defmodule Transport.Test.Transport.Jobs.ImportDatasetMonthlyMetricsTestJob do count: 43 } ] = DB.Repo.all(DB.DatasetMonthlyMetric) + + # `updated_at` has been updated to reflect that this row has changed + assert DateTime.after?(updated_at, inserted_at) end end diff --git a/ops_tests/ops_tests.exs b/ops_tests/ops_tests.exs index a01d5d563f..fd27355d4e 100644 --- a/ops_tests/ops_tests.exs +++ b/ops_tests/ops_tests.exs @@ -3,27 +3,21 @@ ExUnit.start() Mix.install([ - {:req, "~> 0.2.1"} + {:req, "~> 0.4.8"}, + {:dns, "~> 2.4.0"} ]) defmodule Transport.OpsTests do - use ExUnit.Case + use ExUnit.Case, async: true - def get_header!(headers, header) do - {_header, value} = - headers - |> Enum.find(fn {k, _} -> k == header end) - - value - end - - def assert_redirect(from: url, to: target_url) do - %{status: 301, headers: headers} = - Req.build(:get, url) - |> Req.run!() - - assert get_header!(headers, "location") == target_url - end + # See https://developers.clever-cloud.com/doc/administrate/domain-names/#your-application-runs-in-the-europeparis-par-zone + @domain_name "transport.data.gouv.fr" + @clever_cloud_ip_addresses [ + {46, 252, 181, 103}, + {46, 252, 181, 104}, + {185, 42, 117, 108}, + {185, 42, 117, 109} + ] test "correct DOMAIN_NAME for prod-worker" do assert_redirect( @@ -38,4 +32,74 @@ defmodule Transport.OpsTests do to: "https://workers.prochainement.transport.data.gouv.fr/" ) end + + test "redirects from www to non-www" do + assert_redirect(from: "https://www.#{@domain_name}", to: "https://#{@domain_name}/") + end + + describe "Check DNS records" do + test "main A/CNAME records" do + {:ok, ips} = DNS.resolve(@domain_name, :a) + assert MapSet.new(ips) == MapSet.new(@clever_cloud_ip_addresses) + + # CNAMEs to Clever Cloud + [ + "prochainement", + "proxy", + "proxy.prochainement", + "validation", + "workers", + "workers.prochainement", + "www" + ] + |> Enum.each(fn subdomain -> + record = "#{subdomain}.#{@domain_name}" + assert {:ok, [~c"domain.par.clever-cloud.com"]} == DNS.resolve(record, :cname), "Wrong DNS record for #{record}" + end) + + # Satellite websites + assert {:ok, [~c"transport-blog.netlify.app"]} == DNS.resolve("blog.#{@domain_name}", :cname) + assert {:ok, [~c"transport-contribuer.netlify.app"]} == DNS.resolve("contribuer.#{@domain_name}", :cname) + assert {:ok, [~c"hosting.gitbook.com"]} == DNS.resolve("doc.#{@domain_name}", :cname) + assert {:ok, [~c"stats.uptimerobot.com"]} == DNS.resolve("status.#{@domain_name}", :cname) + end + + test "MX records" do + {:ok, records} = DNS.resolve(@domain_name, :mx) + assert MapSet.new([{10, ~c"mx1.alwaysdata.com"}, {20, ~c"mx2.alwaysdata.com"}]) == MapSet.new(records) + assert {:ok, [{100, ~c"mx.sendgrid.net"}]} = DNS.resolve("front-mail.#{@domain_name}", :mx) + end + + test "SPF, DKIM and DMARC" do + # SPF + {:ok, records} = DNS.resolve(@domain_name, :txt) + + assert Enum.member?(records, [ + ~c"v=spf1 include:spf.mailjet.com include:_spf.alwaysdata.com include:_spf.scw-tem.cloud include:servers.mcsv.net -all" + ]) + + assert {:ok, [[~c"v=spf1 include:sendgrid.net ~all"]]} = DNS.resolve("front-mail.#{@domain_name}", :txt) + + # DKIM + assert {:ok, _} = DNS.resolve("37d278a7-e548-4029-a58d-111bdcf23d46._domainkey.#{@domain_name}", :txt) + assert {:ok, _} = DNS.resolve("default._domainkey.#{@domain_name}", :txt) + assert {:ok, _} = DNS.resolve("fnt._domainkey.#{@domain_name}", :txt) + assert {:ok, _} = DNS.resolve("mailjet._domainkey.#{@domain_name}", :txt) + assert {:ok, [~c"dkim2.mcsv.net"]} == DNS.resolve("k2._domainkey.#{@domain_name}", :cname) + assert {:ok, [~c"dkim3.mcsv.net"]} == DNS.resolve("k3._domainkey.#{@domain_name}", :cname) + + # DMARC + assert {:ok, [[~c"v=DMARC1;p=quarantine;"]]} == DNS.resolve("_dmarc.#{@domain_name}", :txt) + end + end + + def get_header!(headers, header) do + {_header, [value]} = Enum.find(headers, fn {k, _} -> k == header end) + value + end + + def assert_redirect(from: url, to: target_url) do + %Req.Response{status: 301, headers: headers} = Req.get!(url, redirect: false) + assert get_header!(headers, "location") == target_url + end end