Automate picking the OS versions (#182)

Co-authored-by: Eric Meadows-Jönsson <[email protected]>

lib/bob/docker_hub.ex

@@ -17,6 +17,11 @@ defmodule Bob.DockerHub do
   end
 
   def fetch_repo_tags(repo) do
+    (@dockerhub_url <> "v2/repositories/#{repo}/tags?page=${page}&page_size=100")
+    |> dockerhub_request()
+  end
+
+  def fetch_repo_tags_from_cache(repo) do
     Bob.DockerHub.Cache.lookup(repo, fn ->
       (@dockerhub_url <> "v2/repositories/#{repo}/tags?page=${page}&page_size=100")
       |> dockerhub_request()

lib/bob/job/docker_checker.ex

@@ -4,33 +4,59 @@ defmodule Bob.Job.DockerChecker do
 
   @archs ["amd64", "arm64"]
 
-  # TODO: Automate picking the OS versions
-
-  @builds %{
-    "alpine" => [
-      "3.19.1",
-      "3.18.6",
-      "3.17.7",
-      "3.16.9"
-    ],
-    "ubuntu" => [
-      # 22.04
-      "jammy-20240125",
-      # 20.04
-      "focal-20240123"
-    ],
-    "debian" => [
-      # 12
-      "bookworm-20240130",
-      "bookworm-20240130-slim",
-      # 11
-      "bullseye-20240130",
-      "bullseye-20240130-slim",
-      # 10
-      "buster-20240130",
-      "buster-20240130-slim"
+  def builds() do
+    [
+      {"alpine",
+       [
+         ~r/^3\.19\.\d+$/,
+         ~r/^3\.18\.\d+$/,
+         ~r/^3\.17\.\d+$/,
+         ~r/^3\.16\.\d+$/
+       ]},
+      {"ubuntu",
+       [
+         # 24.04
+         ~r/^noble-\d{8}$/,
+         # 22.04
+         ~r/^jammy-\d{8}$/,
+         # 20.04
+         ~r/^focal-\d{8}$/
+       ]},
+      {"debian",
+       [
+         # 12
+         ~r/^bookworm-\d{8}$/,
+         ~r/^bookworm-\d{8}-slim$/,
+         # 11
+         ~r/^bullseye-\d{8}$/,
+         ~r/^bullseye-\d{8}-slim$/,
+         # 10
+         ~r/^buster-\d{8}$/,
+         ~r/^buster-\d{8}-slim$/
+       ]}
     ]
-  }
+    |> Task.async_stream(
+      fn {repo, regexes} ->
+        {repo, tags(repo, regexes)}
+      end,
+      ordered: false,
+      timeout: 300_000
+    )
+    |> Enum.map(fn {:ok, repo_and_tags} -> repo_and_tags end)
+    |> Map.new()
+  end
+
+  defp tags(repo, regexes) do
+    tags =
+      ("library/" <> repo)
+      |> Bob.DockerHub.fetch_repo_tags()
+      |> Enum.map(&elem(&1, 0))
+      |> Enum.sort(&(&1 >= &2))
+
+    Enum.map(regexes, fn regex ->
+      Enum.find(tags, &(&1 =~ regex))
+    end)
+  end
 
   def run() do
     erlang()
@@ -58,7 +84,7 @@ defmodule Bob.Job.DockerChecker do
   def expected_erlang_tags() do
     refs = erlang_refs()
 
-    Stream.flat_map(@builds, fn {os, os_versions} ->
+    Stream.flat_map(builds(), fn {os, os_versions} ->
       Stream.flat_map(refs, fn ref ->
         if build_erlang_ref?(os, ref) do
           Stream.flat_map(os_versions, fn os_version ->
@@ -206,7 +232,7 @@ defmodule Bob.Job.DockerChecker do
 
   def erlang_tags(arch) do
     "hexpm/erlang-#{arch}"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Stream.map(fn {tag, [^arch]} ->
       [erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
       {erlang, os, os_version, arch}
@@ -223,10 +249,11 @@ defmodule Bob.Job.DockerChecker do
   end
 
   def expected_elixir_tags() do
+    builds = builds()
     refs = elixir_builds()
 
     Stream.flat_map(erlang_tags(), fn {erlang, os, os_version, erlang_arch} ->
-      if not skip_elixir_for_erlang?(erlang) and os_version in @builds[os] do
+      if not skip_elixir_for_erlang?(erlang) and os_version in builds[os] do
         Stream.flat_map(refs, fn {"v" <> elixir, otp_major} ->
           if not skip_elixir?(elixir) and compatible_elixir_and_erlang?(otp_major, erlang) do
             [{elixir, erlang, os, os_version, erlang_arch}]
@@ -279,7 +306,7 @@ defmodule Bob.Job.DockerChecker do
 
   def elixir_tags(arch) do
     "hexpm/elixir-#{arch}"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Enum.map(fn {tag, [^arch]} ->
       [elixir, erlang, os, os_version] =
         Regex.run(@elixir_tag_regex, tag, capture: :all_but_first)
@@ -355,7 +382,7 @@ defmodule Bob.Job.DockerChecker do
 
   def erlang_manifest_tags() do
     "hexpm/erlang"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Map.new(fn {tag, archs} ->
       [erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
       {{erlang, os, os_version}, archs}
@@ -364,7 +391,7 @@ defmodule Bob.Job.DockerChecker do
 
   def elixir_manifest_tags() do
     "hexpm/elixir"
-    |> Bob.DockerHub.fetch_repo_tags()
+    |> Bob.DockerHub.fetch_repo_tags_from_cache()
     |> Map.new(fn {tag, archs} ->
       [elixir, erlang, os, os_version] =
         Regex.run(@elixir_tag_regex, tag, capture: :all_but_first)