chore: move sitemap_index.xml and robots.txt to top-level host (#746)

config/test.exs

@@ -28,13 +28,15 @@ config :beacon, Beacon.BeaconTest.ProxyEndpoint,
   url: [port: 80],
   secret_key_base: "dVxFbSNspBVvkHPN5m6FE6iqNtMnhrmPNw7mO57CJ6beUADllH0ux3nhAI1ic65X",
   live_view: [signing_salt: "LKBurgGF"],
-  check_origin: false
+  check_origin: false,
+  render_errors: [formats: [html: Beacon.Web.ErrorHTML]],
+  debug_errors: true
 
 config :beacon, Beacon.BeaconTest.Endpoint,
   url: [host: "site_a.com", port: 4001],
   secret_key_base: "dVxFbSNspBVvkHPN5m6FE6iqNtMnhrmPNw7mO57CJ6beUADllH0ux3nhAI1ic65X",
   live_view: [signing_salt: "LKBurgGF"],
-  render_errors: [view: Beacon.BeaconTest.ErrorView],
+  render_errors: [formats: [html: Beacon.Web.ErrorHTML]],
   pubsub_server: Beacon.BeaconTest.PubSub,
   check_origin: false,
   debug_errors: true
@@ -43,7 +45,7 @@ config :beacon, Beacon.BeaconTest.EndpointB,
   url: [host: "site_b.com", port: 4002],
   secret_key_base: "dVxFbSNspBVvkHPN5m6FE6iqNtMnhrmPNw7mO57CJ6beUADllH0ux3nhAI1ic65X",
   live_view: [signing_salt: "LKBurgGF"],
-  render_errors: [view: Beacon.BeaconTest.ErrorView],
+  render_errors: [formats: [html: Beacon.Web.ErrorHTML]],
   pubsub_server: Beacon.BeaconTest.PubSub,
   check_origin: false,
   debug_errors: true

lib/beacon/proxy_endpoint.ex

@@ -16,15 +16,59 @@ defmodule Beacon.ProxyEndpoint do
       Module.put_attribute(__MODULE__, :__beacon_proxy_fallback__, fallback)
 
       use Phoenix.Endpoint, otp_app: otp_app
+      import Plug.Conn, only: [put_resp_content_type: 2, put_resp_header: 3, halt: 1]
+      import Phoenix.Controller, only: [accepts: 2, put_view: 2, render: 3]
 
       socket "/live", Phoenix.LiveView.Socket,
         websocket: [connect_info: [session: @session_options]],
         longpoll: [connect_info: [session: @session_options]]
 
+      plug :robots
+      plug :sitemap_index
       plug :proxy
 
+      defp robots(%{path_info: ["robots.txt"]} = conn, _opts) do
+        sitemap_index_url =
+          String.Chars.URI.to_string(%URI{scheme: Atom.to_string(conn.scheme), host: conn.host, port: conn.port, path: "/sitemap_index.xml"})
+
+        conn
+        |> accepts(["txt"])
+        |> put_view(Beacon.Web.RobotsTxt)
+        |> put_resp_content_type("text/txt")
+        |> put_resp_header("cache-control", "public max-age=300")
+        |> render(:robots, sitemap_index_url: sitemap_index_url)
+        |> halt()
+      end
+
+      defp robots(conn, _opts), do: conn
+
+      defp sitemap_index(%{path_info: ["sitemap_index.xml"]} = conn, _opts) do
+        sites = Beacon.ProxyEndpoint.sites_per_host(conn.host)
+
+        conn
+        |> accepts(["xml"])
+        |> put_view(Beacon.Web.SitemapXML)
+        |> put_resp_content_type("text/xml")
+        |> put_resp_header("cache-control", "public max-age=300")
+        |> render(:sitemap_index, urls: get_sitemap_urls(sites))
+        |> halt()
+      end
+
+      defp sitemap_index(conn, _opts), do: conn
+
+      defp get_sitemap_urls(sites) do
+        sites
+        |> Enum.map(fn site ->
+          routes_module = Beacon.Loader.fetch_routes_module(site)
+          Beacon.apply_mfa(site, routes_module, :public_sitemap_url, [])
+        end)
+        |> Enum.reject(&is_nil/1)
+        |> Enum.sort()
+        |> Enum.uniq()
+      end
+
       # TODO: cache endpoint resolver
-      def proxy(%{host: host} = conn, opts) do
+      defp proxy(%{host: host} = conn, opts) do
         matching_endpoint = fn ->
           Enum.reduce_while(Beacon.Registry.running_sites(), @__beacon_proxy_fallback__, fn site, default ->
             %{endpoint: endpoint} = Beacon.Config.fetch!(site)
@@ -136,4 +180,17 @@ defmodule Beacon.ProxyEndpoint do
   defp port_to_integer({:system, env_var}), do: port_to_integer(System.get_env(env_var))
   defp port_to_integer(port) when is_binary(port), do: String.to_integer(port)
   defp port_to_integer(port) when is_integer(port), do: port
+
+  @doc false
+  def sites_per_host(host) when is_binary(host) do
+    Enum.reduce(Beacon.Registry.running_sites(), [], fn site, acc ->
+      %{endpoint: endpoint} = Beacon.Config.fetch!(site)
+
+      if endpoint.host() == host do
+        [site | acc]
+      else
+        acc
+      end
+    end)
+  end
 end

lib/beacon/router.ex

@@ -91,7 +91,7 @@ defmodule Beacon.Router do
   defp prelude do
     quote do
       Module.register_attribute(__MODULE__, :beacon_sites, accumulate: true)
-      import Beacon.Router, only: [beacon_site: 2, beacon_sitemap_index: 1]
+      import Beacon.Router, only: [beacon_site: 2]
       @before_compile unquote(__MODULE__)
     end
   end
@@ -118,7 +118,8 @@ defmodule Beacon.Router do
   @doc """
   Mounts a site in the `prefix` in your host application router.
 
-  This will automatically serve `sitemap.xml` and `robots.txt` files from the `prefix` path defined for this site.
+  This will automatically serve `sitemap.xml` from the `prefix` path defined for this site,
+  and also `robots.txt` and `sitemap_index.xml` in the top-level host.
 
   ## Options
 
@@ -148,7 +149,6 @@ defmodule Beacon.Router do
           get "/__beacon_assets__/js-:md5", Beacon.Web.AssetsController, :js, assigns: %{site: opts[:site]}
 
           get "/sitemap.xml", Beacon.Web.SitemapController, :show, as: :beacon_sitemap, assigns: %{site: opts[:site]}
-          get "/robots.txt", Beacon.Web.RobotsController, :show, as: :beacon_robots, assigns: %{site: opts[:site]}
 
           # simulate a beacon page inside site prefix so we can check this site is reachable?/2
           get "/__beacon_check__", Beacon.Web.CheckController, :check, metadata: %{site: opts[:site]}
@@ -161,67 +161,6 @@ defmodule Beacon.Router do
     end
   end
 
-  @doc """
-  Creates a sitemap index at the given path (including the filename and extension).
-
-  ## Example
-
-    defmodule MyApp.Router do
-      ...
-      scope "/" do
-        pipe_through :browser
-
-        beacon_sitemap_index "/sitemap_index.xml"
-
-        beacon_site "/other", site: :other
-        beacon_site "/", site: :home
-      end
-    end
-
-  In the above example, there are two Beacon sites, so Beacon will serve two sitemaps:
-    * `my_domain.com/sitemap.xml` for site `:home`
-    * `my_domain.com/other/sitemap.xml` for site `:other`
-
-  Then Beacon will reference both of those sitemaps in the top-level index:
-    * `my_domain.com/sitemap_index.xml`
-
-  Note that `beacon_sitemap_index` will include the sitemap URL of all mounted sites
-  in the router, so that macro should be at the root and not duplicated.
-
-  ## Requirements
-
-  Note that your sitemap index cannot have a path which is "deeper" in the directory structure than
-  your Beacon sites (which will be contained in the index).
-
-  For example, the following is NOT allowed:
-
-    scope "/" do
-      ...
-      beacon_sitemap_index "/root/nested/sitemap_index.xml"
-
-      beacon_site "/root", site: :root
-    end
-
-  However, the opposite case (nesting the sites deeper than the index) is perfectly fine:
-
-    scope "/" do
-      ...
-      beacon_sitemap_index "/sitemap_index.xml"
-
-      beacon_site "/nested/path/to/site", site: :nested
-    end
-
-  """
-  defmacro beacon_sitemap_index(path_with_filename) do
-    quote bind_quoted: binding(), location: :keep, generated: true do
-      import Phoenix.Router, only: [scope: 3, get: 4]
-
-      scope "/", alias: false, as: false do
-        get path_with_filename, Beacon.Web.SitemapController, :index, as: :beacon_sitemap
-      end
-    end
-  end
-
   @doc false
   @spec __options__(keyword()) :: {atom(), atom(), keyword()}
   def __options__(opts) do

lib/beacon/web/controllers/sitemap_controller.ex

@@ -2,18 +2,7 @@ defmodule Beacon.Web.SitemapController do
   @moduledoc false
   use Beacon.Web, :controller
 
-  def init(action) when action in [:index, :show], do: action
-
-  def call(conn, :index) do
-    sites = Beacon.Private.router(conn).__beacon_sites__()
-
-    conn
-    |> accepts(["xml"])
-    |> put_view(Beacon.Web.SitemapXML)
-    |> put_resp_content_type("text/xml")
-    |> put_resp_header("cache-control", "public max-age=300")
-    |> render(:sitemap_index, urls: get_sitemap_urls(sites))
-  end
+  def init(action) when action in [:show], do: action
 
   def call(%{assigns: %{site: site}} = conn, :show) do
     conn
@@ -24,16 +13,6 @@ defmodule Beacon.Web.SitemapController do
     |> render(:sitemap, pages: get_pages(site))
   end
 
-  defp get_sitemap_urls(sites) do
-    sites
-    |> Enum.map(fn {site, _} ->
-      routes_module = Beacon.Loader.fetch_routes_module(site)
-      Beacon.apply_mfa(site, routes_module, :public_sitemap_url, [])
-    end)
-    |> Enum.reject(&is_nil/1)
-    |> Enum.sort()
-  end
-
   defp get_pages(site) do
     routes_module = Beacon.Loader.fetch_routes_module(site)
 

lib/beacon/web/robots/robots.txt.eex

@@ -1,3 +1,3 @@
 # http://www.robotstxt.org
 User-agent: *
-Sitemap: <%= @sitemap_url %>
+Sitemap: <%= @sitemap_index_url %>

lib/beacon/web/sitemap/sitemap.xml.eex

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-    <%= for page <- @pages do %><url>
-        <loc><%= page.loc %></loc>
-        <lastmod><%= page.lastmod %></lastmod>
-    </url><% end %>
+  <%= for page <- @pages do %><url>
+    <loc><%= page.loc %></loc>
+    <lastmod><%= page.lastmod %></lastmod>
+  </url><% end %>
 </urlset>

test/beacon_web/controllers/robots_controller_test.exs

@@ -2,23 +2,12 @@ defmodule Beacon.Web.RobotsControllerTest do
   use Beacon.Web.ConnCase, async: false
 
   test "show", %{conn: conn} do
-    conn = get(conn, "/robots.txt")
+    conn = get(%{conn | host: "site_a.com"}, "/robots.txt")
 
     assert response(conn, 200) == """
            # http://www.robotstxt.org
            User-agent: *
-           Sitemap: http://site_a.com/sitemap.xml
-           """
-
-    assert response_content_type(conn, :txt) =~ "charset=utf-8"
-
-    # site: :not_booted
-    conn = get(conn, "/other/robots.txt")
-
-    assert response(conn, 200) == """
-           # http://www.robotstxt.org
-           User-agent: *
-           Sitemap: http://site_a.com/other/sitemap.xml
+           Sitemap: http://site_a.com/sitemap_index.xml
            """
 
     assert response_content_type(conn, :txt) =~ "charset=utf-8"