From a7fb8d1dda6a45e1ddadd0c248750f4502278daa Mon Sep 17 00:00:00 2001 From: Marcin Koba Date: Tue, 23 Apr 2024 12:36:25 +0200 Subject: [PATCH 1/2] Make sure that TelemetryMetricsStatsD can be started if host can't be resolved If the StatsD collector is not up for some reason and the application can't resolve the name by calling `:inet.gethostbyname(host)` then it prevents the entire application from starting. This commit makes sure that the TelemetryMetricsStatsD can be started in the errored state and depending on the `host_resolution_interval` it can be recovered or not. If `host_resolution_interval` isn't passed, then TelemetryMetricsStatsD will not try to resolve the host again. However, when it is passed, it can try to recover it in `host_resolution_interval` milliseconds. --- lib/telemetry_metrics_statsd.ex | 115 +++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/lib/telemetry_metrics_statsd.ex b/lib/telemetry_metrics_statsd.ex index c165419..56cca22 100644 --- a/lib/telemetry_metrics_statsd.ex +++ b/lib/telemetry_metrics_statsd.ex @@ -419,31 +419,36 @@ defmodule TelemetryMetricsStatsd do Process.flag(:trap_exit, true) metrics = Map.fetch!(options, :metrics) - udp_config = + {initialized_properly, udp_config} = case options.host do - {:local, _} = host -> %{host: host} + {:local, _} = host -> {true, %{host: host}} _ -> configure_host_resolution(options) end - udps = - for _ <- 1..options.pool_size do - {:ok, udp} = UDP.open(udp_config) - {:udp, udp} - end - pool_id = :ets.new(__MODULE__, [:bag, :protected, read_concurrency: true]) - :ets.insert(pool_id, udps) handler_ids = - EventHandler.attach( - metrics, - self(), - pool_id, - options.mtu, - options.prefix, - options.formatter, - options.global_tags - ) + if initialized_properly do + udps = + for _ <- 1..options.pool_size do + {:ok, udp} = UDP.open(udp_config) + {:udp, udp} + end + + :ets.insert(pool_id, udps) + + EventHandler.attach( + metrics, + self(), + pool_id, + options.mtu, + options.prefix, + options.formatter, + options.global_tags + ) + else + [] + end {:ok, %{ @@ -452,7 +457,8 @@ defmodule TelemetryMetricsStatsd do pool_id: pool_id, host: options.host, port: options.port, - host_resolution_interval: options.host_resolution_interval + host_resolution_interval: options.host_resolution_interval, + initialized_properly: initialized_properly }} end @@ -492,28 +498,40 @@ defmodule TelemetryMetricsStatsd do @impl true def handle_info(:resolve_host, state) do - %{host: host, udp_config: %{host: current_address}, host_resolution_interval: interval} = + %{ + host: host, + udp_config: udp_config, + host_resolution_interval: interval, + initialized_properly: initialized_properly + } = state new_state = case :inet.gethostbyname(host) do {:ok, hostent(h_addr_list: ips)} -> - if Enum.member?(ips, current_address) do - state - else - [new_address | _] = ips - update_host(state, new_address) + cond do + !initialized_properly -> + [ip_address | _] = ips + Logger.debug("Resolved host #{host} to #{:inet.ntoa(ip_address)} IP address.") + update_host(state, ip_address) + + Enum.member?(ips, udp_config.host) -> + state end {:error, reason} -> - Logger.log( - @log_level_warning, - "Failed to resolve the hostname #{host}: #{inspect(reason)}. " <> - "Using the previously resolved address of #{:inet.ntoa(current_address)}." - ) - end + if(!initialized_properly) do + Logger.log( + @log_level_warning, + "Failed to resolve the hostname #{host}: #{inspect(reason)}. " <> + "Previously resolved hostname was unsuccessful. The library will not send any metrics. " <> + "Retrying to resolve it again in #{interval} milliseconds." + ) + end - Process.send_after(self(), :resolve_host, interval) + Process.send_after(self(), :resolve_host, interval) + state + end {:noreply, new_state} end @@ -538,7 +556,7 @@ defmodule TelemetryMetricsStatsd do inet_address_family: inet_address_family }) when is_tuple(host) do - %{host: host, port: port, inet_address_family: inet_address_family} + {true, %{host: host, port: port, inet_address_family: inet_address_family}} end defp configure_host_resolution(%{ @@ -548,9 +566,20 @@ defmodule TelemetryMetricsStatsd do host_resolution_interval: interval }) when is_integer(interval) do - {:ok, hostent(h_addr_list: [ip | _ips])} = :inet.gethostbyname(host, inet_address_family) - Process.send_after(self(), :resolve_host, interval) - %{host: ip, port: port, inet_address_family: inet_address_family} + case :inet.gethostbyname(host, inet_address_family) do + {:ok, hostent(h_addr_list: [ip | _ips])} -> + {true, %{host: ip, port: port, inet_address_family: inet_address_family}} + + {:error, reason} -> + Logger.log( + @log_level_warning, + "Failed to resolve the hostname #{host}: #{inspect(reason)}. " <> + "Retrying to resolve it again in #{interval} milliseconds." + ) + + Process.send_after(self(), :resolve_host, interval) + {false, %{host: nil, port: port, inet_address_family: inet_address_family}} + end end defp configure_host_resolution(%{ @@ -558,8 +587,18 @@ defmodule TelemetryMetricsStatsd do port: port, inet_address_family: inet_address_family }) do - {:ok, hostent(h_addr_list: [ip | _ips])} = :inet.gethostbyname(host, inet_address_family) - %{host: ip, port: port, inet_address_family: inet_address_family} + case :inet.gethostbyname(host, inet_address_family) do + {:ok, hostent(h_addr_list: [ip | _ips])} -> + {true, %{host: ip, port: port, inet_address_family: inet_address_family}} + + {:error, reason} -> + Logger.log( + @log_level_warning, + "Failed to resolve the hostname #{host}: #{inspect(reason)}. Metrics will not be sent at all." + ) + + {false, %{host: nil, port: port, inet_address_family: inet_address_family}} + end end defp update_pool(pool_id, new_host, new_port) do From 5c0e4aaebf8e86ef29789e25eae064a1fd4eb882 Mon Sep 17 00:00:00 2001 From: Marcin Koba Date: Wed, 24 Apr 2024 15:40:17 +0200 Subject: [PATCH 2/2] setting --- lib/telemetry_metrics_statsd.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/telemetry_metrics_statsd.ex b/lib/telemetry_metrics_statsd.ex index 56cca22..780cd43 100644 --- a/lib/telemetry_metrics_statsd.ex +++ b/lib/telemetry_metrics_statsd.ex @@ -547,7 +547,7 @@ defmodule TelemetryMetricsStatsd do %{pool_id: pool_id, udp_config: %{port: port} = udp_config} = state update_pool(pool_id, new_address, port) - %{state | udp_config: %{udp_config | host: new_address}} + %{state | udp_config: %{udp_config | host: new_address}, initialized_properly: true} end defp configure_host_resolution(%{