From 8187deff365328fff1fa41d7794350120764125d Mon Sep 17 00:00:00 2001 From: Colin Higgins Date: Wed, 12 Feb 2020 16:56:59 -0500 Subject: [PATCH] Kill processes on process exit (#638) * Kill processes on exit --- src/Datadog.Trace/Tracer.cs | 28 ++++- ...essManager.cs => TracingProcessManager.cs} | 104 +++++++++++++----- 2 files changed, 101 insertions(+), 31 deletions(-) rename src/Datadog.Trace/{TracerSubProcessManager.cs => TracingProcessManager.cs} (56%) diff --git a/src/Datadog.Trace/Tracer.cs b/src/Datadog.Trace/Tracer.cs index 849a2b81083b..5bbfaf09bd5b 100644 --- a/src/Datadog.Trace/Tracer.cs +++ b/src/Datadog.Trace/Tracer.cs @@ -34,7 +34,7 @@ public class Tracer : IDatadogTracer static Tracer() { - TracerSubProcessManager.StartStandaloneAgentProcessesWhenConfigured(); + TracingProcessManager.StartProcesses(); // create the default global Tracer Instance = new Tracer(); } @@ -111,6 +111,7 @@ internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ISampler samp // Register callbacks to make sure we flush the traces before exiting AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit; + AppDomain.CurrentDomain.DomainUnload += CurrentDomain_DomainUnload; AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException; Console.CancelKeyPress += Console_CancelKeyPress; @@ -444,17 +445,36 @@ private void InitializeLibLogScopeEventSubscriber(IScopeManager scopeManager) private void CurrentDomain_ProcessExit(object sender, EventArgs e) { - _agentWriter.FlushAndCloseAsync().Wait(); + RunShutdownTasks(); } private void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) { - _agentWriter.FlushAndCloseAsync().Wait(); + RunShutdownTasks(); } private void Console_CancelKeyPress(object sender, ConsoleCancelEventArgs e) { - _agentWriter.FlushAndCloseAsync().Wait(); + RunShutdownTasks(); + } + + private void CurrentDomain_DomainUnload(object sender, EventArgs e) + { + RunShutdownTasks(); + } + + private void RunShutdownTasks() + { + try + { + _agentWriter.FlushAndCloseAsync().Wait(); + } + catch (Exception ex) + { + DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error flushing traces on shutdown.")); + } + + TracingProcessManager.StopProcesses(); } private void HeartbeatCallback(object state) diff --git a/src/Datadog.Trace/TracerSubProcessManager.cs b/src/Datadog.Trace/TracingProcessManager.cs similarity index 56% rename from src/Datadog.Trace/TracerSubProcessManager.cs rename to src/Datadog.Trace/TracingProcessManager.cs index c8030528b706..e82a0e9a5e04 100644 --- a/src/Datadog.Trace/TracerSubProcessManager.cs +++ b/src/Datadog.Trace/TracingProcessManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading; @@ -8,42 +9,72 @@ namespace Datadog.Trace { - internal class TracerSubProcessManager + internal class TracingProcessManager { - private static Task _traceAgentMonitor; - private static Task _dogStatsDMonitor; + private static readonly List Processes = new List() + { + new ProcessMetadata() + { + Name = "datadog-trace-agent", + ProcessPathKey = ConfigurationKeys.TraceAgentPath, + ProcessArgumentsKey = ConfigurationKeys.TraceAgentArgs, + }, + new ProcessMetadata() + { + Name = "dogstatsd", + ProcessPathKey = ConfigurationKeys.DogStatsDPath, + ProcessArgumentsKey = ConfigurationKeys.DogStatsDArgs, + } + }; - public static void StartStandaloneAgentProcessesWhenConfigured() + public static void StopProcesses() { - try + foreach (var subProcessMetadata in Processes) { - var traceAgentPath = Environment.GetEnvironmentVariable(ConfigurationKeys.TraceAgentPath); + SafelyKillProcess(subProcessMetadata); + } + } - if (!string.IsNullOrWhiteSpace(traceAgentPath)) - { - var traceProcessArgs = Environment.GetEnvironmentVariable(ConfigurationKeys.TraceAgentArgs); - _traceAgentMonitor = StartProcessWithKeepAlive(traceAgentPath, traceProcessArgs); - } - else + public static void StartProcesses() + { + try + { + foreach (var subProcessMetadata in Processes) { - DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", ConfigurationKeys.TraceAgentPath)); - } - - var dogStatsDPath = Environment.GetEnvironmentVariable(ConfigurationKeys.DogStatsDPath); + var processPath = Environment.GetEnvironmentVariable(subProcessMetadata.ProcessPathKey); - if (!string.IsNullOrWhiteSpace(dogStatsDPath)) - { - var dogStatsDArgs = Environment.GetEnvironmentVariable(ConfigurationKeys.DogStatsDArgs); - _dogStatsDMonitor = StartProcessWithKeepAlive(dogStatsDPath, dogStatsDArgs); + if (!string.IsNullOrWhiteSpace(processPath)) + { + var processArgs = Environment.GetEnvironmentVariable(subProcessMetadata.ProcessArgumentsKey); + subProcessMetadata.KeepAliveTask = + StartProcessWithKeepAlive(processPath, processArgs, subProcessMetadata); + } + else + { + DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", subProcessMetadata.Name)); + } } - else + } + catch (Exception ex) + { + DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error when attempting to start standalone agent processes.")); + } + } + + private static void SafelyKillProcess(ProcessMetadata metadata) + { + try + { + if (metadata.Process != null && !metadata.Process.HasExited) { - DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", ConfigurationKeys.DogStatsDPath)); + metadata.Process.Kill(); } + + metadata.KeepAliveTask?.Dispose(); } catch (Exception ex) { - DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error when attempting to start standalone agent processes.")); + DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name)); } } @@ -66,7 +97,7 @@ private static bool ProgramIsRunning(string fullPath) return false; } - private static Task StartProcessWithKeepAlive(string path, string args) + private static Task StartProcessWithKeepAlive(string path, string args, ProcessMetadata metadata) { DatadogLogging.RegisterStartupLog(log => log.Debug("Starting keep alive for {0}.", path)); @@ -82,6 +113,12 @@ private static Task StartProcessWithKeepAlive(string path, string args) { try { + if (metadata.Process != null && metadata.Process.HasExited == false) + { + DatadogLogging.RegisterStartupLog(log => log.Debug("We already have an active reference to {0}.", path)); + continue; + } + if (ProgramIsRunning(path)) { DatadogLogging.RegisterStartupLog(log => log.Debug("{0} is already running.", path)); @@ -97,11 +134,11 @@ private static Task StartProcessWithKeepAlive(string path, string args) DatadogLogging.RegisterStartupLog(log => log.Debug("Starting {0}.", path)); - var process = Process.Start(startInfo); + metadata.Process = Process.Start(startInfo); - Thread.Sleep(150); + Thread.Sleep(200); - if (process == null || process.HasExited) + if (metadata.Process == null || metadata.Process.HasExited) { DatadogLogging.RegisterStartupLog(log => log.Error("{0} has failed to start.", path)); sequentialFailures++; @@ -136,5 +173,18 @@ private static Task StartProcessWithKeepAlive(string path, string args) } }); } + + private class ProcessMetadata + { + public string Name { get; set; } + + public Process Process { get; set; } + + public Task KeepAliveTask { get; set; } + + public string ProcessPathKey { get; set; } + + public string ProcessArgumentsKey { get; set; } + } } }