Kill processes on process exit (#638)

* Kill processes on exit
DataDog · Feb 12, 2020 · 8187def · 8187def
1 parent cb80625
commit 8187def
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 31 deletions.
diff --git a/src/Datadog.Trace/Tracer.cs b/src/Datadog.Trace/Tracer.cs
@@ -34,7 +34,7 @@ public class Tracer : IDatadogTracer
 
         static Tracer()
         {
-            TracerSubProcessManager.StartStandaloneAgentProcessesWhenConfigured();
+            TracingProcessManager.StartProcesses();
             // create the default global Tracer
             Instance = new Tracer();
         }
@@ -111,6 +111,7 @@ internal Tracer(TracerSettings settings, IAgentWriter agentWriter, ISampler samp
 
             // Register callbacks to make sure we flush the traces before exiting
             AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit;
+            AppDomain.CurrentDomain.DomainUnload += CurrentDomain_DomainUnload;
             AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
             Console.CancelKeyPress += Console_CancelKeyPress;
 
@@ -444,17 +445,36 @@ private void InitializeLibLogScopeEventSubscriber(IScopeManager scopeManager)
 
         private void CurrentDomain_ProcessExit(object sender, EventArgs e)
         {
-            _agentWriter.FlushAndCloseAsync().Wait();
+            RunShutdownTasks();
         }
 
         private void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)
         {
-            _agentWriter.FlushAndCloseAsync().Wait();
+            RunShutdownTasks();
         }
 
         private void Console_CancelKeyPress(object sender, ConsoleCancelEventArgs e)
         {
-            _agentWriter.FlushAndCloseAsync().Wait();
+            RunShutdownTasks();
+        }
+
+        private void CurrentDomain_DomainUnload(object sender, EventArgs e)
+        {
+            RunShutdownTasks();
+        }
+
+        private void RunShutdownTasks()
+        {
+            try
+            {
+                _agentWriter.FlushAndCloseAsync().Wait();
+            }
+            catch (Exception ex)
+            {
+                DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error flushing traces on shutdown."));
+            }
+
+            TracingProcessManager.StopProcesses();
         }
 
         private void HeartbeatCallback(object state)

diff --git a/src/Datadog.Trace/TracerSubProcessManager.cs → src/Datadog.Trace/TracingProcessManager.cs b/src/Datadog.Trace/TracerSubProcessManager.cs → src/Datadog.Trace/TracingProcessManager.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Threading;
@@ -8,42 +9,72 @@
 
 namespace Datadog.Trace
 {
-    internal class TracerSubProcessManager
+    internal class TracingProcessManager
     {
-        private static Task _traceAgentMonitor;
-        private static Task _dogStatsDMonitor;
+        private static readonly List<ProcessMetadata> Processes = new List<ProcessMetadata>()
+        {
+            new ProcessMetadata()
+            {
+                Name = "datadog-trace-agent",
+                ProcessPathKey = ConfigurationKeys.TraceAgentPath,
+                ProcessArgumentsKey = ConfigurationKeys.TraceAgentArgs,
+            },
+            new ProcessMetadata()
+            {
+                Name = "dogstatsd",
+                ProcessPathKey = ConfigurationKeys.DogStatsDPath,
+                ProcessArgumentsKey = ConfigurationKeys.DogStatsDArgs,
+            }
+        };
 
-        public static void StartStandaloneAgentProcessesWhenConfigured()
+        public static void StopProcesses()
         {
-            try
+            foreach (var subProcessMetadata in Processes)
             {
-                var traceAgentPath = Environment.GetEnvironmentVariable(ConfigurationKeys.TraceAgentPath);
+                SafelyKillProcess(subProcessMetadata);
+            }
+        }
 
-                if (!string.IsNullOrWhiteSpace(traceAgentPath))
-                {
-                    var traceProcessArgs = Environment.GetEnvironmentVariable(ConfigurationKeys.TraceAgentArgs);
-                    _traceAgentMonitor = StartProcessWithKeepAlive(traceAgentPath, traceProcessArgs);
-                }
-                else
+        public static void StartProcesses()
+        {
+            try
+            {
+                foreach (var subProcessMetadata in Processes)
                 {
-                    DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", ConfigurationKeys.TraceAgentPath));
-                }
-
-                var dogStatsDPath = Environment.GetEnvironmentVariable(ConfigurationKeys.DogStatsDPath);
+                    var processPath = Environment.GetEnvironmentVariable(subProcessMetadata.ProcessPathKey);
 
-                if (!string.IsNullOrWhiteSpace(dogStatsDPath))
-                {
-                    var dogStatsDArgs = Environment.GetEnvironmentVariable(ConfigurationKeys.DogStatsDArgs);
-                    _dogStatsDMonitor = StartProcessWithKeepAlive(dogStatsDPath, dogStatsDArgs);
+                    if (!string.IsNullOrWhiteSpace(processPath))
+                    {
+                        var processArgs = Environment.GetEnvironmentVariable(subProcessMetadata.ProcessArgumentsKey);
+                        subProcessMetadata.KeepAliveTask =
+                            StartProcessWithKeepAlive(processPath, processArgs, subProcessMetadata);
+                    }
+                    else
+                    {
+                        DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", subProcessMetadata.Name));
+                    }
                 }
-                else
+            }
+            catch (Exception ex)
+            {
+                DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error when attempting to start standalone agent processes."));
+            }
+        }
+
+        private static void SafelyKillProcess(ProcessMetadata metadata)
+        {
+            try
+            {
+                if (metadata.Process != null && !metadata.Process.HasExited)
                 {
-                    DatadogLogging.RegisterStartupLog(log => log.Debug("There is no path configured for {0}.", ConfigurationKeys.DogStatsDPath));
+                    metadata.Process.Kill();
                 }
+
+                metadata.KeepAliveTask?.Dispose();
             }
             catch (Exception ex)
             {
-                DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Error when attempting to start standalone agent processes."));
+                DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name));
             }
         }
 
@@ -66,7 +97,7 @@ private static bool ProgramIsRunning(string fullPath)
             return false;
         }
 
-        private static Task StartProcessWithKeepAlive(string path, string args)
+        private static Task StartProcessWithKeepAlive(string path, string args, ProcessMetadata metadata)
         {
             DatadogLogging.RegisterStartupLog(log => log.Debug("Starting keep alive for {0}.", path));
 
@@ -82,6 +113,12 @@ private static Task StartProcessWithKeepAlive(string path, string args)
                         {
                             try
                             {
+                                if (metadata.Process != null && metadata.Process.HasExited == false)
+                                {
+                                    DatadogLogging.RegisterStartupLog(log => log.Debug("We already have an active reference to {0}.", path));
+                                    continue;
+                                }
+
                                 if (ProgramIsRunning(path))
                                 {
                                     DatadogLogging.RegisterStartupLog(log => log.Debug("{0} is already running.", path));
@@ -97,11 +134,11 @@ private static Task StartProcessWithKeepAlive(string path, string args)
 
                                 DatadogLogging.RegisterStartupLog(log => log.Debug("Starting {0}.", path));
 
-                                var process = Process.Start(startInfo);
+                                metadata.Process = Process.Start(startInfo);
 
-                                Thread.Sleep(150);
+                                Thread.Sleep(200);
 
-                                if (process == null || process.HasExited)
+                                if (metadata.Process == null || metadata.Process.HasExited)
                                 {
                                     DatadogLogging.RegisterStartupLog(log => log.Error("{0} has failed to start.", path));
                                     sequentialFailures++;
@@ -136,5 +173,18 @@ private static Task StartProcessWithKeepAlive(string path, string args)
                     }
                 });
         }
+
+        private class ProcessMetadata
+        {
+            public string Name { get; set; }
+
+            public Process Process { get; set; }
+
+            public Task KeepAliveTask { get; set; }
+
+            public string ProcessPathKey { get; set; }
+
+            public string ProcessArgumentsKey { get; set; }
+        }
     }
 }