diff --git a/tracer/build/_build/Build.ExplorationTests.cs b/tracer/build/_build/Build.ExplorationTests.cs index 63b6090a47f6..abc0b2f44bcf 100644 --- a/tracer/build/_build/Build.ExplorationTests.cs +++ b/tracer/build/_build/Build.ExplorationTests.cs @@ -158,6 +158,45 @@ Target RunExplorationTests }) ; + Target SetUpSnapshotExplorationTests + => _ => _ + .Description("Sets up the Snapshot Exploration Test") + .Requires(() => ExplorationTestUseCase) + .After(Clean, BuildTracerHome) + .Executes(() => + { + if (ExplorationTestUseCase != global::ExplorationTestUseCase.Debugger) + { + return; + } + + GitCloneBuild(); + SetUpSnapshotExplorationTestsInternal(); + }); + + Target RunSnapshotExplorationTests + => _ => _ + .Description("Runs the Snapshot Exploration Test") + .Requires(() => ExplorationTestUseCase) + .After(Clean, BuildTracerHome, BuildNativeLoader, SetUpSnapshotExplorationTests) + .Executes(() => + { + if (ExplorationTestUseCase != global::ExplorationTestUseCase.Debugger) + { + return; + } + + // FileSystemTasks.EnsureCleanDirectory(TestLogsDirectory); + try + { + RunSnapshotExplorationTestsInternal(); + } + finally + { + CopyDumpsToBuildData(); + } + }); + Dictionary GetEnvironmentVariables(ExplorationTestDescription testDescription, TargetFramework framework) { var envVariables = new Dictionary @@ -224,44 +263,44 @@ void RunUnitTest(ExplorationTestDescription testDescription) Logger.Information($"Running exploration test {testDescription.Name}."); - if (Framework != null && !testDescription.IsFrameworkSupported(Framework)) - { - throw new InvalidOperationException($"The framework '{Framework}' is not listed in the project's target frameworks of {testDescription.Name}"); - } - if (Framework == null) { foreach (var targetFramework in testDescription.SupportedFrameworks) { var envVariables = GetEnvironmentVariables(testDescription, targetFramework); - Test(targetFramework, envVariables); + Test(testDescription, targetFramework, envVariables); } } else { + if (!testDescription.IsFrameworkSupported(Framework)) + { + throw new InvalidOperationException($"The framework '{Framework}' is not listed in the project's target frameworks of {testDescription.Name}"); + } + var envVariables = GetEnvironmentVariables(testDescription, Framework); - Test(Framework, envVariables); + Test(testDescription, Framework, envVariables); } + } - void Test(TargetFramework targetFramework, Dictionary envVariables) - { - DotNetTest( - x => - { - x = x - .SetProjectFile(testDescription.GetTestTargetPath(ExplorationTestsDirectory, targetFramework, BuildConfiguration)) - .EnableNoRestore() - .EnableNoBuild() - .SetConfiguration(BuildConfiguration) - .SetFramework(targetFramework) - .SetProcessEnvironmentVariables(envVariables) - .SetIgnoreFilter(testDescription.TestsToIgnore) - .WithMemoryDumpAfter(100) - ; - - return x; - }); - } + void Test(ExplorationTestDescription testDescription, TargetFramework targetFramework, Dictionary envVariables) + { + DotNetTest( + x => + { + x = x + .SetProjectFile(testDescription.GetTestTargetPath(ExplorationTestsDirectory, targetFramework, BuildConfiguration)) + .EnableNoRestore() + .EnableNoBuild() + .SetConfiguration(BuildConfiguration) + .SetFramework(targetFramework) + .SetProcessEnvironmentVariables(envVariables) + .SetIgnoreFilter(testDescription.TestsToIgnore) + .WithMemoryDumpAfter(100) + ; + + return x; + }); } private void CreateLineProbesIfNeeded() @@ -554,7 +593,7 @@ class ExplorationTestDescription public bool ShouldRun { get; set; } = true; public bool LineProbesEnabled { get; set; } - + public bool IsSnapshotScenario { get; set; } public string GetTestTargetPath(AbsolutePath explorationTestsDirectory, TargetFramework framework, Configuration buildConfiguration) { var projectPath = $"{explorationTestsDirectory}/{Name}/{PathToUnitTestProject}"; @@ -616,7 +655,8 @@ public static ExplorationTestDescription GetExplorationTestDescription(Explorati "Google.Protobuf.CodedInputStreamTest.MaliciousRecursion", "Google.Protobuf.CodedInputStreamTest.MaliciousRecursion_UnknownFields", "Google.Protobuf.CodedInputStreamTest.RecursionLimitAppliedWhileSkippingGroup", - "Google.Protobuf.JsonParserTest.MaliciousRecursion" + "Google.Protobuf.JsonParserTest.MaliciousRecursion", + "Google.Protobuf.Test.RefStructCompatibilityTest" }, LineProbesEnabled = true }, diff --git a/tracer/build/_build/Build.SnapshotExplorationTest.cs b/tracer/build/_build/Build.SnapshotExplorationTest.cs new file mode 100644 index 000000000000..1c82cda0fcec --- /dev/null +++ b/tracer/build/_build/Build.SnapshotExplorationTest.cs @@ -0,0 +1,367 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using Logger = Serilog.Log; + +partial class Build +{ + const string SnapshotExplorationTestProbesFileName = "SnapshotExplorationTestProbes.csv"; + const string SnapshotExplorationTestReportFileName = "SnapshotExplorationTestReport.csv"; + const string SnapshotExplorationEnabledKey = "DD_INTERNAL_SNAPSHOT_EXPLORATION_TEST_ENABLED"; + const string SnapshotExplorationProbesPathKey = "DD_INTERNAL_SNAPSHOT_EXPLORATION_TEST_PROBES_PATH"; + const string SnapshotExplorationReportPathKey = "DD_INTERNAL_SNAPSHOT_EXPLORATION_TEST_REPORT_PATH"; + const char SpecialSeparator = '#'; + + void RunSnapshotExplorationTestsInternal() + { + if (ExplorationTestName.HasValue) + { + Logger.Information($"Provided snapshot exploration test name is {ExplorationTestName}."); + var testDescription = ExplorationTestDescription.GetExplorationTestDescription(ExplorationTestName.Value); + RunSnapshotExplorationTest(testDescription); + } + else + { + Logger.Information("Snapshot exploration test name is not provided, running all."); + foreach (var testDescription in ExplorationTestDescription.GetAllExplorationTestDescriptions()) + { + RunSnapshotExplorationTest(testDescription); + } + } + } + + void RunSnapshotExplorationTest(ExplorationTestDescription testDescription) + { + if (!testDescription.ShouldRun) + { + Logger.Information($"Skipping exploration test: {testDescription.Name}."); + return; + } + + Logger.Information($"Running exploration test: {testDescription.Name}."); + + var frameworks = Framework == null ? testDescription.SupportedFrameworks : new[] { Framework }; + foreach (var framework in frameworks) + { + if (!testDescription.IsFrameworkSupported(framework)) + { + Logger.Information($"Skipping exploration test: {testDescription.Name}."); + Logger.Warning($"The framework '{framework}' is not listed in the project's target frameworks of {testDescription.Name}"); + } + + testDescription.IsSnapshotScenario = true; + var envVariables = GetEnvironmentVariables(testDescription, framework); + Test(testDescription, framework, envVariables); + VerifySnapshotExplorationTestResults(envVariables[SnapshotExplorationProbesPathKey], envVariables[SnapshotExplorationReportPathKey]); + } + } + + void SetUpSnapshotExplorationTestsInternal() + { + if (ExplorationTestName.HasValue) + { + Logger.Information($"Provided snapshot exploration test name is {ExplorationTestName}."); + var testDescription = ExplorationTestDescription.GetExplorationTestDescription(ExplorationTestName.Value); + CreateSnapshotExplorationTestCsv(testDescription); + } + else + { + Logger.Information("Snapshot exploration test name is not provided, running all."); + foreach (var testDescription in ExplorationTestDescription.GetAllExplorationTestDescriptions()) + { + CreateSnapshotExplorationTestCsv(testDescription); + } + } + } + + void CreateSnapshotExplorationTestCsv(ExplorationTestDescription testDescription) + { + var csvBuilder = new StringBuilder(); + csvBuilder.AppendLine("type name (FQN),method name,method signature,probeId,is instance method"); + var frameworks = Framework != null ? new[] { Framework } : testDescription.SupportedFrameworks; + + foreach (var framework in frameworks) + { + var testRootPath = testDescription.GetTestTargetPath(ExplorationTestsDirectory, framework, BuildConfiguration); + var tracerAssemblyPath = GetTracerAssemblyPath(framework); + var tracer = Assembly.LoadFile(tracerAssemblyPath); + var extractorType = tracer.GetType("Datadog.Trace.Debugger.Symbols.SymbolExtractor"); + var createMethod = extractorType?.GetMethod("Create", BindingFlags.Static | BindingFlags.Public); + var getClassSymbols = extractorType?.GetMethod("GetClassSymbols", BindingFlags.Instance | BindingFlags.NonPublic, Type.EmptyTypes); + var testAssembliesPaths = GetAllTestAssemblies(testRootPath); + + foreach (var testAssemblyPath in testAssembliesPaths) + { + var currentAssembly = Assembly.LoadFile(testAssemblyPath); + var symbolExtractor = createMethod?.Invoke(null, new object[] { currentAssembly }); + if (getClassSymbols?.Invoke(symbolExtractor, null) is not IEnumerable classSymbols) + { + continue; + } + + var processedScopes = ReflectionHelper.ProcessIEnumerable(classSymbols); + + foreach (var scope in processedScopes) + { + if (scope["ScopeType"].ToString() != "Class") + { + continue; + } + + var typeName = scope["Name"].ToString(); + ProcessNestedScopes((List>)scope["Scopes"], typeName, csvBuilder); + } + } + + File.WriteAllText(Path.Combine(testRootPath, SnapshotExplorationTestProbesFileName), csvBuilder.ToString()); + } + + return; + + void ProcessNestedScopes(List> scopes, string typeName, StringBuilder csvBuilder) + { + if (scopes == null) + { + return; + } + + foreach (var scope in scopes) + { + if (scope["ScopeType"].ToString() == "Class") + { + var nestedTypeName = scope["Name"].ToString(); + ProcessNestedScopes((List>)scope["Scopes"], nestedTypeName, csvBuilder); + continue; + } + + if (scope["ScopeType"].ToString() == "Method") // todo: closure + { + var isStatic = false; + var ls = scope["LanguageSpecifics"]; + if (ls?.GetType().GetProperty("Annotaions")?.GetValue(scope["Annotations"]) is IList annotations) + { + isStatic = (int.Parse(annotations[0], NumberStyles.HexNumber) & 0x0010) > 0; + } + + var returnType = ls?.GetType().GetProperty("ReturnType", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(ls)?.ToString(); + if (TryGetLine(typeName, scope["Name"].ToString(), returnType, (List>)scope["Symbols"], Guid.NewGuid(), isStatic, out var line)) + { + csvBuilder.AppendLine(line); + } + else + { + Logger.Warning($"Error to add probe info for: {line}"); + } + } + } + } + } + + bool TryGetLine(string type, string method, string returnType, List> methodParameters, Guid guid, bool isStatic, out string line) + { + try + { + var typeName = SanitiseName(type); + var methodName = SanitiseName(method); + var methodSignature = GetMethodSignature(returnType, methodParameters); + line = $"{typeName},{methodName},{SanitiseName(methodSignature)},{Guid.NewGuid()},{isStatic}"; + return !string.IsNullOrEmpty(typeName) && !string.IsNullOrEmpty(methodName) && !string.IsNullOrEmpty(returnType); + } + catch (Exception e) + { + line = $"Type: {type}, Method: {method}"; + return false; + } + + string SanitiseName(string name) => name == null ? string.Empty : name.Replace(',', SpecialSeparator); + + string GetMethodSignature(string returnType, List> symbols) + { + if (symbols == null) + { + return string.Empty; + } + + var parameterTypes = + (from symbol in symbols + where symbol["SymbolType"].ToString() == "Arg" + select symbol["Type"].ToString()) + .ToList(); + return $"{returnType} ({string.Join(SpecialSeparator, parameterTypes)})"; + } + } + + public void VerifySnapshotExplorationTestResults(string probesPath, string reportPath) + { + var definedProbes = ReadDefinedProbes(probesPath); + if (definedProbes == null || definedProbes.Count == 0) + { + throw new Exception("Snapshot exploration test failed. Could not read probes file"); + } + + var installedProbeIds = ReadInstalledProbeIdsFromNativeLogs(); + if (installedProbeIds == null || definedProbes.Count == 0) + { + throw new Exception("Snapshot exploration test failed. Could not read installed probes file"); + } + + var probesReport = ReadReportedSnapshotProbesIds(reportPath); + if (probesReport == null || definedProbes.Count == 0) + { + throw new Exception("Snapshot exploration test failed. Could not read report file"); + } + + var invalidOrErrorProbes = probesReport.Where(p => !p.IsValid || p.HasError).ToList(); + var missingFromReport = installedProbeIds.Except(probesReport.ToDictionary(info => info.ProbeId, info => info.Name)).ToList(); + var notInstalled = definedProbes.Except(installedProbeIds).ToList(); + + LogProbeCollection("Invalid or error probes", invalidOrErrorProbes.ToDictionary(info => info.ProbeId, info => info.Name).ToList()); + LogProbeCollection("Probes missing from report", missingFromReport); + LogProbeCollection("Defined probes not installed", notInstalled); + + var successfullyCollectedCount = installedProbeIds.Intersect(definedProbes).Count(); + var successPercentage = (double)successfullyCollectedCount / definedProbes.Count * 100; + + Logger.Information($"Successfully collected {successPercentage:F2}% of probes."); + + if (invalidOrErrorProbes.Any() || missingFromReport.Any() /*do we want to fail in case of not installed probe?*/) + { + throw new Exception("Snapshot exploration test failed."); + } + } + + void LogProbeCollection(string collectionName, List> probes) + { + foreach (var probe in probes) + { + Logger.Error($"{collectionName}: ID: {probe.Key}, Name: {probe.Value ?? string.Empty}"); + } + } + + public static Dictionary ReadDefinedProbes(string probesPath) + { + if (string.IsNullOrEmpty(probesPath)) + { + throw new ArgumentException("Report path cannot be null or empty", nameof(probesPath)); + } + + if (!File.Exists(probesPath)) + { + throw new FileNotFoundException("The specified report file does not exist", probesPath); + } + + return File.ReadLines(probesPath) + .Skip(1) // Skip the header row + .Select(line => line.Split(',')) + .Where(parts => parts.Length >= 5) + .ToDictionary( + parts => parts[3].Trim(), // probe_id as key + parts => $"{parts[1].Trim()} {parts[2].Trim()}" // method_name + signature as value + ); + } + + List ReadReportedSnapshotProbesIds(string reportPath) + { + if (string.IsNullOrEmpty(reportPath)) + { + throw new ArgumentException("Report path cannot be null or empty", nameof(reportPath)); + } + + if (!File.Exists(reportPath)) + { + throw new FileNotFoundException("The specified report file does not exist", reportPath); + } + + return File.ReadLines(reportPath) + .Skip(1) // Skip the header row + .Select(ParseCsvLine) + .ToList(); + } + + ProbeReportInfo ParseCsvLine(string line) + { + var parts = line.Split(','); + + if (parts.Length != 3 || parts.Any(string.IsNullOrWhiteSpace)) + { + var probeId = parts.Length > 0 ? parts[0].Trim() : "missing id"; + var name = parts.Length > 1 ? parts[1].Trim() : "missing name"; + return new ProbeReportInfo(probeId, name, false, true); + } + + var isValid = bool.TryParse(parts[2].Trim(), out var parsedIsValid) && parsedIsValid; + return new ProbeReportInfo(parts[0].Trim(), parts[1].Trim(), isValid, false); + } + + Dictionary ReadInstalledProbeIdsFromNativeLogs() + { + return new Dictionary { { "id", "method name" } }; + } + + record ProbeReportInfo(string ProbeId, string Name, bool IsValid, bool HasError) + { + public string ProbeId { get; set; } = ProbeId; + public string Name { get; set; } = Name; + public bool IsValid { get; set; } = IsValid; + public bool HasError { get; set; } = HasError; + } + + static class ReflectionHelper + { + internal static IEnumerable> ProcessIEnumerable(IEnumerable enumerable) + { + foreach (var item in enumerable) + { + if (item == null) + { + continue; + } + + yield return GetObjectProperties(item); + } + } + + private static IDictionary GetObjectProperties(object obj) + { + if (obj == null) + { + return null; + } + + var properties = new Dictionary(); + var type = obj.GetType(); + + foreach (var prop in type.GetProperties(BindingFlags.NonPublic | BindingFlags.Instance)) + { + try + { + if (prop.Name is "Scopes" or "Symbols" or "ScopeType" or "Name" or "LanguageSpecifics" or "Type" or "SymbolType") + { + var value = prop.GetValue(obj); + properties[prop.Name] = value; + + if (prop.Name == "Scopes" && value is IEnumerable nestedScopes) + { + properties[prop.Name] = ProcessIEnumerable(nestedScopes).ToList(); + } + else if (prop.Name == "Symbols" && value is IEnumerable symbols) + { + properties[prop.Name] = ProcessIEnumerable(symbols).ToList(); + } + } + } + catch (Exception ex) + { + properties[prop.Name] = $"Error accessing property: {ex.Message}"; + } + } + + return properties; + } + } +} diff --git a/tracer/build/_build/BuildVariables.cs b/tracer/build/_build/BuildVariables.cs index 7f07cd7dc9f9..36490f53b5dd 100644 --- a/tracer/build/_build/BuildVariables.cs +++ b/tracer/build/_build/BuildVariables.cs @@ -15,8 +15,16 @@ public void AddDebuggerEnvironmentVariables(Dictionary envVars, envVars.Add("DD_DYNAMIC_INSTRUMENTATION_ENABLED", "1"); envVars.Add("DD_INTERNAL_DEBUGGER_INSTRUMENT_ALL", "1"); + // envVars.Add("DD_INTERNAL_WAIT_FOR_DEBUGGER_ATTACH", "1"); - if (description.LineProbesEnabled) + if (description.IsSnapshotScenario) + { + envVars.Add(SnapshotExplorationEnabledKey, "1"); + var testRootPath = description.GetTestTargetPath(ExplorationTestsDirectory, framework, BuildConfiguration); + envVars.Add(SnapshotExplorationProbesPathKey, Path.Combine(testRootPath, SnapshotExplorationTestProbesFileName)); + envVars.Add(SnapshotExplorationReportPathKey, Path.Combine(testRootPath, SnapshotExplorationTestReportFileName)); + } + else if (description.LineProbesEnabled) { envVars.Add("DD_INTERNAL_DEBUGGER_INSTRUMENT_ALL_LINES", "1"); var testRootPath = description.GetTestTargetPath(ExplorationTestsDirectory, framework, BuildConfiguration); diff --git a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs index 8feb3024e712..a0a5c38aff08 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs @@ -24,6 +24,7 @@ using Datadog.Trace.ServiceFabric; using Datadog.Trace.Telemetry; using Datadog.Trace.Telemetry.Metrics; +using Datadog.Trace.Util; namespace Datadog.Trace.ClrProfiler { @@ -545,34 +546,56 @@ private static void InitLiveDebugger(Tracer tracer) // Service Name must be lowercase, otherwise the agent will not be able to find the service var serviceName = DynamicInstrumentationHelper.ServiceName; - var discoveryService = tracer.TracerManager.DiscoveryService; - Task.Run( - async () => - { - // TODO: LiveDebugger should be initialized in TracerManagerFactory so it can respond - // to changes in ExporterSettings etc. + if (debuggerSettings.IsSnapshotExplorationTestEnabled) + { + var liveDebugger = LiveDebuggerFactory.Create(new DiscoveryServiceMock(), RcmSubscriptionManager.Instance, settings, serviceName, tracer.TracerManager.Telemetry, debuggerSettings, tracer.TracerManager.GitMetadataTagsProvider); + Log.Debug("Initializing live debugger for snapshot exploration test."); + liveDebugger.WithProbesFromFile(); + Task.Run( + async () => + { + try + { + await liveDebugger.InitializeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Error(ex, "Error initializing live debugger."); + } + }); + } + else + { + var discoveryService = tracer.TracerManager.DiscoveryService; - try + Task.Run( + async () => { - var sw = Stopwatch.StartNew(); - var isDiscoverySuccessful = await WaitForDiscoveryService(discoveryService).ConfigureAwait(false); - TelemetryFactory.Metrics.RecordDistributionSharedInitTime(MetricTags.InitializationComponent.DiscoveryService, sw.ElapsedMilliseconds); + // TODO: LiveDebugger should be initialized in TracerManagerFactory so it can respond + // to changes in ExporterSettings etc. - if (isDiscoverySuccessful) + try { - var liveDebugger = LiveDebuggerFactory.Create(discoveryService, RcmSubscriptionManager.Instance, settings, serviceName, tracer.TracerManager.Telemetry, debuggerSettings, tracer.TracerManager.GitMetadataTagsProvider); + var sw = Stopwatch.StartNew(); + var isDiscoverySuccessful = await WaitForDiscoveryService(discoveryService).ConfigureAwait(false); + TelemetryFactory.Metrics.RecordDistributionSharedInitTime(MetricTags.InitializationComponent.DiscoveryService, sw.ElapsedMilliseconds); - Log.Debug("Initializing live debugger."); + if (isDiscoverySuccessful) + { + var liveDebugger = LiveDebuggerFactory.Create(discoveryService, RcmSubscriptionManager.Instance, settings, serviceName, tracer.TracerManager.Telemetry, debuggerSettings, tracer.TracerManager.GitMetadataTagsProvider); + + Log.Debug("Initializing live debugger."); - await InitializeLiveDebugger(liveDebugger).ConfigureAwait(false); + await InitializeLiveDebugger(liveDebugger).ConfigureAwait(false); + } } - } - catch (Exception ex) - { - Log.Error(ex, "Error initializing live debugger."); - } - }); + catch (Exception ex) + { + Log.Error(ex, "Error initializing live debugger."); + } + }); + } } // /!\ This method is called by reflection in the SampleHelpers diff --git a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs index 18358b29ea6c..987819e6d990 100644 --- a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs +++ b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs @@ -146,6 +146,27 @@ internal static class Debugger /// /// public const string MaxExceptionAnalysisLimit = "DD_EXCEPTION_REPLAY_MAX_EXCEPTION_ANALYSIS_LIMIT"; + + /// + /// Configuration key for enabling or disabling snapshot exploration test. + /// Default value is false (disabled). + /// + /// + public const string IsSnapshotExplorationTestEnabled = "DD_INTERNAL_SNAPSHOT_EXPLORATION_TEST_ENABLED"; + + /// + /// Configuration key for snapshot exploration test probe path. + /// Default value is empty. + /// + /// + public const string SnapshotExplorationTestProbesPath = "DD_INTERNAL_SNAPSHOT_EXPLORATION_TEST_PROBES_PATH"; + + /// + /// Configuration key for snapshot exploration test report path. + /// Default value is empty. + /// + /// + public const string SnapshotExplorationTestReportPath = "DD_INTERNAL_SNAPSHOT_EXPLORATION_TEST_REPORT_PATH"; } } } diff --git a/tracer/src/Datadog.Trace/Debugger/BoundLineProbeLocation.cs b/tracer/src/Datadog.Trace/Debugger/BoundLineProbeLocation.cs new file mode 100644 index 000000000000..09631e3abda2 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/BoundLineProbeLocation.cs @@ -0,0 +1,31 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using Datadog.Trace.Debugger.Configurations.Models; + +namespace Datadog.Trace.Debugger; + +internal record BoundLineProbeLocation +{ + public BoundLineProbeLocation(ProbeDefinition probe, Guid mvid, int methodToken, int bytecodeOffset, int lineNumber) + { + ProbeDefinition = probe; + MVID = mvid; + MethodToken = methodToken; + BytecodeOffset = bytecodeOffset; + LineNumber = lineNumber; + } + + public ProbeDefinition ProbeDefinition { get; set; } + + public Guid MVID { get; set; } + + public int MethodToken { get; set; } + + public int BytecodeOffset { get; set; } + + public int LineNumber { get; set; } +} diff --git a/tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs b/tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs index 1b8019d923c3..e7db09f2a858 100644 --- a/tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs +++ b/tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs @@ -80,7 +80,7 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Enumerable.Empty(); - SymDbThirdPartyDetectionIncludes = new HashSet([..symDb3rdPartyIncludeLibraries, ..ThirdPartyDetectionIncludes]).ToImmutableHashSet(); + SymDbThirdPartyDetectionIncludes = new HashSet([.. symDb3rdPartyIncludeLibraries, .. ThirdPartyDetectionIncludes]).ToImmutableHashSet(); var symDb3rdPartyExcludeLibraries = config .WithKeys(ConfigurationKeys.Debugger.SymDbThirdPartyDetectionExcludes) @@ -88,7 +88,7 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Enumerable.Empty(); - SymDbThirdPartyDetectionExcludes = new HashSet([..symDb3rdPartyExcludeLibraries, ..ThirdPartyDetectionExcludes]).ToImmutableHashSet(); + SymDbThirdPartyDetectionExcludes = new HashSet([.. symDb3rdPartyExcludeLibraries, .. ThirdPartyDetectionExcludes]).ToImmutableHashSet(); DiagnosticsIntervalSeconds = config .WithKeys(ConfigurationKeys.Debugger.DiagnosticsInterval) @@ -115,6 +115,10 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te Enumerable.Empty(); RedactedTypes = new HashSet(redactedTypes, StringComparer.OrdinalIgnoreCase); + + IsSnapshotExplorationTestEnabled = config.WithKeys(ConfigurationKeys.Debugger.IsSnapshotExplorationTestEnabled).AsBool(false); + SnapshotExplorationTestProbesPath = config.WithKeys(ConfigurationKeys.Debugger.SnapshotExplorationTestProbesPath).AsString(string.Empty); + SnapshotExplorationTestReportPath = config.WithKeys(ConfigurationKeys.Debugger.SnapshotExplorationTestReportPath).AsString(string.Empty); } public bool Enabled { get; } @@ -145,6 +149,12 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te public HashSet RedactedTypes { get; } + public bool IsSnapshotExplorationTestEnabled { get; set; } + + public string SnapshotExplorationTestProbesPath { get; set; } + + public string SnapshotExplorationTestReportPath { get; set; } + public static DebuggerSettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry) { return new DebuggerSettings(source, telemetry); diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebugging.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebugging.cs index fc1a04ffd9ca..f42bce525dd8 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebugging.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebugging.cs @@ -26,7 +26,7 @@ internal class ExceptionDebugging private static bool _isDisabled; private static SnapshotUploader? _uploader; - private static SnapshotSink? _snapshotSink; + private static ISnapshotSink? _snapshotSink; public static ExceptionReplaySettings Settings { diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/DiscoveryServiceMock.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/DiscoveryServiceMock.cs new file mode 100644 index 000000000000..8e389e4481e5 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/DiscoveryServiceMock.cs @@ -0,0 +1,42 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#nullable enable + +using System; +using System.Threading.Tasks; +using Datadog.Trace.Agent.DiscoveryService; + +namespace Datadog.Trace.Debugger.Helpers +{ + internal class DiscoveryServiceMock : IDiscoveryService + { + internal bool Called { get; private set; } + + public void SubscribeToChanges(Action callback) + { + Called = true; + callback( + new AgentConfiguration( + configurationEndpoint: "configurationEndpoint", + debuggerEndpoint: "debuggerEndpoint", + diagnosticsEndpoint: "diagnosticsEndpoint", + symbolDbEndpoint: "symbolDbEndpoint", + agentVersion: "agentVersion", + statsEndpoint: "traceStatsEndpoint", + dataStreamsMonitoringEndpoint: "dataStreamsMonitoringEndpoint", + eventPlatformProxyEndpoint: "eventPlatformProxyEndpoint", + telemetryProxyEndpoint: "telemetryProxyEndpoint", + tracerFlareEndpoint: "tracerFlareEndpoint", + clientDropP0: false, + spanMetaStructs: true)); + } + + public void RemoveSubscription(Action callback) + { + } + + public Task DisposeAsync() => Task.CompletedTask; + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/LiveDebugger.ExplorationTests.cs b/tracer/src/Datadog.Trace/Debugger/LiveDebugger.ExplorationTests.cs new file mode 100644 index 000000000000..c99cc490d89d --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/LiveDebugger.ExplorationTests.cs @@ -0,0 +1,74 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Configurations.Models; + +#nullable enable + +namespace Datadog.Trace.Debugger +{ + internal partial class LiveDebugger + { + internal async Task InitializeSync() + { + await InitializeAsync().ConfigureAwait(false); + return this; + } + + internal void WithProbesFromFile() + { + var probes = ReadProbesFromCsv(Settings.SnapshotExplorationTestProbesPath); + UpdateAddedProbeInstrumentations(probes); + } + + private List ReadProbesFromCsv(string filePath) + { + const char parametersSeparator = '#'; + var probes = new List(); + using var reader = new StreamReader(filePath); + + // Skip header + reader.ReadLine(); + + while (reader.ReadLine() is { } line) + { + var parts = line.Split(','); + if (parts.Length != 5) + { + Log.Warning("Invalid CSV line: {Line}", line); + continue; + } + + var probe = new LogProbe + { + Id = parts[3], // probeId + Where = new Where + { + TypeName = parts[0], // target type name (FQN) + MethodName = parts[1], // target method name + Signature = parts[2].Replace(parametersSeparator, ','), // signature + }, + EvaluateAt = EvaluateAt.Exit + }; + + // ReSharper disable once UnusedVariable + if (bool.TryParse(parts[4], out var isInstanceMethod) && probes.Count % 2 == 0) + { + const string condition = """{ "ne": [ { "ref": "this" }, null ]}"""; + + // Add condition for half of the instance methods + probe.When = new SnapshotSegment("ref this != null", condition, string.Empty); + } + + probes.Add(probe); + } + + return probes; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/LiveDebugger.cs b/tracer/src/Datadog.Trace/Debugger/LiveDebugger.cs index a4dda3a0e03d..bb1f421fa3a7 100644 --- a/tracer/src/Datadog.Trace/Debugger/LiveDebugger.cs +++ b/tracer/src/Datadog.Trace/Debugger/LiveDebugger.cs @@ -3,11 +3,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // -#pragma warning disable SA1402 // FileMayOnlyContainASingleType - StyleCop did not enforce this for records initially #nullable disable using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -25,12 +25,13 @@ using Datadog.Trace.DogStatsd; using Datadog.Trace.Logging; using Datadog.Trace.RemoteConfigurationManagement; +using Datadog.Trace.Util; using Datadog.Trace.Vendors.StatsdClient; using ProbeInfo = Datadog.Trace.Debugger.Expressions.ProbeInfo; namespace Datadog.Trace.Debugger { - internal class LiveDebugger + internal partial class LiveDebugger { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(LiveDebugger)); private static readonly object GlobalLock = new(); @@ -176,7 +177,6 @@ bool CanInitialize() Task StartAsync() { LifetimeManager.Instance.AddShutdownTask(ShutdownTask); - _probeStatusPoller.StartPolling(); _symbolsUploader.StartFlushingAsync(); _diagnosticsUploader.StartFlushingAsync(); @@ -217,53 +217,53 @@ internal void UpdateAddedProbeInstrumentations(IReadOnlyList ad switch (GetProbeLocationType(probe)) { case ProbeLocationType.Line: - { - var lineProbeResult = _lineProbeResolver.TryResolveLineProbe(probe, out var location); - var status = lineProbeResult.Status; - var message = lineProbeResult.Message; - - Log.Information("Finished resolving line probe for ProbeID {ProbeID}. Result was '{Status}'. Message was: '{Message}'", probe.Id, status, message); - switch (status) { - case LiveProbeResolveStatus.Bound: - lineProbes.Add(new NativeLineProbeDefinition(location.ProbeDefinition.Id, location.MVID, location.MethodToken, (int)location.BytecodeOffset, location.LineNumber, location.ProbeDefinition.Where.SourceFile)); - fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0)); - ProbeExpressionsProcessor.Instance.AddProbeProcessor(probe); - SetRateLimit(probe); - break; - case LiveProbeResolveStatus.Unbound: - Log.Information("ProbeID {ProbeID} is unbound.", probe.Id); - _unboundProbes.Add(probe); - fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0, new ProbeStatus(probe.Id, Sink.Models.Status.RECEIVED, errorMessage: null))); - break; - case LiveProbeResolveStatus.Error: - fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0, new ProbeStatus(probe.Id, Sink.Models.Status.ERROR, errorMessage: message))); - break; + var lineProbeResult = _lineProbeResolver.TryResolveLineProbe(probe, out var location); + var status = lineProbeResult.Status; + var message = lineProbeResult.Message; + + Log.Information("Finished resolving line probe for ProbeID {ProbeID}. Result was '{Status}'. Message was: '{Message}'", probe.Id, status, message); + switch (status) + { + case LiveProbeResolveStatus.Bound: + lineProbes.Add(new NativeLineProbeDefinition(location.ProbeDefinition.Id, location.MVID, location.MethodToken, (int)location.BytecodeOffset, location.LineNumber, location.ProbeDefinition.Where.SourceFile)); + fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0)); + ProbeExpressionsProcessor.Instance.AddProbeProcessor(probe); + SetRateLimit(probe); + break; + case LiveProbeResolveStatus.Unbound: + Log.Information("ProbeID {ProbeID} is unbound.", probe.Id); + _unboundProbes.Add(probe); + fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0, new ProbeStatus(probe.Id, Sink.Models.Status.RECEIVED, errorMessage: null))); + break; + case LiveProbeResolveStatus.Error: + fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0, new ProbeStatus(probe.Id, Sink.Models.Status.ERROR, errorMessage: message))); + break; + } + + break; } - break; - } - case ProbeLocationType.Method: - { - SignatureParser.TryParse(probe.Where.Signature, out var signature); - - fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0)); - if (probe is SpanProbe) { - var spanDefinition = new NativeSpanProbeDefinition(probe.Id, probe.Where.TypeName, probe.Where.MethodName, signature); - spanProbes.Add(spanDefinition); - } - else - { - var nativeDefinition = new NativeMethodProbeDefinition(probe.Id, probe.Where.TypeName, probe.Where.MethodName, signature); - methodProbes.Add(nativeDefinition); - ProbeExpressionsProcessor.Instance.AddProbeProcessor(probe); - SetRateLimit(probe); - } + SignatureParser.TryParse(probe.Where.Signature, out var signature); + + fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0)); + if (probe is SpanProbe) + { + var spanDefinition = new NativeSpanProbeDefinition(probe.Id, probe.Where.TypeName, probe.Where.MethodName, signature); + spanProbes.Add(spanDefinition); + } + else + { + var nativeDefinition = new NativeMethodProbeDefinition(probe.Id, probe.Where.TypeName, probe.Where.MethodName, signature); + methodProbes.Add(nativeDefinition); + ProbeExpressionsProcessor.Instance.AddProbeProcessor(probe); + SetRateLimit(probe); + } - break; - } + break; + } case ProbeLocationType.Unrecognized: fetchProbeStatus.Add(new FetchProbeStatus(probe.Id, probe.Version ?? 0, new ProbeStatus(probe.Id, Sink.Models.Status.ERROR, errorMessage: "Unknown probe type"))); @@ -283,13 +283,19 @@ internal void UpdateAddedProbeInstrumentations(IReadOnlyList ad } } - private static void SetRateLimit(ProbeDefinition probe) + private void SetRateLimit(ProbeDefinition probe) { if (probe is not LogProbe logProbe) { return; } + if (Settings.IsSnapshotExplorationTestEnabled) + { + ProbeRateLimiter.Instance.TryAddSampler(probe.Id, NopAdaptiveSampler.Instance); + return; + } + if (logProbe.Sampling is { } sampling) { ProbeRateLimiter.Instance.SetRate(probe.Id, (int)sampling.SnapshotsPerSecond); @@ -524,27 +530,3 @@ private void DiscoveryCallback(AgentConfiguration x) => _isRcmAvailable = !string.IsNullOrEmpty(x.ConfigurationEndpoint); } } - -internal record BoundLineProbeLocation -{ - public BoundLineProbeLocation(ProbeDefinition probe, Guid mvid, int methodToken, int bytecodeOffset, int lineNumber) - { - ProbeDefinition = probe; - MVID = mvid; - MethodToken = methodToken; - BytecodeOffset = bytecodeOffset; - LineNumber = lineNumber; - } - - public ProbeDefinition ProbeDefinition { get; set; } - - public Guid MVID { get; set; } - - public int MethodToken { get; set; } - - public int BytecodeOffset { get; set; } - - public int LineNumber { get; set; } -} - -#pragma warning restore SA1402 // FileMayOnlyContainASingleType - StyleCop did not enforce this for records initially diff --git a/tracer/src/Datadog.Trace/Debugger/LiveDebuggerFactory.cs b/tracer/src/Datadog.Trace/Debugger/LiveDebuggerFactory.cs index 204cc8351d15..ef45d10f6992 100644 --- a/tracer/src/Datadog.Trace/Debugger/LiveDebuggerFactory.cs +++ b/tracer/src/Datadog.Trace/Debugger/LiveDebuggerFactory.cs @@ -39,10 +39,10 @@ public static LiveDebugger Create(IDiscoveryService discoveryService, IRcmSubscr telemetry.ProductChanged(TelemetryProductType.DynamicInstrumentation, enabled: true, error: null); var snapshotSlicer = SnapshotSlicer.Create(debuggerSettings); - var snapshotStatusSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer); + var snapshotSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer); var diagnosticsSink = DiagnosticsSink.Create(serviceName, debuggerSettings); - var debuggerUploader = CreateSnaphotUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, false), snapshotStatusSink); + var debuggerUploader = CreateSnaphotUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, false), snapshotSink); var diagnosticsUploader = CreateDiagnosticsUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, true), diagnosticsSink); var lineProbeResolver = LineProbeResolver.Create(debuggerSettings.ThirdPartyDetectionExcludes, debuggerSettings.ThirdPartyDetectionIncludes); var probeStatusPoller = ProbeStatusPoller.Create(diagnosticsSink, debuggerSettings); @@ -83,12 +83,12 @@ private static IDogStatsd GetDogStatsd(ImmutableTracerSettings tracerSettings, s return statsd; } - private static SnapshotUploader CreateSnaphotUploader(IDiscoveryService discoveryService, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider, IApiRequestFactory apiFactory, SnapshotSink snapshotStatusSink) + private static SnapshotUploader CreateSnaphotUploader(IDiscoveryService discoveryService, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider, IApiRequestFactory apiFactory, ISnapshotSink snapshotSink) { var snapshotBatchUploadApi = DebuggerUploadApiFactory.CreateSnapshotUploadApi(apiFactory, discoveryService, gitMetadataTagsProvider); var snapshotBatchUploader = BatchUploader.Create(snapshotBatchUploadApi); - var debuggerSink = SnapshotUploader.Create(snapshotStatusSink, snapshotBatchUploader, debuggerSettings); + var debuggerSink = SnapshotUploader.Create(snapshotSink, snapshotBatchUploader, debuggerSettings); return debuggerSink; } diff --git a/tracer/src/Datadog.Trace/Debugger/Sink/DebuggerUploaderBase.cs b/tracer/src/Datadog.Trace/Debugger/Sink/DebuggerUploaderBase.cs index 08139bcb1c18..413813687dcc 100644 --- a/tracer/src/Datadog.Trace/Debugger/Sink/DebuggerUploaderBase.cs +++ b/tracer/src/Datadog.Trace/Debugger/Sink/DebuggerUploaderBase.cs @@ -113,6 +113,15 @@ private int ReconsiderFlushInterval(int currentInterval) public void Dispose() { - _cancellationSource.Cancel(); + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _cancellationSource.Cancel(); + } } } diff --git a/tracer/src/Datadog.Trace/Debugger/Sink/ISnapshotSink.cs b/tracer/src/Datadog.Trace/Debugger/Sink/ISnapshotSink.cs new file mode 100644 index 000000000000..eea560a937cc --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Sink/ISnapshotSink.cs @@ -0,0 +1,19 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable +using System; +using System.Collections.Generic; + +namespace Datadog.Trace.Debugger.Sink; + +internal interface ISnapshotSink : IDisposable +{ + public void Add(string probeId, string snapshot); + + IList GetSnapshots(); + + int RemainingCapacity(); +} diff --git a/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotExplorationTestSink.cs b/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotExplorationTestSink.cs new file mode 100644 index 000000000000..a4ad5b3d436b --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotExplorationTestSink.cs @@ -0,0 +1,179 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Models; +using Datadog.Trace.Debugger.Snapshots; +using Datadog.Trace.Logging; +using Datadog.Trace.VendoredMicrosoftCode.System.Collections.Immutable; +using Datadog.Trace.Vendors.Newtonsoft.Json; + +namespace Datadog.Trace.Debugger.Sink +{ + internal sealed class SnapshotExplorationTestSink : ISnapshotSink + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private readonly SnapshotSlicer _snapshotSlicer; + private readonly ProbeReportWriter _reportWriter; + + internal SnapshotExplorationTestSink(string reportFilePath, SnapshotSlicer snapshotSlicer) + { + _snapshotSlicer = snapshotSlicer; + _reportWriter = new ProbeReportWriter(reportFilePath); + } + + public void Add(string probeId, string snapshot) + { + var slicedSnapshot = _snapshotSlicer.SliceIfNeeded(probeId, snapshot); + _reportWriter.Enqueue(probeId, slicedSnapshot); + } + + public IList GetSnapshots() + { + return ImmutableList.Empty; + } + + public int RemainingCapacity() + { + return 1000; + } + + public void Dispose() + { + _reportWriter.Dispose(); + } + + private sealed class ProbeReportWriter : IDisposable + { + private readonly string _filePath; + private readonly BlockingCollection _writeQueue; + private readonly Task _writerTask; + private readonly CancellationTokenSource _cts; + private readonly int _bufferSize; + private bool _disposed; + + public ProbeReportWriter(string filePath, int bufferSize = 4096) + { + _filePath = filePath ?? throw new ArgumentNullException(nameof(filePath)); + _bufferSize = bufferSize; + _writeQueue = new BlockingCollection(); + _cts = new CancellationTokenSource(); + _writerTask = Task.Run(WriteProcess, _cts.Token); + } + + ~ProbeReportWriter() + { + Dispose(false); + } + + internal void Enqueue(string probeId, string snapshot) + { + try + { + _writeQueue.Add(new IdAndSnapshot(probeId, snapshot)); + } + catch (Exception e) + { + Log.Error(e, "Failed to queue snapshot."); + } + } + + private async Task WriteProcess() + { + var failures = 0; + const int maxFailures = 10; + using var writer = new StreamWriter(_filePath, true, Encoding.UTF8, _bufferSize); + writer.AutoFlush = false; + while (!_writeQueue.IsCompleted && !_cts.IsCancellationRequested) + { + try + { + if (!_writeQueue.TryTake(out var info, 200, _cts.Token)) + { + continue; + } + + var methodFullName = GetMethodFullName(info.Snapshot); + var line = $"{info.Id},{methodFullName ?? "N/A"},{!string.IsNullOrEmpty(methodFullName)}"; + await writer.WriteLineAsync(line).ConfigureAwait(false); + + if (writer.BaseStream.Position >= _bufferSize) + { + await writer.FlushAsync().ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + break; + } + catch (ThreadAbortException) + { + throw; + } + catch (Exception e) + { + if (++failures >= maxFailures) + { + Log.Error(e, "Stopping writing probe report. There were too many errors during the writing process."); + throw; + } + + Log.Error(e, "Error writing to probe report file."); + } + } + + await writer.FlushAsync().ConfigureAwait(false); + } + + private string? GetMethodFullName(string snapshot) + { + try + { + var parsedSnapshot = JsonConvert.DeserializeObject(snapshot); + return $"{parsedSnapshot.Logger.Name}.{parsedSnapshot.Logger.Method}"; + } + catch (Exception) + { + return null; + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + if (disposing) + { + _cts.Cancel(); + _writeQueue.CompleteAdding(); + _writerTask.Wait(); + _cts.Dispose(); + _writeQueue.Dispose(); + } + + _disposed = true; + } + } + + private record IdAndSnapshot(string Id, string Snapshot); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotSink.cs b/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotSink.cs index 53b6a724cff9..c4eec68988bc 100644 --- a/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotSink.cs +++ b/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotSink.cs @@ -7,10 +7,11 @@ using System.Collections.Generic; using Datadog.Trace.Debugger.Snapshots; using Datadog.Trace.Util; +using Datadog.Trace.VendoredMicrosoftCode.System.Collections.Immutable; namespace Datadog.Trace.Debugger.Sink { - internal class SnapshotSink + internal class SnapshotSink : ISnapshotSink { private const int DefaultQueueLimit = 1000; @@ -25,9 +26,16 @@ internal SnapshotSink(int batchSize, SnapshotSlicer snapshotSlicer) _queue = new BoundedConcurrentQueue(DefaultQueueLimit); } - public static SnapshotSink Create(DebuggerSettings settings, SnapshotSlicer snapshotSlicer) + public static ISnapshotSink Create(DebuggerSettings settings, SnapshotSlicer snapshotSlicer) { - return new SnapshotSink(settings.UploadBatchSize, snapshotSlicer); + if (settings.IsSnapshotExplorationTestEnabled) + { + return new SnapshotExplorationTestSink(settings.SnapshotExplorationTestReportPath, snapshotSlicer); + } + else + { + return new SnapshotSink(settings.UploadBatchSize, snapshotSlicer); + } } public void Add(string probeId, string snapshot) @@ -35,7 +43,7 @@ public void Add(string probeId, string snapshot) _queue.TryEnqueue(_snapshotSlicer.SliceIfNeeded(probeId, snapshot)); } - public List GetSnapshots() + public IList GetSnapshots() { var snapshots = new List(); var counter = 0; @@ -56,5 +64,9 @@ public int RemainingCapacity() { return DefaultQueueLimit - _queue.Count; } + + public void Dispose() + { + } } } diff --git a/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotUploader.cs b/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotUploader.cs index 7d0607e402b4..2590d338f99d 100644 --- a/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotUploader.cs +++ b/tracer/src/Datadog.Trace/Debugger/Sink/SnapshotUploader.cs @@ -10,11 +10,11 @@ namespace Datadog.Trace.Debugger.Sink { internal class SnapshotUploader : DebuggerUploaderBase, ISnapshotUploader { - private readonly SnapshotSink _snapshotSink; + private readonly ISnapshotSink _snapshotSink; private readonly IBatchUploader _snapshotBatchUploader; private SnapshotUploader( - SnapshotSink snapshotSink, + ISnapshotSink snapshotSink, IBatchUploader snapshotBatchUploader, DebuggerSettings settings) : base(settings) @@ -23,7 +23,7 @@ private SnapshotUploader( _snapshotSink = snapshotSink; } - public static SnapshotUploader Create(SnapshotSink snapshotSink, IBatchUploader snapshotBatchUploader, DebuggerSettings settings) + public static SnapshotUploader Create(ISnapshotSink snapshotSink, IBatchUploader snapshotBatchUploader, DebuggerSettings settings) { return new SnapshotUploader(snapshotSink, snapshotBatchUploader, settings); } @@ -46,5 +46,11 @@ public void Add(string probeId, string snapshot) { _snapshotSink.Add(probeId, snapshot); } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + _snapshotSink.Dispose(); + } } } diff --git a/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolExtractor.cs b/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolExtractor.cs index ef5484e05964..b7a09e0a4f79 100644 --- a/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolExtractor.cs +++ b/tracer/src/Datadog.Trace/Debugger/Symbols/SymbolExtractor.cs @@ -684,16 +684,17 @@ void PopulateClosureMethod(MethodDefinition generatedMethod, TypeDefinition owne } var argsSymbol = CreateArgSymbolArray(method, parameters); - int index = 0; + int argsIndex = 0; + int paramIndex = 0; var methodSig = method.DecodeSignature(new TypeProvider(false), 0); var paramTypesMatchArgSymbols = methodSig.ParameterTypes.Length == argsSymbol.Length || methodSig.ParameterTypes.Length == argsSymbol.Length - 1; foreach (var parameterHandle in parameters) { var parameterDef = MetadataReader.GetParameter(parameterHandle); - if (index == 0 && !method.IsStaticMethod()) + if (argsIndex == 0 && !method.IsStaticMethod()) { - argsSymbol[index] = new Symbol { Name = "this", SymbolType = SymbolType.Arg, Line = UnknownFieldAndArgLine, Type = method.GetDeclaringType().FullName(MetadataReader) }; - index++; + argsSymbol[argsIndex] = new Symbol { Name = "this", SymbolType = SymbolType.Arg, Line = UnknownFieldAndArgLine, Type = method.GetDeclaringType().FullName(MetadataReader) }; + argsIndex++; if (parameterDef.IsHiddenThis()) { @@ -701,14 +702,15 @@ void PopulateClosureMethod(MethodDefinition generatedMethod, TypeDefinition owne } } - argsSymbol[index] = new Symbol + argsSymbol[argsIndex] = new Symbol { Name = MetadataReader.GetString(parameterDef.Name), SymbolType = SymbolType.Arg, Line = UnknownFieldAndArgLine, - Type = paramTypesMatchArgSymbols ? methodSig.ParameterTypes[parameterDef.IsHiddenThis() ? index : index - 1] : "Unknown" + Type = paramTypesMatchArgSymbols ? methodSig.ParameterTypes[paramIndex] : "Unknown" }; - index++; + argsIndex++; + paramIndex++; } return argsSymbol; diff --git a/tracer/test/Datadog.Trace.Tests/Debugger/LiveDebuggerTests.cs b/tracer/test/Datadog.Trace.Tests/Debugger/LiveDebuggerTests.cs index bd94945bc6ec..083489b6dd55 100644 --- a/tracer/test/Datadog.Trace.Tests/Debugger/LiveDebuggerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Debugger/LiveDebuggerTests.cs @@ -13,6 +13,7 @@ using Datadog.Trace.Debugger; using Datadog.Trace.Debugger.Configurations; using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Helpers; using Datadog.Trace.Debugger.Models; using Datadog.Trace.Debugger.ProbeStatuses; using Datadog.Trace.Debugger.Sink; @@ -77,36 +78,6 @@ public async Task DebuggerDisabled_ServicesNotCalled() rcmSubscriptionManagerMock.ProductKeys.Contains(RcmProducts.LiveDebugging).Should().BeFalse(); } - private class DiscoveryServiceMock : IDiscoveryService - { - internal bool Called { get; private set; } - - public void SubscribeToChanges(Action callback) - { - Called = true; - callback( - new AgentConfiguration( - configurationEndpoint: "configurationEndpoint", - debuggerEndpoint: "debuggerEndpoint", - diagnosticsEndpoint: "diagnosticsEndpoint", - symbolDbEndpoint: "symbolDbEndpoint", - agentVersion: "agentVersion", - statsEndpoint: "traceStatsEndpoint", - dataStreamsMonitoringEndpoint: "dataStreamsMonitoringEndpoint", - eventPlatformProxyEndpoint: "eventPlatformProxyEndpoint", - telemetryProxyEndpoint: "telemetryProxyEndpoint", - tracerFlareEndpoint: "tracerFlareEndpoint", - clientDropP0: false, - spanMetaStructs: true)); - } - - public void RemoveSubscription(Action callback) - { - } - - public Task DisposeAsync() => Task.CompletedTask; - } - private class RcmSubscriptionManagerMock : IRcmSubscriptionManager { public bool HasAnySubscription { get; }