From 0ab074390689880fe9269804a0156341d30251f6 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 21 Jan 2025 18:48:00 -0500 Subject: [PATCH] [Rgen] Compile all platforms provided in parallel. (#22024) Allow the command line to be used with more than one rsp. That allows to pass all the rsp for the different platforms we support allowing us to merge the compilation results later to be used by the transformation. --------- Co-authored-by: GitHub Actions Autoformatter --- .../CompilationResult.cs | 20 +++ src/rgen/Microsoft.Macios.Transformer/Main.cs | 67 ++++++--- .../Transformer.cs | 141 +++++++++++------- .../TransformerTests.cs | 2 +- 4 files changed, 156 insertions(+), 74 deletions(-) create mode 100644 src/rgen/Microsoft.Macios.Transformer/CompilationResult.cs diff --git a/src/rgen/Microsoft.Macios.Transformer/CompilationResult.cs b/src/rgen/Microsoft.Macios.Transformer/CompilationResult.cs new file mode 100644 index 00000000000..379906bf36b --- /dev/null +++ b/src/rgen/Microsoft.Macios.Transformer/CompilationResult.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Xamarin.Utils; + +namespace Microsoft.Macios.Transformer; + +/// +/// Holds the result of a compilation. This is a helper record to make code cleaner in the transformer. +/// +/// The platform the compilation targeted. +/// The compilation result. +/// All compilation errors. +record CompilationResult (ApplePlatform Platform, Compilation Compilation, ImmutableArray Errors) { + + public (ApplePlatform Platform, Compilation compilation) ToTuple () + => (Platform, Compilation); +} diff --git a/src/rgen/Microsoft.Macios.Transformer/Main.cs b/src/rgen/Microsoft.Macios.Transformer/Main.cs index b1644a75537..2774134b00b 100644 --- a/src/rgen/Microsoft.Macios.Transformer/Main.cs +++ b/src/rgen/Microsoft.Macios.Transformer/Main.cs @@ -5,6 +5,7 @@ using Serilog.Core; using Serilog.Events; using Serilog.Templates; +using Xamarin.Utils; using static System.Console; public class Program { @@ -14,7 +15,7 @@ public class Program { public static int Main (string [] args) { // Create options - var rspOption = new Option ( + var rspOption = new Option ( new [] { "--response-file", "--rsp" }, "Path to the RSP file" ); @@ -63,16 +64,16 @@ public static int Main (string [] args) } // Set handler for parsing and executing - rootCmd.SetHandler (async (rspPath, destPath, workingDirectory, sdkPath, force, verbosity) => { - WriteLine ($"Microsoft.Macios.Transformer v{typeof (Program).Assembly.GetName ().Version}, (c) Microsoft Corporation. All rights reserved.\n"); + rootCmd.SetHandler (async (rspPlatformPaths, destPath, workingDirectory, sdkPath, force, verbosity) => { + WriteLine ( + $"Microsoft.Macios.Transformer v{typeof (Program).Assembly.GetName ().Version}, (c) Microsoft Corporation. All rights reserved.\n"); // Convert local to absolute, expand ~ - rspPath = ToAbsolutePath (rspPath); workingDirectory = ToAbsolutePath (workingDirectory); destPath = ToAbsolutePath (destPath); sdkPath = ToAbsolutePath (sdkPath); - ValidateRsp (rspPath); + ValidateRsp (rspPlatformPaths, out var rspFiles); ValidateSdk (sdkPath); ValidateWorkingDirectory (workingDirectory); ValidateVerbosity (verbosity); @@ -88,18 +89,9 @@ public static int Main (string [] args) "[{@t:HH:mm:ss} {@l:u3} {Substring(SourceContext, LastIndexOf(SourceContext, '.') + 1)} (Thread: {ThreadId})] {@m}\n")) .CreateLogger (); - // Parse the .rsp file with Roslyn's CSharpCommandLineParser - var args = new [] { $"@{rspPath}" }; - var parseResult = CSharpCommandLineParser.Default.Parse (args, workingDirectory, null); - logger.Information ("RSP parsed. Errors: {ParserErrorLength}", parseResult.Errors.Length); - foreach (var resultError in parseResult.Errors) { - logger.Error ("{Error}", resultError); - WriteLine (resultError); - } - await Transformer.Execute ( destinationDirectory: destPath, - rspFile: rspPath, + rspFiles: rspFiles, workingDirectory: workingDirectory, sdkDirectory: sdkPath ); @@ -119,13 +111,46 @@ static string ToAbsolutePath (string path) return absolutePath; } - static void ValidateRsp (string path) + static void ValidateRsp (string [] paths, out List<(ApplePlatform Platform, string RspPath)> rspFiles) { - if (string.IsNullOrWhiteSpace (path) || !File.Exists (path)) - throw new FileNotFoundException ("RSP file not found."); - if ((File.GetAttributes (path) & FileAttributes.Directory) == FileAttributes.Directory) - throw new Exception ("RSP path is a directory."); - using var stream = File.OpenRead (path); // validate readability + rspFiles = []; + // loop over all the strings, split them by the ':' and retrieve the platform and path. Then + // validate the path + foreach (var cmdPath in paths) { + var parts = cmdPath.Split (':'); + if (parts.Length != 2) + throw new Exception ("Invalid RSP format. Expected platform:path"); + ApplePlatform platform; + var rspPath = ToAbsolutePath (parts [1]); + switch (parts [0]) { + case "ios": + platform = ApplePlatform.iOS; + break; + case "tvos": + platform = ApplePlatform.TVOS; + break; + case "macos": + platform = ApplePlatform.MacOSX; + break; + case "maccatalys": + platform = ApplePlatform.MacCatalyst; + break; + default: + platform = ApplePlatform.None; + break; + } + + if (platform == ApplePlatform.None) + throw new Exception ("Invalid platform in RSP file."); + + if (string.IsNullOrWhiteSpace (rspPath) || !File.Exists (rspPath)) + throw new FileNotFoundException ($"RSP '{rspPath}' file not found."); + if ((File.GetAttributes (rspPath) & FileAttributes.Directory) == FileAttributes.Directory) + throw new Exception ($"RSP {rspPath} is a directory."); + using var stream = File.OpenRead (rspPath); // validate readability + logger.Debug ("Adding RSP file {RspPath} for platform {Platform}", rspPath, platform); + rspFiles.Add ((platform, rspPath)); + } } static void ValidateWorkingDirectory (string path) diff --git a/src/rgen/Microsoft.Macios.Transformer/Transformer.cs b/src/rgen/Microsoft.Macios.Transformer/Transformer.cs index a1acec3abaa..cea8cb38437 100644 --- a/src/rgen/Microsoft.Macios.Transformer/Transformer.cs +++ b/src/rgen/Microsoft.Macios.Transformer/Transformer.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Text; using Marille; @@ -11,6 +12,7 @@ using Microsoft.Macios.Transformer.Extensions; using Microsoft.Macios.Transformer.Workers; using Serilog; +using Xamarin.Utils; namespace Microsoft.Macios.Transformer; @@ -21,14 +23,16 @@ namespace Microsoft.Macios.Transformer; class Transformer { readonly static ILogger logger = Log.ForContext (); readonly string destinationDirectory; - readonly Compilation compilation; + readonly ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilations; readonly HashSet? namespaceFilter; readonly Dictionary> transformers = new (); - internal Transformer (string destination, Compilation compilationResult, IEnumerable? namespaces = null) + internal Transformer (string destination, + ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilationsResult, + IEnumerable? namespaces = null) { destinationDirectory = destination; - compilation = compilationResult; + compilations = compilationsResult; if (namespaces is not null) namespaceFilter = new HashSet (namespaces); @@ -66,7 +70,8 @@ internal async Task CreateHub () { // get the attrs, based on those return the correct topic to use var attrs = symbol.GetAttributeData (); - logger.Debug ("Symbol '{SymbolName}' has [{Attributes}] attributes", symbol.Name, string.Join (", ", attrs.Keys)); + logger.Debug ("Symbol '{SymbolName}' has [{Attributes}] attributes", symbol.Name, + string.Join (", ", attrs.Keys)); logger.Debug ("Symbol '{SymbolName}' kind is '{SymbolKind}'", symbol.Name, symbol.TypeKind); if (symbol.TypeKind == TypeKind.Enum) { @@ -133,6 +138,7 @@ internal bool Skip (SyntaxTree syntaxTree, ISymbol symbol, [NotNullWhen (false)] // filtered out return true; } + outputDirectory = Path.Combine (destinationDirectory, symbolNamespace); // If the syntax tree comes from the output directory, we skip it because this is a manual binding return syntaxTree.FilePath.StartsWith (outputDirectory); @@ -153,37 +159,40 @@ internal async Task Execute () var hub = await CreateHub (); // with the hub created, loop over the syntax trees and create the messages to be sent to the hub - foreach (var tree in compilation.SyntaxTrees) { - var model = compilation.GetSemanticModel (tree); - // the bindings have A LOT of interfaces, we cannot get a symbol for the entire tree - var declarations = (await tree.GetRootAsync ()) - .DescendantNodes () - .OfType ().ToArray (); - - logger.Debug ("Found '{Declarations}' interfaces in '{FilePath}'", declarations.Length, tree.FilePath); - - // loop over the declarations and send them to the hub - foreach (var declaration in declarations) { - var symbol = model.GetDeclaredSymbol (declaration); - if (symbol is null) { - // skip the transformation because the symbol is null - logger.Warning ("Could not get the symbol for '{Declaration}'", declaration.Identifier); - continue; - } - - if (Skip (tree, symbol, out var outputDirectory)) { - // matched the filter - logger.Information ("Skipping '{SymbolName}' because it was filtered out", symbol.Name); - continue; - } - - // create the destination directory if needed, this is the only location we should be creating directories - Directory.CreateDirectory (outputDirectory); - - var topicName = SelectTopic (symbol); - if (topicName is not null && transformers.TryGetValue (topicName, out var transformer)) { - await hub.PublishAsync (topicName, transformer.CreateMessage (tree, symbol)); - logger.Information ("Published '{SymbolName}' to '{TopicName}'", symbol.Name, topicName); + // TODO: this is a temporary nested loop to exercise the channels, this will be removed + foreach (var (platform, compilation) in compilations) { + foreach (var tree in compilation.SyntaxTrees) { + var model = compilation.GetSemanticModel (tree); + // the bindings have A LOT of interfaces, we cannot get a symbol for the entire tree + var declarations = (await tree.GetRootAsync ()) + .DescendantNodes () + .OfType ().ToArray (); + + logger.Debug ("Found '{Declarations}' interfaces in '{FilePath}'", declarations.Length, tree.FilePath); + + // loop over the declarations and send them to the hub + foreach (var declaration in declarations) { + var symbol = model.GetDeclaredSymbol (declaration); + if (symbol is null) { + // skip the transformation because the symbol is null + logger.Warning ("Could not get the symbol for '{Declaration}'", declaration.Identifier); + continue; + } + + if (Skip (tree, symbol, out var outputDirectory)) { + // matched the filter + logger.Information ("Skipping '{SymbolName}' because it was filtered out", symbol.Name); + continue; + } + + // create the destination directory if needed, this is the only location we should be creating directories + Directory.CreateDirectory (outputDirectory); + + var topicName = SelectTopic (symbol); + if (topicName is not null && transformers.TryGetValue (topicName, out var transformer)) { + await hub.PublishAsync (topicName, transformer.CreateMessage (tree, symbol)); + logger.Information ("Published '{SymbolName}' to '{TopicName}'", symbol.Name, topicName); + } } } } @@ -193,15 +202,20 @@ internal async Task Execute () await hub.CloseAllAsync (); } - public static Task Execute (string destinationDirectory, string rspFile, string workingDirectory, - string sdkDirectory) + static Task + CreateCompilationAsync (ApplePlatform platform, string rspFile, string workingDirectory, string sdkDirectory) + => Task.Run (() => { + logger.Debug ("Executing compilation for '{Platform}'", platform); + return CreateCompilation (platform, rspFile, workingDirectory, sdkDirectory); + }); + + static CompilationResult CreateCompilation ( + ApplePlatform platform, string rspFile, string workingDirectory, string sdkDirectory) { - logger.Information ("Executing transformation"); - // the transformation works as follows. We first need to parse the rsp file to create a compilation - // to do so we relay on the csharp compiler, there is not much we really need to do. If the parsing - // is wrong, we throw an exception. + // perform the compilation on a background thread, that way we can have several of them at the same time var parseResult = CSharpCommandLineParser.Default.ParseRsp ( rspFile, workingDirectory, sdkDirectory); + logger.Information ("Rsp file {RspFile} parsed with {ParseErrors} errors", rspFile, parseResult.Errors.Length); // add NET to the preprocessor directives var preprocessorDirectives = parseResult.ParseOptions.PreprocessorSymbolNames.ToList (); @@ -223,22 +237,45 @@ public static Task Execute (string destinationDirectory, string rspFile, string syntaxTrees: parsedSource, references: references, options: parseResult.CompilationOptions); + var errors = compilation.GetDiagnostics () + .Where (d => d.Severity == DiagnosticSeverity.Error) + .ToImmutableArray (); + return new (platform, compilation, errors); + } - var diagnostics = compilation.GetDiagnostics (); - logger.Debug ("Total diagnostics length {DiagnosticsLength}", diagnostics.Length); - // collect all the compilation errors, ignoring the warnings, if any error is found, we throw an exception - var errors = diagnostics.Where (d => d.Severity == DiagnosticSeverity.Error).ToArray (); - if (errors.Length > 0) { + public static async Task Execute (string destinationDirectory, + List<(ApplePlatform Platform, string RspPath)> rspFiles, string workingDirectory, + string sdkDirectory) + { + logger.Information ("Executing transformation"); + // use the async method to run several compilations in parallel, that way we do not block on eachother + var compilationTasks = new List> (); + foreach (var (platform, rspPath) in rspFiles) { + compilationTasks.Add (CreateCompilationAsync (platform, rspPath, workingDirectory, sdkDirectory)); + } + + var compilations = await Task.WhenAll (compilationTasks); + + // verify we have no errors in the compilations + foreach (var (platform, api, errors) in compilations) { + if (errors.Length == 0) + continue; + + logger.Error ("Compilation failed from {Platform} with {ErrorCount} errors", platform, errors.Length); var sb = new StringBuilder (); - foreach (var resultError in errors) { - sb.AppendLine ($"{resultError}"); + foreach (var error in errors) { + sb.AppendLine (error.ToString ()); } - logger.Error ("Error during workspace compilation: {Error}", sb); - throw new Exception ($"Error during workspace compilation: {sb}"); + + throw new Exception (sb.ToString ()); } + ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilationTuples = [ + .. compilations + .Select (c => c.ToTuple ()) + ]; // create a new transformer with the compilation result and the syntax trees - var transformer = new Transformer (destinationDirectory, compilation); - return transformer.Execute (); + var transformer = new Transformer (destinationDirectory, compilationTuples); + await transformer.Execute (); } } diff --git a/tests/rgen/Microsoft.Macios.Transformer.Tests/TransformerTests.cs b/tests/rgen/Microsoft.Macios.Transformer.Tests/TransformerTests.cs index 3d573705138..5470a1cbc1b 100644 --- a/tests/rgen/Microsoft.Macios.Transformer.Tests/TransformerTests.cs +++ b/tests/rgen/Microsoft.Macios.Transformer.Tests/TransformerTests.cs @@ -81,7 +81,7 @@ public void SkipTests (ApplePlatform platform, (string Source, string Path) sour var symbol = semanticModel.GetDeclaredSymbol (declaration); Assert.NotNull (symbol); - var transformer = new Transformer (targetDirectory, compilation, targetNamespaces); + var transformer = new Transformer (targetDirectory, [(platform, compilation)], targetNamespaces); if (expectedDestination is not null) expectedDestination = Path.Combine (targetDirectory, expectedDestination); Assert.Equal (expectedResult, transformer.Skip (syntaxTree, symbol, out var destination));