Skip to content

Commit

Permalink
[Rgen] Compile all platforms provided in parallel. (#22024)
Browse files Browse the repository at this point in the history 
Allow the command line to be used with more than one rsp. That allows to
pass all the rsp for the different platforms we support allowing us to
merge the compilation results later to be used by the transformation.

---------

Co-authored-by: GitHub Actions Autoformatter <[email protected]>
  • Loading branch information
@mandel-macaque
mandel-macaque and GitHub Actions Autoformatter authored Jan 21, 2025
1 parent 9516bf1 commit 0ab0743
Show file tree
Hide file tree
Showing 4 changed files with 156 additions and 74 deletions.
20 changes: 20 additions & 0 deletions src/rgen/Microsoft.Macios.Transformer/CompilationResult.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Collections.Immutable;
using Microsoft.CodeAnalysis;
using Xamarin.Utils;

namespace Microsoft.Macios.Transformer;

/// <summary>
/// Holds the result of a compilation. This is a helper record to make code cleaner in the transformer.
/// </summary>
/// <param name="Platform">The platform the compilation targeted.</param>
/// <param name="Compilation">The compilation result.</param>
/// <param name="Errors">All compilation errors.</param>
record CompilationResult (ApplePlatform Platform, Compilation Compilation, ImmutableArray<Diagnostic> Errors) {

public (ApplePlatform Platform, Compilation compilation) ToTuple ()
=> (Platform, Compilation);
}
67 changes: 46 additions & 21 deletions src/rgen/Microsoft.Macios.Transformer/Main.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Serilog.Core;
using Serilog.Events;
using Serilog.Templates;
using Xamarin.Utils;
using static System.Console;

public class Program {
Expand All @@ -14,7 +15,7 @@ public class Program {
public static int Main (string [] args)
{
// Create options
var rspOption = new Option<string> (
var rspOption = new Option<string []> (
new [] { "--response-file", "--rsp" },
"Path to the RSP file"
);
Expand Down Expand Up @@ -63,16 +64,16 @@ public static int Main (string [] args)
}

// Set handler for parsing and executing
rootCmd.SetHandler (async (rspPath, destPath, workingDirectory, sdkPath, force, verbosity) => {
WriteLine ($"Microsoft.Macios.Transformer v{typeof (Program).Assembly.GetName ().Version}, (c) Microsoft Corporation. All rights reserved.\n");
rootCmd.SetHandler (async (rspPlatformPaths, destPath, workingDirectory, sdkPath, force, verbosity) => {
WriteLine (
$"Microsoft.Macios.Transformer v{typeof (Program).Assembly.GetName ().Version}, (c) Microsoft Corporation. All rights reserved.\n");

// Convert local to absolute, expand ~
rspPath = ToAbsolutePath (rspPath);
workingDirectory = ToAbsolutePath (workingDirectory);
destPath = ToAbsolutePath (destPath);
sdkPath = ToAbsolutePath (sdkPath);

ValidateRsp (rspPath);
ValidateRsp (rspPlatformPaths, out var rspFiles);
ValidateSdk (sdkPath);
ValidateWorkingDirectory (workingDirectory);
ValidateVerbosity (verbosity);
Expand All @@ -88,18 +89,9 @@ public static int Main (string [] args)
"[{@t:HH:mm:ss} {@l:u3} {Substring(SourceContext, LastIndexOf(SourceContext, '.') + 1)} (Thread: {ThreadId})] {@m}\n"))
.CreateLogger ();

// Parse the .rsp file with Roslyn's CSharpCommandLineParser
var args = new [] { $"@{rspPath}" };
var parseResult = CSharpCommandLineParser.Default.Parse (args, workingDirectory, null);
logger.Information ("RSP parsed. Errors: {ParserErrorLength}", parseResult.Errors.Length);
foreach (var resultError in parseResult.Errors) {
logger.Error ("{Error}", resultError);
WriteLine (resultError);
}

await Transformer.Execute (
destinationDirectory: destPath,
rspFile: rspPath,
rspFiles: rspFiles,
workingDirectory: workingDirectory,
sdkDirectory: sdkPath
);
Expand All @@ -119,13 +111,46 @@ static string ToAbsolutePath (string path)
return absolutePath;
}

static void ValidateRsp (string path)
static void ValidateRsp (string [] paths, out List<(ApplePlatform Platform, string RspPath)> rspFiles)
{
if (string.IsNullOrWhiteSpace (path) || !File.Exists (path))
throw new FileNotFoundException ("RSP file not found.");
if ((File.GetAttributes (path) & FileAttributes.Directory) == FileAttributes.Directory)
throw new Exception ("RSP path is a directory.");
using var stream = File.OpenRead (path); // validate readability
rspFiles = [];
// loop over all the strings, split them by the ':' and retrieve the platform and path. Then
// validate the path
foreach (var cmdPath in paths) {
var parts = cmdPath.Split (':');
if (parts.Length != 2)
throw new Exception ("Invalid RSP format. Expected platform:path");
ApplePlatform platform;
var rspPath = ToAbsolutePath (parts [1]);
switch (parts [0]) {
case "ios":
platform = ApplePlatform.iOS;
break;
case "tvos":
platform = ApplePlatform.TVOS;
break;
case "macos":
platform = ApplePlatform.MacOSX;
break;
case "maccatalys":
platform = ApplePlatform.MacCatalyst;
break;
default:
platform = ApplePlatform.None;
break;
}

if (platform == ApplePlatform.None)
throw new Exception ("Invalid platform in RSP file.");

if (string.IsNullOrWhiteSpace (rspPath) || !File.Exists (rspPath))
throw new FileNotFoundException ($"RSP '{rspPath}' file not found.");
if ((File.GetAttributes (rspPath) & FileAttributes.Directory) == FileAttributes.Directory)
throw new Exception ($"RSP {rspPath} is a directory.");
using var stream = File.OpenRead (rspPath); // validate readability
logger.Debug ("Adding RSP file {RspPath} for platform {Platform}", rspPath, platform);
rspFiles.Add ((platform, rspPath));
}
}

static void ValidateWorkingDirectory (string path)
Expand Down
141 changes: 89 additions & 52 deletions src/rgen/Microsoft.Macios.Transformer/Transformer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.Text;
using Marille;
Expand All @@ -11,6 +12,7 @@
using Microsoft.Macios.Transformer.Extensions;
using Microsoft.Macios.Transformer.Workers;
using Serilog;
using Xamarin.Utils;

namespace Microsoft.Macios.Transformer;

Expand All @@ -21,14 +23,16 @@ namespace Microsoft.Macios.Transformer;
class Transformer {
readonly static ILogger logger = Log.ForContext<Transformer> ();
readonly string destinationDirectory;
readonly Compilation compilation;
readonly ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilations;
readonly HashSet<string>? namespaceFilter;
readonly Dictionary<string, ITransformer<(string Path, string SymbolName)>> transformers = new ();

internal Transformer (string destination, Compilation compilationResult, IEnumerable<string>? namespaces = null)
internal Transformer (string destination,
ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilationsResult,
IEnumerable<string>? namespaces = null)
{
destinationDirectory = destination;
compilation = compilationResult;
compilations = compilationsResult;
if (namespaces is not null)
namespaceFilter = new HashSet<string> (namespaces);

Expand Down Expand Up @@ -66,7 +70,8 @@ internal async Task<Hub> CreateHub ()
{
// get the attrs, based on those return the correct topic to use
var attrs = symbol.GetAttributeData ();
logger.Debug ("Symbol '{SymbolName}' has [{Attributes}] attributes", symbol.Name, string.Join (", ", attrs.Keys));
logger.Debug ("Symbol '{SymbolName}' has [{Attributes}] attributes", symbol.Name,
string.Join (", ", attrs.Keys));
logger.Debug ("Symbol '{SymbolName}' kind is '{SymbolKind}'", symbol.Name, symbol.TypeKind);

if (symbol.TypeKind == TypeKind.Enum) {
Expand Down Expand Up @@ -133,6 +138,7 @@ internal bool Skip (SyntaxTree syntaxTree, ISymbol symbol, [NotNullWhen (false)]
// filtered out
return true;
}

outputDirectory = Path.Combine (destinationDirectory, symbolNamespace);
// If the syntax tree comes from the output directory, we skip it because this is a manual binding
return syntaxTree.FilePath.StartsWith (outputDirectory);
Expand All @@ -153,37 +159,40 @@ internal async Task Execute ()
var hub = await CreateHub ();

// with the hub created, loop over the syntax trees and create the messages to be sent to the hub
foreach (var tree in compilation.SyntaxTrees) {
var model = compilation.GetSemanticModel (tree);
// the bindings have A LOT of interfaces, we cannot get a symbol for the entire tree
var declarations = (await tree.GetRootAsync ())
.DescendantNodes ()
.OfType<BaseTypeDeclarationSyntax> ().ToArray ();

logger.Debug ("Found '{Declarations}' interfaces in '{FilePath}'", declarations.Length, tree.FilePath);

// loop over the declarations and send them to the hub
foreach (var declaration in declarations) {
var symbol = model.GetDeclaredSymbol (declaration);
if (symbol is null) {
// skip the transformation because the symbol is null
logger.Warning ("Could not get the symbol for '{Declaration}'", declaration.Identifier);
continue;
}

if (Skip (tree, symbol, out var outputDirectory)) {
// matched the filter
logger.Information ("Skipping '{SymbolName}' because it was filtered out", symbol.Name);
continue;
}

// create the destination directory if needed, this is the only location we should be creating directories
Directory.CreateDirectory (outputDirectory);

var topicName = SelectTopic (symbol);
if (topicName is not null && transformers.TryGetValue (topicName, out var transformer)) {
await hub.PublishAsync (topicName, transformer.CreateMessage (tree, symbol));
logger.Information ("Published '{SymbolName}' to '{TopicName}'", symbol.Name, topicName);
// TODO: this is a temporary nested loop to exercise the channels, this will be removed
foreach (var (platform, compilation) in compilations) {
foreach (var tree in compilation.SyntaxTrees) {
var model = compilation.GetSemanticModel (tree);
// the bindings have A LOT of interfaces, we cannot get a symbol for the entire tree
var declarations = (await tree.GetRootAsync ())
.DescendantNodes ()
.OfType<BaseTypeDeclarationSyntax> ().ToArray ();

logger.Debug ("Found '{Declarations}' interfaces in '{FilePath}'", declarations.Length, tree.FilePath);

// loop over the declarations and send them to the hub
foreach (var declaration in declarations) {
var symbol = model.GetDeclaredSymbol (declaration);
if (symbol is null) {
// skip the transformation because the symbol is null
logger.Warning ("Could not get the symbol for '{Declaration}'", declaration.Identifier);
continue;
}

if (Skip (tree, symbol, out var outputDirectory)) {
// matched the filter
logger.Information ("Skipping '{SymbolName}' because it was filtered out", symbol.Name);
continue;
}

// create the destination directory if needed, this is the only location we should be creating directories
Directory.CreateDirectory (outputDirectory);

var topicName = SelectTopic (symbol);
if (topicName is not null && transformers.TryGetValue (topicName, out var transformer)) {
await hub.PublishAsync (topicName, transformer.CreateMessage (tree, symbol));
logger.Information ("Published '{SymbolName}' to '{TopicName}'", symbol.Name, topicName);
}
}
}
}
Expand All @@ -193,15 +202,20 @@ internal async Task Execute ()
await hub.CloseAllAsync ();
}

public static Task Execute (string destinationDirectory, string rspFile, string workingDirectory,
string sdkDirectory)
static Task<CompilationResult>
CreateCompilationAsync (ApplePlatform platform, string rspFile, string workingDirectory, string sdkDirectory)
=> Task.Run (() => {
logger.Debug ("Executing compilation for '{Platform}'", platform);
return CreateCompilation (platform, rspFile, workingDirectory, sdkDirectory);
});

static CompilationResult CreateCompilation (
ApplePlatform platform, string rspFile, string workingDirectory, string sdkDirectory)
{
logger.Information ("Executing transformation");
// the transformation works as follows. We first need to parse the rsp file to create a compilation
// to do so we relay on the csharp compiler, there is not much we really need to do. If the parsing
// is wrong, we throw an exception.
// perform the compilation on a background thread, that way we can have several of them at the same time
var parseResult = CSharpCommandLineParser.Default.ParseRsp (
rspFile, workingDirectory, sdkDirectory);
logger.Information ("Rsp file {RspFile} parsed with {ParseErrors} errors", rspFile, parseResult.Errors.Length);

// add NET to the preprocessor directives
var preprocessorDirectives = parseResult.ParseOptions.PreprocessorSymbolNames.ToList ();
Expand All @@ -223,22 +237,45 @@ public static Task Execute (string destinationDirectory, string rspFile, string
syntaxTrees: parsedSource,
references: references,
options: parseResult.CompilationOptions);
var errors = compilation.GetDiagnostics ()
.Where (d => d.Severity == DiagnosticSeverity.Error)
.ToImmutableArray ();
return new (platform, compilation, errors);
}

var diagnostics = compilation.GetDiagnostics ();
logger.Debug ("Total diagnostics length {DiagnosticsLength}", diagnostics.Length);
// collect all the compilation errors, ignoring the warnings, if any error is found, we throw an exception
var errors = diagnostics.Where (d => d.Severity == DiagnosticSeverity.Error).ToArray ();
if (errors.Length > 0) {
public static async Task Execute (string destinationDirectory,
List<(ApplePlatform Platform, string RspPath)> rspFiles, string workingDirectory,
string sdkDirectory)
{
logger.Information ("Executing transformation");
// use the async method to run several compilations in parallel, that way we do not block on eachother
var compilationTasks = new List<Task<CompilationResult>> ();
foreach (var (platform, rspPath) in rspFiles) {
compilationTasks.Add (CreateCompilationAsync (platform, rspPath, workingDirectory, sdkDirectory));
}

var compilations = await Task.WhenAll (compilationTasks);

// verify we have no errors in the compilations
foreach (var (platform, api, errors) in compilations) {
if (errors.Length == 0)
continue;

logger.Error ("Compilation failed from {Platform} with {ErrorCount} errors", platform, errors.Length);
var sb = new StringBuilder ();
foreach (var resultError in errors) {
sb.AppendLine ($"{resultError}");
foreach (var error in errors) {
sb.AppendLine (error.ToString ());
}
logger.Error ("Error during workspace compilation: {Error}", sb);
throw new Exception ($"Error during workspace compilation: {sb}");

throw new Exception (sb.ToString ());
}

ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilationTuples = [
.. compilations
.Select (c => c.ToTuple ())
];
// create a new transformer with the compilation result and the syntax trees
var transformer = new Transformer (destinationDirectory, compilation);
return transformer.Execute ();
var transformer = new Transformer (destinationDirectory, compilationTuples);
await transformer.Execute ();
}
}
2 changes: 1 addition & 1 deletion tests/rgen/Microsoft.Macios.Transformer.Tests/TransformerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void SkipTests (ApplePlatform platform, (string Source, string Path) sour
var symbol = semanticModel.GetDeclaredSymbol (declaration);
Assert.NotNull (symbol);

var transformer = new Transformer (targetDirectory, compilation, targetNamespaces);
var transformer = new Transformer (targetDirectory, [(platform, compilation)], targetNamespaces);
if (expectedDestination is not null)
expectedDestination = Path.Combine (targetDirectory, expectedDestination);
Assert.Equal (expectedResult, transformer.Skip (syntaxTree, symbol, out var destination));
Expand Down

0 comments on commit 0ab0743

Please sign in to comment.