Skip to content

Commit

Permalink
Merge branch 'main' into dev/mandel/transformer-availability
Browse files Browse the repository at this point in the history
  • Loading branch information
@mandel-macaque
mandel-macaque committed Jan 23, 2025
2 parents 8705fb3 + c8f1ff7 commit d949aaa
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 80 deletions.
15 changes: 15 additions & 0 deletions src/ObjCBindings/ExportTag.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,21 @@ public enum Property : Int64 {
/// </summary>
CustomMarshalDirective = 1 << 5,

/// <summary>
/// Apply to strings parameters that are merely retained or assigned,
/// not copied this is an exception as it is advised in the coding
/// standard for Objective-C to avoid this, but a few properties do use
/// this. Use this falg for properties flagged with `retain' or
/// `assign', which look like this:
///
/// @property (retain) NSString foo;
/// @property (assign) NSString assigned;
///
/// This forced the generator to create an NSString before calling the
/// API instead of using the fast string marshalling code.
/// </summary>
DisableZeroCopy = 1 << 6,

}
}

20 changes: 20 additions & 0 deletions src/rgen/Microsoft.Macios.Transformer/CompilationResult.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Collections.Immutable;
using Microsoft.CodeAnalysis;
using Xamarin.Utils;

namespace Microsoft.Macios.Transformer;

/// <summary>
/// Holds the result of a compilation. This is a helper record to make code cleaner in the transformer.
/// </summary>
/// <param name="Platform">The platform the compilation targeted.</param>
/// <param name="Compilation">The compilation result.</param>
/// <param name="Errors">All compilation errors.</param>
record CompilationResult (ApplePlatform Platform, Compilation Compilation, ImmutableArray<Diagnostic> Errors) {

public (ApplePlatform Platform, Compilation compilation) ToTuple ()
=> (Platform, Compilation);
}
67 changes: 46 additions & 21 deletions src/rgen/Microsoft.Macios.Transformer/Main.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Serilog.Core;
using Serilog.Events;
using Serilog.Templates;
using Xamarin.Utils;
using static System.Console;

public class Program {
Expand All @@ -14,7 +15,7 @@ public class Program {
public static int Main (string [] args)
{
// Create options
var rspOption = new Option<string> (
var rspOption = new Option<string []> (
new [] { "--response-file", "--rsp" },
"Path to the RSP file"
);
Expand Down Expand Up @@ -63,16 +64,16 @@ public static int Main (string [] args)
}

// Set handler for parsing and executing
rootCmd.SetHandler (async (rspPath, destPath, workingDirectory, sdkPath, force, verbosity) => {
WriteLine ($"Microsoft.Macios.Transformer v{typeof (Program).Assembly.GetName ().Version}, (c) Microsoft Corporation. All rights reserved.\n");
rootCmd.SetHandler (async (rspPlatformPaths, destPath, workingDirectory, sdkPath, force, verbosity) => {
WriteLine (
$"Microsoft.Macios.Transformer v{typeof (Program).Assembly.GetName ().Version}, (c) Microsoft Corporation. All rights reserved.\n");

// Convert local to absolute, expand ~
rspPath = ToAbsolutePath (rspPath);
workingDirectory = ToAbsolutePath (workingDirectory);
destPath = ToAbsolutePath (destPath);
sdkPath = ToAbsolutePath (sdkPath);

ValidateRsp (rspPath);
ValidateRsp (rspPlatformPaths, out var rspFiles);
ValidateSdk (sdkPath);
ValidateWorkingDirectory (workingDirectory);
ValidateVerbosity (verbosity);
Expand All @@ -88,18 +89,9 @@ public static int Main (string [] args)
"[{@t:HH:mm:ss} {@l:u3} {Substring(SourceContext, LastIndexOf(SourceContext, '.') + 1)} (Thread: {ThreadId})] {@m}\n"))
.CreateLogger ();

// Parse the .rsp file with Roslyn's CSharpCommandLineParser
var args = new [] { $"@{rspPath}" };
var parseResult = CSharpCommandLineParser.Default.Parse (args, workingDirectory, null);
logger.Information ("RSP parsed. Errors: {ParserErrorLength}", parseResult.Errors.Length);
foreach (var resultError in parseResult.Errors) {
logger.Error ("{Error}", resultError);
WriteLine (resultError);
}

await Transformer.Execute (
destinationDirectory: destPath,
rspFile: rspPath,
rspFiles: rspFiles,
workingDirectory: workingDirectory,
sdkDirectory: sdkPath
);
Expand All @@ -119,13 +111,46 @@ static string ToAbsolutePath (string path)
return absolutePath;
}

static void ValidateRsp (string path)
static void ValidateRsp (string [] paths, out List<(ApplePlatform Platform, string RspPath)> rspFiles)
{
if (string.IsNullOrWhiteSpace (path) || !File.Exists (path))
throw new FileNotFoundException ("RSP file not found.");
if ((File.GetAttributes (path) & FileAttributes.Directory) == FileAttributes.Directory)
throw new Exception ("RSP path is a directory.");
using var stream = File.OpenRead (path); // validate readability
rspFiles = [];
// loop over all the strings, split them by the ':' and retrieve the platform and path. Then
// validate the path
foreach (var cmdPath in paths) {
var parts = cmdPath.Split (':');
if (parts.Length != 2)
throw new Exception ("Invalid RSP format. Expected platform:path");
ApplePlatform platform;
var rspPath = ToAbsolutePath (parts [1]);
switch (parts [0]) {
case "ios":
platform = ApplePlatform.iOS;
break;
case "tvos":
platform = ApplePlatform.TVOS;
break;
case "macos":
platform = ApplePlatform.MacOSX;
break;
case "maccatalys":
platform = ApplePlatform.MacCatalyst;
break;
default:
platform = ApplePlatform.None;
break;
}

if (platform == ApplePlatform.None)
throw new Exception ("Invalid platform in RSP file.");

if (string.IsNullOrWhiteSpace (rspPath) || !File.Exists (rspPath))
throw new FileNotFoundException ($"RSP '{rspPath}' file not found.");
if ((File.GetAttributes (rspPath) & FileAttributes.Directory) == FileAttributes.Directory)
throw new Exception ($"RSP {rspPath} is a directory.");
using var stream = File.OpenRead (rspPath); // validate readability
logger.Debug ("Adding RSP file {RspPath} for platform {Platform}", rspPath, platform);
rspFiles.Add ((platform, rspPath));
}
}

static void ValidateWorkingDirectory (string path)
Expand Down
141 changes: 89 additions & 52 deletions src/rgen/Microsoft.Macios.Transformer/Transformer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
using System.Text;
using Marille;
Expand All @@ -11,6 +12,7 @@
using Microsoft.Macios.Transformer.Extensions;
using Microsoft.Macios.Transformer.Workers;
using Serilog;
using Xamarin.Utils;

namespace Microsoft.Macios.Transformer;

Expand All @@ -21,14 +23,16 @@ namespace Microsoft.Macios.Transformer;
class Transformer {
readonly static ILogger logger = Log.ForContext<Transformer> ();
readonly string destinationDirectory;
readonly Compilation compilation;
readonly ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilations;
readonly HashSet<string>? namespaceFilter;
readonly Dictionary<string, ITransformer<(string Path, string SymbolName)>> transformers = new ();

internal Transformer (string destination, Compilation compilationResult, IEnumerable<string>? namespaces = null)
internal Transformer (string destination,
ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilationsResult,
IEnumerable<string>? namespaces = null)
{
destinationDirectory = destination;
compilation = compilationResult;
compilations = compilationsResult;
if (namespaces is not null)
namespaceFilter = new HashSet<string> (namespaces);

Expand Down Expand Up @@ -66,7 +70,8 @@ internal async Task<Hub> CreateHub ()
{
// get the attrs, based on those return the correct topic to use
var attrs = symbol.GetAttributeData ();
logger.Debug ("Symbol '{SymbolName}' has [{Attributes}] attributes", symbol.Name, string.Join (", ", attrs.Keys));
logger.Debug ("Symbol '{SymbolName}' has [{Attributes}] attributes", symbol.Name,
string.Join (", ", attrs.Keys));
logger.Debug ("Symbol '{SymbolName}' kind is '{SymbolKind}'", symbol.Name, symbol.TypeKind);

if (symbol.TypeKind == TypeKind.Enum) {
Expand Down Expand Up @@ -133,6 +138,7 @@ internal bool Skip (SyntaxTree syntaxTree, ISymbol symbol, [NotNullWhen (false)]
// filtered out
return true;
}

outputDirectory = Path.Combine (destinationDirectory, symbolNamespace);
// If the syntax tree comes from the output directory, we skip it because this is a manual binding
return syntaxTree.FilePath.StartsWith (outputDirectory);
Expand All @@ -153,37 +159,40 @@ internal async Task Execute ()
var hub = await CreateHub ();

// with the hub created, loop over the syntax trees and create the messages to be sent to the hub
foreach (var tree in compilation.SyntaxTrees) {
var model = compilation.GetSemanticModel (tree);
// the bindings have A LOT of interfaces, we cannot get a symbol for the entire tree
var declarations = (await tree.GetRootAsync ())
.DescendantNodes ()
.OfType<BaseTypeDeclarationSyntax> ().ToArray ();

logger.Debug ("Found '{Declarations}' interfaces in '{FilePath}'", declarations.Length, tree.FilePath);

// loop over the declarations and send them to the hub
foreach (var declaration in declarations) {
var symbol = model.GetDeclaredSymbol (declaration);
if (symbol is null) {
// skip the transformation because the symbol is null
logger.Warning ("Could not get the symbol for '{Declaration}'", declaration.Identifier);
continue;
}

if (Skip (tree, symbol, out var outputDirectory)) {
// matched the filter
logger.Information ("Skipping '{SymbolName}' because it was filtered out", symbol.Name);
continue;
}

// create the destination directory if needed, this is the only location we should be creating directories
Directory.CreateDirectory (outputDirectory);

var topicName = SelectTopic (symbol);
if (topicName is not null && transformers.TryGetValue (topicName, out var transformer)) {
await hub.PublishAsync (topicName, transformer.CreateMessage (tree, symbol));
logger.Information ("Published '{SymbolName}' to '{TopicName}'", symbol.Name, topicName);
// TODO: this is a temporary nested loop to exercise the channels, this will be removed
foreach (var (platform, compilation) in compilations) {
foreach (var tree in compilation.SyntaxTrees) {
var model = compilation.GetSemanticModel (tree);
// the bindings have A LOT of interfaces, we cannot get a symbol for the entire tree
var declarations = (await tree.GetRootAsync ())
.DescendantNodes ()
.OfType<BaseTypeDeclarationSyntax> ().ToArray ();

logger.Debug ("Found '{Declarations}' interfaces in '{FilePath}'", declarations.Length, tree.FilePath);

// loop over the declarations and send them to the hub
foreach (var declaration in declarations) {
var symbol = model.GetDeclaredSymbol (declaration);
if (symbol is null) {
// skip the transformation because the symbol is null
logger.Warning ("Could not get the symbol for '{Declaration}'", declaration.Identifier);
continue;
}

if (Skip (tree, symbol, out var outputDirectory)) {
// matched the filter
logger.Information ("Skipping '{SymbolName}' because it was filtered out", symbol.Name);
continue;
}

// create the destination directory if needed, this is the only location we should be creating directories
Directory.CreateDirectory (outputDirectory);

var topicName = SelectTopic (symbol);
if (topicName is not null && transformers.TryGetValue (topicName, out var transformer)) {
await hub.PublishAsync (topicName, transformer.CreateMessage (tree, symbol));
logger.Information ("Published '{SymbolName}' to '{TopicName}'", symbol.Name, topicName);
}
}
}
}
Expand All @@ -193,15 +202,20 @@ internal async Task Execute ()
await hub.CloseAllAsync ();
}

public static Task Execute (string destinationDirectory, string rspFile, string workingDirectory,
string sdkDirectory)
static Task<CompilationResult>
CreateCompilationAsync (ApplePlatform platform, string rspFile, string workingDirectory, string sdkDirectory)
=> Task.Run (() => {
logger.Debug ("Executing compilation for '{Platform}'", platform);
return CreateCompilation (platform, rspFile, workingDirectory, sdkDirectory);
});

static CompilationResult CreateCompilation (
ApplePlatform platform, string rspFile, string workingDirectory, string sdkDirectory)
{
logger.Information ("Executing transformation");
// the transformation works as follows. We first need to parse the rsp file to create a compilation
// to do so we relay on the csharp compiler, there is not much we really need to do. If the parsing
// is wrong, we throw an exception.
// perform the compilation on a background thread, that way we can have several of them at the same time
var parseResult = CSharpCommandLineParser.Default.ParseRsp (
rspFile, workingDirectory, sdkDirectory);
logger.Information ("Rsp file {RspFile} parsed with {ParseErrors} errors", rspFile, parseResult.Errors.Length);

// add NET to the preprocessor directives
var preprocessorDirectives = parseResult.ParseOptions.PreprocessorSymbolNames.ToList ();
Expand All @@ -223,22 +237,45 @@ public static Task Execute (string destinationDirectory, string rspFile, string
syntaxTrees: parsedSource,
references: references,
options: parseResult.CompilationOptions);
var errors = compilation.GetDiagnostics ()
.Where (d => d.Severity == DiagnosticSeverity.Error)
.ToImmutableArray ();
return new (platform, compilation, errors);
}

var diagnostics = compilation.GetDiagnostics ();
logger.Debug ("Total diagnostics length {DiagnosticsLength}", diagnostics.Length);
// collect all the compilation errors, ignoring the warnings, if any error is found, we throw an exception
var errors = diagnostics.Where (d => d.Severity == DiagnosticSeverity.Error).ToArray ();
if (errors.Length > 0) {
public static async Task Execute (string destinationDirectory,
List<(ApplePlatform Platform, string RspPath)> rspFiles, string workingDirectory,
string sdkDirectory)
{
logger.Information ("Executing transformation");
// use the async method to run several compilations in parallel, that way we do not block on eachother
var compilationTasks = new List<Task<CompilationResult>> ();
foreach (var (platform, rspPath) in rspFiles) {
compilationTasks.Add (CreateCompilationAsync (platform, rspPath, workingDirectory, sdkDirectory));
}

var compilations = await Task.WhenAll (compilationTasks);

// verify we have no errors in the compilations
foreach (var (platform, api, errors) in compilations) {
if (errors.Length == 0)
continue;

logger.Error ("Compilation failed from {Platform} with {ErrorCount} errors", platform, errors.Length);
var sb = new StringBuilder ();
foreach (var resultError in errors) {
sb.AppendLine ($"{resultError}");
foreach (var error in errors) {
sb.AppendLine (error.ToString ());
}
logger.Error ("Error during workspace compilation: {Error}", sb);
throw new Exception ($"Error during workspace compilation: {sb}");

throw new Exception (sb.ToString ());
}

ImmutableArray<(ApplePlatform Platform, Compilation Compilation)> compilationTuples = [
.. compilations
.Select (c => c.ToTuple ())
];
// create a new transformer with the compilation result and the syntax trees
var transformer = new Transformer (destinationDirectory, compilation);
return transformer.Execute ();
var transformer = new Transformer (destinationDirectory, compilationTuples);
await transformer.Execute ();
}
}
12 changes: 6 additions & 6 deletions tests/rgen/Microsoft.Macios.Transformer.Tests/Microsoft.Macios.Transformer.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
<IsPackable>false</IsPackable>

<!-- Ensure we do have the api-definitions -->
<RootDirectory>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)\..\..\..'))</RootDirectory>
<DotnetBuildDirectory>$(RootDirectory)/src/build/dotnet</DotnetBuildDirectory>
<SourceDirectory>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)\..\..\..\src'))</SourceDirectory>
<DotnetBuildDirectory>build/dotnet</DotnetBuildDirectory>
</PropertyGroup>

<ItemGroup>
Expand Down Expand Up @@ -53,10 +53,10 @@

<Target Name="BuildTestLibraries" BeforeTargets="BeforeBuild">
<Message Text="Processing platform: %(Platform.Identity)" />
<Exec Command="make -j8 -C $(RootDirectory) $(DotnetBuildDirectory)/ios/apidefinition-ios.dll"/>
<Exec Command="make -j8 -C $(RootDirectory) $(DotnetBuildDirectory)/macos/apidefinition-macos.dll"/>
<Exec Command="make -j8 -C $(RootDirectory) $(DotnetBuildDirectory)/tvos/apidefinition-tvos.dll"/>
<Exec Command="make -j8 -C $(RootDirectory) $(DotnetBuildDirectory)/maccatalyst/apidefinition-maccatalyst.dll"/>
<Exec Command="make -j8 -C $(SourceDirectory) $(DotnetBuildDirectory)/ios/apidefinition-ios.dll"/>
<Exec Command="make -j8 -C $(SourceDirectory) $(DotnetBuildDirectory)/macos/apidefinition-macos.dll"/>
<Exec Command="make -j8 -C $(SourceDirectory) $(DotnetBuildDirectory)/tvos/apidefinition-tvos.dll"/>
<Exec Command="make -j8 -C $(SourceDirectory) $(DotnetBuildDirectory)/maccatalyst/apidefinition-maccatalyst.dll"/>
</Target>

</Project>
2 changes: 1 addition & 1 deletion tests/rgen/Microsoft.Macios.Transformer.Tests/TransformerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void SkipTests (ApplePlatform platform, (string Source, string Path) sour
var symbol = semanticModel.GetDeclaredSymbol (declaration);
Assert.NotNull (symbol);

var transformer = new Transformer (targetDirectory, compilation, targetNamespaces);
var transformer = new Transformer (targetDirectory, [(platform, compilation)], targetNamespaces);
if (expectedDestination is not null)
expectedDestination = Path.Combine (targetDirectory, expectedDestination);
Assert.Equal (expectedResult, transformer.Skip (syntaxTree, symbol, out var destination));
Expand Down

0 comments on commit d949aaa

Please sign in to comment.