Install dotnet tools locally in the Nuget Packages

In commit 7a398b0 we added two new tools for compressing textures, `crunch` and `basisu`.
The problem there is that the management of the `.config/dotnet-tool.json` file
by the users was becoming an issue.  We needed a more automatic way to install
the required tooling.

The problem is using the standard `dotnet tool install` calls requires
a `.config/dotnet-tool.json` file to be present in either the current directory or
a directory that is in the path ABOVE the current one. You would use the
`--create-manifest-if-needed` flag to create the manifest, but that will still
leave the users having to manage and upgrade the .json file every time we do a release.

So lets get the pipeline to install the tooling itself.
The `dotnet tool install` command has an additional argument`--tool-path` this allows
us to say where we want the tool installed. Once that has happened we get a native
binary launcher in `--tool-path` which allows us to launch the app directly without
using the `dotnet` executable. So what this allows us to do is install the tooling
locally in the directory that the content pipeline is installed. This will usually
be the global `.nuget/package` directory.

We need to keep an eye on the `DOTNET_ROOT` environment variable when installing the
tooling, just in case a user (or CI) wants to use a custom dotnet install.

The one downside is users will no longer be able to run `dotnet mgcb` directly in
their project directory unless they install the tooling manually.

The long term plan is to bundle all these tools into a single native library
which can be called directly by the content pipeline.

.github/workflows/main.yml

@@ -105,7 +105,6 @@ jobs:
       - name: Test
         run: dotnet test Tools/MonoGame.Tools.Tests/MonoGame.Tools.Tests.csproj --blame-hang-timeout 5m -c Release --filter="TestCategory!=Audio"
         env:
-          DOTNET_ROOT: ${{github.workspace}}/dotnet64
           MGFXC_WINE_PATH: /Users/runner/.winemonogame
           CI: true
         if: runner.os == 'macOS'
@@ -231,11 +230,6 @@ jobs:
           path: tests-windowsdx
         if: runner.os == 'Windows'
 
-      - name: Install Tools
-        run: |
-          dotnet tool install --create-manifest-if-needed mgcb-basisu
-          dotnet tool install --create-manifest-if-needed mgcb-crunch
-
       - name: Run Tools Tests
         run: dotnet test tests-tools/MonoGame.Tools.Tests.dll --blame-hang-timeout 1m --filter="TestCategory!=Effects"
         env:

MonoGame.Framework.Content.Pipeline/ExternalTool.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Diagnostics;
 using System.IO;
+using System.Net;
 using System.Threading;
 using MonoGame.Framework.Utilities;
 
@@ -17,6 +18,9 @@ namespace Microsoft.Xna.Framework.Content.Pipeline
     /// </summary>
     internal class ExternalTool
     {
+        public static string Crunch = "mgcb-crunch";
+        public static string BasisU = "mgcb-basisu";
+
         public static int Run(string command, string arguments)
         {
             string stdout, stderr;
@@ -27,13 +31,39 @@ public static int Run(string command, string arguments)
             return result;
         }
 
+        public static void RestoreDotnetTool(string command, string toolName)
+        {
+            string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory);
+            if (CurrentPlatform.OS == OS.Linux)
+                path= Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "linux");
+            if (CurrentPlatform.OS == OS.MacOSX)
+                path= Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "osx");
+            if (Directory.Exists (Path.Combine(path, toolName))) 
+                return;
+            Directory.CreateDirectory (path);
+            var exe = CurrentPlatform.OS == OS.Windows ? "dotnet.exe" : "dotnet";
+            var dotnetRoot = Environment.GetEnvironmentVariable ("DOTNET_ROOT");
+            if (!string.IsNullOrEmpty(dotnetRoot)) {
+                exe = Path.Combine(dotnetRoot, exe);
+            }
+            Run (exe, $"tool {command} {toolName} --tool-path .", out string stdOut, out string stdErr,  workingDirectory: path);
+        }
+
+        public static void RestoreDotnetTools()
+        {
+            RestoreDotnetTool("install", Crunch);
+            RestoreDotnetTool("install", BasisU);
+        }
+
         /// <summary>
         /// Run a dotnet tool. The tool should be installed in a .config/dotnet-tools.json file somewhere in the project lineage.
         /// </summary>
         public static int RunDotnetTool(string toolName, string args, out string stdOut, out string stdErr, string stdIn=null, string workingDirectory=null)
         {
-            var finalizedArgs = toolName + " " + args;
-            return ExternalTool.Run("dotnet", finalizedArgs, out stdOut, out stdErr, stdIn, workingDirectory);
+            RestoreDotnetTools ();
+            var exe = FindCommand (toolName);
+            var finalizedArgs =  args;
+            return ExternalTool.Run(exe, finalizedArgs, out stdOut, out stdErr, stdIn, workingDirectory);
         }
 
         public static int Run(string command, string arguments, out string stdout, out string stderr, string stdin = null, string workingDirectory=null)
@@ -68,6 +98,11 @@ public static int Run(string command, string arguments, out string stdout, out s
             if (!string.IsNullOrEmpty(workingDirectory))
                 processInfo.WorkingDirectory = workingDirectory;
 
+            var dotnetRoot = Environment.GetEnvironmentVariable ("DOTNET_ROOT");
+            if (!string.IsNullOrEmpty(dotnetRoot)) {
+                processInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetRoot;
+            }
+
             EnsureExecutable(fullPath);
 
             using (var process = new Process())

MonoGame.Framework.Content.Pipeline/Utilities/BasisUHelpers.cs

@@ -206,7 +206,7 @@ internal static class BasisU
     /// <returns>The exit code for the basisu process. </returns>
     public static int Run(string args, out string stdOut, out string stdErr, string stdIn=null, string workingDirectory=null)
     {
-        return ExternalTool.RunDotnetTool("mgcb-basisu", args, out stdOut, out stdErr, stdIn, workingDirectory);
+        return ExternalTool.RunDotnetTool(ExternalTool.BasisU, args, out stdOut, out stdErr, stdIn, workingDirectory);
     }
 
     /// <summary>

MonoGame.Framework.Content.Pipeline/Utilities/CrunchHelpers.cs

@@ -137,7 +137,7 @@ internal static class Crunch
         /// <returns>The exit code for the basisu process. </returns>
         private static int Run(string args, out string stdOut, out string stdErr)
         {
-            return ExternalTool.RunDotnetTool("mgcb-crunch", args, out stdOut, out stdErr);
+            return ExternalTool.RunDotnetTool(ExternalTool.Crunch, args, out stdOut, out stdErr);
         }
 
         /// <summary>

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.Android.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.DesktopGL.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.WindowsDX.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Application.iOS.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Templates/MonoGame.Templates.CSharp/content/MonoGame.Library.CSharp/.config/dotnet-tools.json

@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "dotnet-mgcb": {
-      "version": "3.8.2.1-develop",
-      "commands": [
-        "mgcb"
-      ]
-    },
     "dotnet-mgcb-editor": {
       "version": "3.8.2.1-develop",
       "commands": [

Tools/MonoGame.Content.Builder.Task/MonoGame.Content.Builder.Task.props

@@ -3,6 +3,7 @@
   <PropertyGroup>
     <DotnetCommand Condition="'$(DotnetCommand)' == ''">dotnet</DotnetCommand>
     <EnableMGCBItems Condition="'$(EnableMGCBItems)' == ''">true</EnableMGCBItems>
+    <MGCBToolDirectory>$(MSBuildThisFileDirectory)dotnet-tools</MGCBToolDirectory>
     <MGCBCommand Condition="'$(MGCBCommand)' == ''">mgcb</MGCBCommand>
   </PropertyGroup>
 

Tools/MonoGame.Content.Builder.Task/MonoGame.Content.Builder.Task.targets

@@ -84,6 +84,12 @@
 
   </Target>
 
+  <!-- Restore the dotnet-mgcb tool to a known location. -->
+  <Target Name="RestoreContentCompiler" Condition="!Exists ('$(MGCBToolDirectory)\$(MGCBCommand)')">
+    <MakeDir Directories="$(MGCBToolDirectory)"/>
+    <Exec Command="&quot;$(DotnetCommand)&quot; tool install dotnet-mgcb --tool-path ." WorkingDirectory="$(MGCBToolDirectory)" />
+  </Target>
+
   <!--
     =====================
     PrepareContentBuilder
@@ -134,16 +140,18 @@
       - ExtraContent: built content files
         - ContentDir: the relative path of the embedded folder to contain the content files
   -->
-  <Target Name="RunContentBuilder" DependsOnTargets="PrepareContentBuilder">
+  <Target Name="RunContentBuilder" DependsOnTargets="RestoreContentCompiler;PrepareContentBuilder">
 
-    <!-- Remove this line if they make dotnet tool restore part of dotnet restore build -->
-    <!-- https://github.com/dotnet/sdk/issues/4241 -->
-    <Exec Command="&quot;$(DotnetCommand)&quot; tool restore" />
+    <PropertyGroup>
+      <_Command Condition="Exists ('$(MGCBToolDirectory)\$(MGCBCommand)')">&quot;$(MGCBToolDirectory)\$(MGCBCommand)&quot;</_Command>
+      <!-- Fallback to old behaviour this allows people to override $(MGCBCommand) with the mgcb.dll -->
+      <_Command Condition=" '$(_Command)' == '' ">&quot;$(DotnetCommand)&quot; &quot;$(MGCBCommand)&quot;</_Command>
+    </PropertyGroup>
 
     <!-- Execute MGCB from the project directory so we use the correct manifest. -->
     <Exec
       Condition="'%(ContentReference.FullPath)' != ''"
-      Command="&quot;$(DotnetCommand)&quot; &quot;$(MGCBCommand)&quot; $(MonoGameMGCBAdditionalArguments) /@:&quot;%(ContentReference.FullPath)&quot; /platform:$(MonoGamePlatform) /outputDir:&quot;%(ContentReference.ContentOutputDir)&quot; /intermediateDir:&quot;%(ContentReference.ContentIntermediateOutputDir)&quot; /workingDir:&quot;%(ContentReference.FullDir)&quot;"
+      Command="$(_Command) $(MonoGameMGCBAdditionalArguments) /@:&quot;%(ContentReference.FullPath)&quot; /platform:$(MonoGamePlatform) /outputDir:&quot;%(ContentReference.ContentOutputDir)&quot; /intermediateDir:&quot;%(ContentReference.ContentIntermediateOutputDir)&quot; /workingDir:&quot;%(ContentReference.FullDir)&quot;"
       WorkingDirectory="$(MSBuildProjectDirectory)" />
 
     <ItemGroup>

build/BuildToolsTasks/BuildContentPipelineTask.cs

@@ -9,11 +9,5 @@ public override void Run(BuildContext context)
     {
         var builderPath = context.GetProjectPath(ProjectType.ContentPipeline);
         context.DotNetPack(builderPath, context.DotNetPackSettings);
-
-        // ensure that the local development has the required dotnet tools.
-        //  this won't actually include the tool manifest in a final build,
-        //  but it will setup a local developer's project
-        context.DotNetTool(builderPath, "tool install --create-manifest-if-needed mgcb-basisu");
-        context.DotNetTool(builderPath, "tool install --create-manifest-if-needed mgcb-crunch");
     }
 }

build/DeployTasks/UploadArtifactsTask.cs

@@ -15,10 +15,52 @@ public override async Task RunAsync(BuildContext context)
             _ => "linux"
         };
 
+        // Clean up build tools if installed
+        // otherwise we get permission issues after extraction
+        // because the zip removes all the permissions.
+        // Plus in windows hidden files (like the .store directory)
+        // are ignored. This causes `dotnet tool` to error.
+        var path = System.IO.Path.Combine(context.BuildOutput, "Tests", "Tools", "Release", "dotnet-tools");
+        if (System.IO.Directory.Exists(path)) {
+            context.Log.Information ($"Deleting: {path}");
+            System.IO.Directory.Delete (path, recursive: true);
+        }
+        if (context.IsRunningOnMacOs())
+        {
+            path = System.IO.Path.Combine(context.BuildOutput, "Tests", "Tools", "Release", "osx");
+            DeleteToolStore (context, path);
+        }
+        if (context.IsRunningOnLinux())
+        {
+            path = System.IO.Path.Combine(context.BuildOutput, "Tests", "Tools", "Release", "linux");
+            DeleteToolStore (context, path);
+        }
+        if (context.IsRunningOnWindows())
+        {
+            path = System.IO.Path.Combine(context.BuildOutput, "Tests", "Tools", "Release");
+            DeleteToolStore (context, path);
+        }
+
+
         await context.GitHubActions().Commands.UploadArtifact(new DirectoryPath(context.NuGetsDirectory.FullPath), $"nuget-{os}");
         await context.GitHubActions().Commands.UploadArtifact(new DirectoryPath(System.IO.Path.Combine(context.BuildOutput, "Tests", "Tools", "Release")), $"tests-tools-{os}");
         await context.GitHubActions().Commands.UploadArtifact(new DirectoryPath(System.IO.Path.Combine(context.BuildOutput, "Tests", "DesktopGL", "Release")), $"tests-desktopgl-{os}");
         if (context.IsRunningOnWindows())
             await context.GitHubActions().Commands.UploadArtifact(new DirectoryPath(System.IO.Path.Combine(context.BuildOutput, "Tests", "WindowsDX", "Release")), $"tests-windowsdx-{os}");
     }
+
+    void DeleteToolStore (BuildContext context, string path)
+    {
+        if (System.IO.Directory.Exists (path)) {
+            var store = System.IO.Path.Combine (path, ".store");
+            if (System.IO.Directory.Exists (store)) {
+                context.Log.Information ($"Deleting: {store}");
+                System.IO.Directory.Delete (store, recursive: true);
+                foreach (var file in System.IO.Directory.GetFiles (path, "mgcb-*", System.IO.SearchOption.TopDirectoryOnly)) {
+                    context.Log.Information ($"Deleting: {file}");
+                    System.IO.File.Delete (file);
+                }
+            }
+        }
+    }
 }