From b75a304a35231814098ac832f89cab38e7849637 Mon Sep 17 00:00:00 2001 From: Charlie Poole Date: Tue, 18 Feb 2025 14:41:24 -0800 Subject: [PATCH] Enhanced Package Testing --- GitVersion.yml | 2 +- recipe/build-settings.cake | 13 ++- recipe/chocolatey-package.cake | 6 +- recipe/dotnet-tool-package.cake | 8 +- recipe/nuget-package.cake | 6 +- recipe/output-checks.cake | 115 +++++++++++++++++++ recipe/package-definition.cake | 84 +++++++++----- recipe/package-test.cake | 72 ++++-------- recipe/publishing.cake | 2 + recipe/test-reports.cake | 99 ++++++++++------- recipe/test-results.cake | 6 +- recipe/test-runners.cake | 190 +++++++++++++++++++------------- recipe/zip-package.cake | 8 +- 13 files changed, 392 insertions(+), 219 deletions(-) create mode 100644 recipe/output-checks.cake diff --git a/GitVersion.yml b/GitVersion.yml index d1a659c..0e0bf78 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,5 +1,5 @@ # This copy of GitVersion.yml is used in building the recipe package itself. -next-version: 1.3.0 +next-version: 1.4.0 mode: ContinuousDelivery legacy-semver-padding: 5 build-metadata-padding: 5 diff --git a/recipe/build-settings.cake b/recipe/build-settings.cake index 1e84984..cdc3dca 100644 --- a/recipe/build-settings.cake +++ b/recipe/build-settings.cake @@ -365,16 +365,17 @@ public static class BuildSettings foreach (var package in Packages) { Console.WriteLine(package.PackageId); - Console.WriteLine(" PackageType: " + package.PackageType); - Console.WriteLine(" PackageFileName: " + package.PackageFileName); - Console.WriteLine(" PackageInstallDirectory: " + package.PackageInstallDirectory); - Console.WriteLine(" PackageTestDirectory: " + package.PackageTestDirectory); + Console.WriteLine(" PackageType: " + package.PackageType); + Console.WriteLine(" PackageFileName: " + package.PackageFileName); + Console.WriteLine(" PackageInstallDirectory: " + package.PackageInstallDirectory); + Console.WriteLine(" PackageTestDirectory: " + package.PackageTestDirectory); + Console.WriteLine(" ExtensionInstallDirectory: " + package.ExtensionInstallDirectory); } var selected = SelectedPackages.Select(p => p.PackageId); if (CommandLineOptions.PackageSelector.Exists) - Console.WriteLine(" SelectedPackages: " + string.Join(", ", selected.ToArray())); + Console.WriteLine(" SelectedPackages: " + string.Join(", ", selected.ToArray())); else - Console.WriteLine(" SelectedPackages: NO SELECTOR SPECIFIED"); + Console.WriteLine(" SelectedPackages: NO SELECTOR SPECIFIED"); Console.WriteLine("\nPUBLISHING"); Console.WriteLine("ShouldPublishToMyGet: " + ShouldPublishToMyGet); diff --git a/recipe/chocolatey-package.cake b/recipe/chocolatey-package.cake index 7645d1b..cfe1dd5 100644 --- a/recipe/chocolatey-package.cake +++ b/recipe/chocolatey-package.cake @@ -4,7 +4,7 @@ public class ChocolateyPackage : PackageDefinition string id, string source, IPackageTestRunner testRunner = null, - TestRunnerSource testRunnerSource = null, + IPackageTestRunner[] testRunners = null, PackageCheck[] checks = null, IEnumerable tests = null) : base( @@ -12,7 +12,7 @@ public class ChocolateyPackage : PackageDefinition id, source, testRunner: testRunner, - testRunnerSource: testRunnerSource, + testRunners: testRunners, checks: checks, tests: tests) { @@ -29,7 +29,7 @@ public class ChocolateyPackage : PackageDefinition // The directory used to contain results of package tests for this package public override string PackageResultDirectory => BuildSettings.ChocolateyResultDirectory + PackageId + "/"; // The directory into which extensions to the test runner are installed - public override string ExtensionInstallDirectory => BuildSettings.PackageTestDirectory; + public override string ExtensionInstallDirectory => BuildSettings.ChocolateyTestDirectory; public override void BuildPackage() { diff --git a/recipe/dotnet-tool-package.cake b/recipe/dotnet-tool-package.cake index d31ba60..64db3e5 100644 --- a/recipe/dotnet-tool-package.cake +++ b/recipe/dotnet-tool-package.cake @@ -6,7 +6,7 @@ public class DotNetToolPackage : NuGetPackage string source, string basePath = null, IPackageTestRunner testRunner = null, - TestRunnerSource testRunnerSource = null, + IPackageTestRunner[] testRunners = null, PackageCheck[] checks = null, PackageCheck[] symbols = null, IEnumerable tests = null) @@ -15,17 +15,19 @@ public class DotNetToolPackage : NuGetPackage source, basePath: basePath, testRunner: testRunner, - testRunnerSource: testRunnerSource, + testRunners: testRunners, checks: checks, symbols: symbols, tests: tests) { } + public override string PackageTestDirectory => PackageInstallDirectory; + public override void InstallPackage() { var arguments = $"tool install {PackageId} --version {BuildSettings.PackageVersion} " + - $"--add-source \"{BuildSettings.PackageDirectory}\" --tool-path \"{PackageTestDirectory}\""; + $"--add-source \"{BuildSettings.PackageDirectory}\" --tool-path \"{PackageInstallDirectory}\""; Console.WriteLine($"Executing dotnet {arguments}"); _context.StartProcess("dotnet", arguments); } diff --git a/recipe/nuget-package.cake b/recipe/nuget-package.cake index e3f5363..636db60 100644 --- a/recipe/nuget-package.cake +++ b/recipe/nuget-package.cake @@ -5,7 +5,7 @@ public class NuGetPackage : PackageDefinition string source, string basePath = null, IPackageTestRunner testRunner = null, - TestRunnerSource testRunnerSource = null, + IPackageTestRunner[] testRunners = null, PackageCheck[] checks = null, PackageCheck[] symbols = null, IEnumerable tests = null) @@ -15,7 +15,7 @@ public class NuGetPackage : PackageDefinition source, basePath: basePath, testRunner: testRunner, - testRunnerSource: testRunnerSource, + testRunners: testRunners, checks: checks, symbols: symbols, tests: tests) @@ -39,7 +39,7 @@ public class NuGetPackage : PackageDefinition // The directory used to contain results of package tests for this package public override string PackageResultDirectory => BuildSettings.NuGetResultDirectory + PackageId + "/"; // The directory into which extensions to the test runner are installed - public override string ExtensionInstallDirectory => BuildSettings.PackageTestDirectory; + public override string ExtensionInstallDirectory => BuildSettings.NuGetTestDirectory; public override void BuildPackage() { diff --git a/recipe/output-checks.cake b/recipe/output-checks.cake new file mode 100644 index 0000000..1b019a0 --- /dev/null +++ b/recipe/output-checks.cake @@ -0,0 +1,115 @@ +////////////////////////////////////////////////////////////////////// +// SYNTAX FOR EXPRESSING A STRING CONSTRAINT +////////////////////////////////////////////////////////////////////// + +//public static class StringConstraints +//{ +// private string _outputText; + +// public static bool Contains(this string output) +// { +// return new StringCOnt(); +// } + +// public OutputCheck(string outputText) +// { +// _outputText = outputText; +// } + +// public bool Contains(string content) +// { +// return output.Contains(content); +// } +//} + +// OutputCheck is used to check content of redirected package test output +public abstract class OutputCheck +{ + protected string _expectedText; + protected int _atleast; + protected int _exactly; + + public OutputCheck(string expectedText, int atleast = 1, int exactly = -1) + { + _expectedText = expectedText; + _atleast = atleast; + _exactly = exactly; + } + + public bool Matches(IEnumerable output) => Matches(string.Join("\r\n", output)); + + public abstract bool Matches(string output); + + public string Message { get; protected set; } +} + +public class OutputContains : OutputCheck +{ + public OutputContains(string expectedText, int atleast = 1, int exactly = -1) : base(expectedText, atleast, exactly) { } + + public override bool Matches(string output) + { + int found = 0; + + int index = output.IndexOf(_expectedText); + int textLength = _expectedText.Length; + int outputLength = output.Length; + while (index >= 0 && index < output.Length - textLength) + { + ++found; + index += textLength; + index = output.IndexOf(_expectedText, index); + } + + if (_atleast > 0 && found >= _atleast || _exactly > 0 && found == _exactly) + return true; + + var sb = new StringBuilder(" Expected: "); + if (_atleast > 0) + { + sb.Append($"at least {_atleast} "); + sb.Append(_atleast == 1 ? "line " : "lines "); + sb.Append($"containing \"{_expectedText}\" but found {found}"); + } + else + { + sb.Append($"exactly {_exactly} "); + sb.Append(_exactly == 1 ? "line " : "lines "); + sb.Append($"containing \"{_expectedText}\" but found {found}"); + } + //sb.Append(_atleast > 0 ? "at least " : "exactly "); + //sb.Append() + //else + // { + // Message = $"at least {_atleast} {_atleast = 1 ? "line" : "lines"} containing \"{_expectedText}\" but found {found}"; + // return false; + // } + + if (_exactly > 0) + if (found == _exactly) + return true; + else + { + Message = $" Expected: at least one line containing \"{_expectedText}\" But none were found"; + return false; + } + + return false; + } +} + +public class OutputDoesNotContain : OutputCheck +{ + public OutputDoesNotContain(string expectedText) : base(expectedText) { } + + public override bool Matches(string output) + { + if (output.Contains(_expectedText)) + { + Message = $" Expected: no lines containing \"{_expectedText}\" But at least one was found"; + return false; + } + + return true; + } +} diff --git a/recipe/package-definition.cake b/recipe/package-definition.cake index 222bb28..ea9ff54 100644 --- a/recipe/package-definition.cake +++ b/recipe/package-definition.cake @@ -25,16 +25,16 @@ public abstract class PackageDefinition string source, string basePath = null, // Defaults to OutputDirectory IPackageTestRunner testRunner = null, - TestRunnerSource testRunnerSource = null, + IPackageTestRunner[] testRunners = null, string extraTestArguments = null, PackageCheck[] checks = null, PackageCheck[] symbols = null, IEnumerable tests = null) { - if (testRunner == null && testRunnerSource == null && tests != null) - throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: TestRunner or TestRunnerSource must be provided if there are tests."); - if (testRunner != null && testRunnerSource != null) - throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: Either TestRunner or TestRunnerSource must be provided, but not both."); + if (testRunner == null && testRunners == null && tests != null) + throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: TestRunner or TestRunners must be provided if there are tests."); + if (testRunner != null && testRunners != null) + throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: Either TestRunner or TestRunners must be provided, but not both."); _context = BuildSettings.Context; @@ -44,7 +44,7 @@ public abstract class PackageDefinition PackageSource = source; BasePath = basePath ?? BuildSettings.OutputDirectory; TestRunner = testRunner; - TestRunnerSource = testRunnerSource; + TestRunners = testRunners; ExtraTestArguments = extraTestArguments; PackageChecks = checks; SymbolChecks = symbols; @@ -56,8 +56,13 @@ public abstract class PackageDefinition public string PackageVersion { get; protected set; } public string PackageSource { get; } public string BasePath { get; } + + // Defaults to null unless the package sets it. + public PackageReference[] BundledExtensions { get; protected set; } = null; + public bool HasBundledExtensions => BundledExtensions != null; + public IPackageTestRunner TestRunner { get; } - public TestRunnerSource TestRunnerSource { get; } + public IPackageTestRunner[] TestRunners { get; } public string ExtraTestArguments { get; } public PackageCheck[] PackageChecks { get; } public PackageCheck[] SymbolChecks { get; protected set; } @@ -74,9 +79,10 @@ public abstract class PackageDefinition public abstract string PackageResultDirectory { get; } // The directory into which extensions to the test runner are installed public abstract string ExtensionInstallDirectory { get; } - + // The directory containing the package executable after installation + public virtual string PackageTestDirectory => $"{PackageInstallDirectory}{PackageId}.{PackageVersion}/"; + public string PackageFilePath => BuildSettings.PackageDirectory + PackageFileName; - public string PackageTestDirectory => $"{PackageInstallDirectory}{PackageId}.{PackageVersion}/"; public bool IsSelectedBy(string selectionExpression) { @@ -200,21 +206,17 @@ public abstract class PackageDefinition _context.CleanDirectory(PackageResultDirectory); - // Ensure we start out each package with no extensions installed. - // If any package test installs an extension, it remains available - // for subsequent tests of the same package only. - //foreach (DirectoryPath dirPath in _context.GetDirectories(ExtensionInstallDirectory + "*")) - //{ - // _context.DeleteDirectory(dirPath, new DeleteDirectorySettings() { Recursive = true }); - // _context.Information("Deleted directory " + dirPath.GetDirectoryName()); - //} + // Ensure we start out each package with no extensions installed. + // If any package test installs an extension, it remains available + // for subsequent tests of the same package only. + RemoveExtensions(); - // Package was defined with either a TestRunnerSource or a single TestRunner. In either - // case, these will all be package test runners and may or may not require installation. - var defaultRunners = TestRunnerSource ?? new TestRunnerSource((TestRunner)TestRunner); + // Package was defined with one or more TestRunners. These + // may or may not require installation. + var defaultRunners = TestRunners ?? new[] { TestRunner }; // Preinstall all runners requiring installation - InstallRunners(defaultRunners.PackageTestRunners); + InstallRunners(defaultRunners); foreach (var packageTest in PackageTests) { @@ -225,13 +227,13 @@ public abstract class PackageDefinition InstallRunners(packageTest.TestRunners); // Use runners from the test if provided, otherwise the default runners - var runners = packageTest.TestRunners.Length > 0 ? packageTest.TestRunners : defaultRunners.PackageTestRunners; - + var runners = packageTest.TestRunners.Length > 0 ? packageTest.TestRunners : defaultRunners; + foreach (var runner in runners) { Console.WriteLine(runner.Version); - var testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/"; - var resultFile = testResultDir + "TestResult.xml"; + string testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/"; + string resultFile = testResultDir + "TestResult.xml"; Banner.Display(packageTest.Description); @@ -239,13 +241,15 @@ public abstract class PackageDefinition string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}"; if (CommandLineOptions.TraceLevel.Value != "Off") arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}"; + bool redirectOutput = packageTest.OutputCheck != null; - int rc = runner.RunPackageTest(arguments); + int rc = runner.RunPackageTest(arguments, redirectOutput); + var actualResult = packageTest.ExpectedResult != null ? new ActualResult(resultFile) : null; + try { - var result = new ActualResult(resultFile); - var report = new PackageTestReport(packageTest, result, runner); + var report = new PackageTestReport(packageTest, actualResult, runner); reporter.AddReport(report); Console.WriteLine(report.Errors.Count == 0 @@ -258,6 +262,15 @@ public abstract class PackageDefinition Console.WriteLine("\nERROR: No result found!"); } + + //else + //{ + // var report = new PackageTestReport(packageTest, rc, runner); + // reporter.AddReport(report); + + // if (rc != packageTest.ExpectedReturnCode) + // Console.WriteLine($"\nERROR: Expected rc = {packageTest.ExpectedReturnCode} but got {rc}!"); + //} } } @@ -283,6 +296,21 @@ public abstract class PackageDefinition extension.InstallExtension(this); } + // Remove all extensions prior to starting a run. Note that we avoid removing the the + // package being developed, which may actually be an extension itself. + protected void RemoveExtensions() + { + foreach (DirectoryPath dirPath in _context.GetDirectories(ExtensionInstallDirectory + "*")) + { + string dirName = dirPath.Segments.Last(); + if ((dirName.StartsWith("NUnit.Extension.") || dirName.StartsWith("nunit-extension-")) && !dirName.StartsWith(PackageId)) + { + _context.DeleteDirectory(dirPath, new DeleteDirectorySettings() { Recursive = true }); + _context.Information("Deleted directory " + dirPath.GetDirectoryName()); + } + } + } + private void InstallRunners(IEnumerable runners) { // Install any runners needing installation diff --git a/recipe/package-test.cake b/recipe/package-test.cake index 98c2917..cfa1d4e 100644 --- a/recipe/package-test.cake +++ b/recipe/package-test.cake @@ -4,73 +4,39 @@ // 1 Run for all CI tests - that is every time we test packages // 2 Run only on PRs, dev builds and when publishing // 3 Run only when publishing -public struct PackageTest +public class PackageTest { - public int Level; - public string Name; - public string Description; - public string Arguments; - public ExpectedResult ExpectedResult; - public IPackageTestRunner[] TestRunners; - public ExtensionSpecifier[] ExtensionsNeeded; + public int Level { get; private set; } + public string Name { get; private set; } - public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult ) - { - if (name == null) - throw new ArgumentNullException(nameof(name)); - if (description == null) - throw new ArgumentNullException(nameof(description)); - if (arguments == null) - throw new ArgumentNullException(nameof(arguments)); - if (expectedResult == null) - throw new ArgumentNullException(nameof(expectedResult)); - - Level = level; - Name = name; - Description = description; - Arguments = arguments; - ExpectedResult = expectedResult; - ExtensionsNeeded = new ExtensionSpecifier[0]; - TestRunners = new IPackageTestRunner[0]; - } + public string Description { get; set; } + public TestRunner TestRunner { get; set; } + public string Arguments { get; set; } + public int ExpectedReturnCode { get; set; } = 0; + public ExpectedResult ExpectedResult { get; set; } + public OutputCheck OutputCheck { get; set; } + public ExtensionSpecifier[] ExtensionsNeeded { get; set; } = new ExtensionSpecifier[0]; + public IPackageTestRunner[] TestRunners { get; set; } = new IPackageTestRunner[0]; - public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params ExtensionSpecifier[] extensionsNeeded ) + public PackageTest(int level, string name) { if (name == null) throw new ArgumentNullException(nameof(name)); - if (description == null) - throw new ArgumentNullException(nameof(description)); - if (arguments == null) - throw new ArgumentNullException(nameof(arguments)); - if (expectedResult == null) - throw new ArgumentNullException(nameof(expectedResult)); - Level = level; + Level = level; Name = name; - Description = description; - Arguments = arguments; - ExpectedResult = expectedResult; - ExtensionsNeeded = extensionsNeeded; - TestRunners = new IPackageTestRunner[0]; + Description = name; } +} - public PackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params IPackageTestRunner[] testRunners ) +public class MultipleRunnerPackageTest : PackageTest +{ + public MultipleRunnerPackageTest(int level, string name, string description, string arguments, ExpectedResult expectedResult, params IPackageTestRunner[] testRunners ) + : base(level, name) { - if (name == null) - throw new ArgumentNullException(nameof(name)); - if (description == null) - throw new ArgumentNullException(nameof(description)); - if (arguments == null) - throw new ArgumentNullException(nameof(arguments)); - if (expectedResult == null) - throw new ArgumentNullException(nameof(expectedResult)); - - Level = level; - Name = name; Description = description; Arguments = arguments; ExpectedResult = expectedResult; TestRunners = testRunners; - ExtensionsNeeded = new ExtensionSpecifier[0]; } } diff --git a/recipe/publishing.cake b/recipe/publishing.cake index e780160..fd6a30a 100644 --- a/recipe/publishing.cake +++ b/recipe/publishing.cake @@ -1,3 +1,5 @@ +using Cake.Git; + public static class PackageReleaseManager { private static ICakeContext _context; diff --git a/recipe/test-reports.cake b/recipe/test-reports.cake index 6bffa45..8ec9306 100644 --- a/recipe/test-reports.cake +++ b/recipe/test-reports.cake @@ -2,67 +2,92 @@ public class PackageTestReport { public PackageTest Test; public ActualResult Result; - public ITestRunner Runner; + public IPackageTestRunner Runner; public List Errors = new List(); public List Warnings = new List(); - public PackageTestReport(PackageTest test, ActualResult actualResult, ITestRunner runner = null) + public PackageTestReport(PackageTest test, ActualResult actualResult, IPackageTestRunner runner = null) { Test = test; Result = actualResult; Runner = runner; var expectedResult = test.ExpectedResult; + var expectedOutput = test.OutputCheck; - ReportMissingFiles(); + if (expectedResult != null) + { + ReportMissingFiles(); + + if (actualResult.OverallResult == null) + Errors.Add(" The test-run element has no result attribute."); + else if (expectedResult.OverallResult != actualResult.OverallResult) + Errors.Add($" Expected: Overall Result = {expectedResult.OverallResult} But was: {actualResult.OverallResult}"); + CheckCounter("Test Count", expectedResult.Total, actualResult.Total); + CheckCounter("Passed", expectedResult.Passed, actualResult.Passed); + CheckCounter("Failed", expectedResult.Failed, actualResult.Failed); + CheckCounter("Warnings", expectedResult.Warnings, actualResult.Warnings); + CheckCounter("Inconclusive", expectedResult.Inconclusive, actualResult.Inconclusive); + CheckCounter("Skipped", expectedResult.Skipped, actualResult.Skipped); + + var expectedAssemblies = expectedResult.Assemblies; + var actualAssemblies = actualResult.Assemblies; + + for (int i = 0; i < expectedAssemblies.Length && i < actualAssemblies.Length; i++) + { + var expected = expectedAssemblies[i]; + var actual = actualAssemblies[i]; + + if (expected.AssemblyName != actual.AssemblyName) + Errors.Add($" Expected: {expected.AssemblyName} But was: {actual.AssemblyName}"); + else if (runner == null || runner.PackageId == "NUnit.ConsoleRunner.NetCore") + { + if (actual.Runtime == null) + Warnings.Add($"Unable to determine actual runtime used for {expected.AssemblyName}"); + else if (expected.Runtime != actual.Runtime) + Errors.Add($" Assembly {actual.AssemblyName} Expected: {expected.Runtime} But was: {actual.Runtime}"); + } + } - if (actualResult.OverallResult == null) - Errors.Add(" The test-run element has no result attribute."); - else if (expectedResult.OverallResult != actualResult.OverallResult) - Errors.Add($" Expected: Overall Result = {expectedResult.OverallResult} But was: {actualResult.OverallResult}"); - CheckCounter("Test Count", expectedResult.Total, actualResult.Total); - CheckCounter("Passed", expectedResult.Passed, actualResult.Passed); - CheckCounter("Failed", expectedResult.Failed, actualResult.Failed); - CheckCounter("Warnings", expectedResult.Warnings, actualResult.Warnings); - CheckCounter("Inconclusive", expectedResult.Inconclusive, actualResult.Inconclusive); - CheckCounter("Skipped", expectedResult.Skipped, actualResult.Skipped); + for (int i = actualAssemblies.Length; i < expectedAssemblies.Length; i++) + Errors.Add($" Assembly {expectedAssemblies[i].AssemblyName} was not found"); - var expectedAssemblies = expectedResult.Assemblies; - var actualAssemblies = actualResult.Assemblies; + for (int i = expectedAssemblies.Length; i < actualAssemblies.Length; i++) + Errors.Add($" Found unexpected assembly {actualAssemblies[i].AssemblyName}"); + } - for (int i = 0; i < expectedAssemblies.Length && i < actualAssemblies.Length; i++) + if (expectedOutput != null) { - var expected = expectedAssemblies[i]; - var actual = actualAssemblies[i]; - - if (expected.AssemblyName != actual.AssemblyName) - Errors.Add($" Expected: {expected.AssemblyName} But was: { actual.AssemblyName}"); - else if (runner == null || runner.PackageId == "NUnit.ConsoleRunner.NetCore") - { - if (actual.Runtime == null) - Warnings.Add($"Unable to determine actual runtime used for {expected.AssemblyName}"); - else if (expected.Runtime != actual.Runtime) - Errors.Add($" Assembly {actual.AssemblyName} Expected: {expected.Runtime} But was: {actual.Runtime}"); - } + if (!expectedOutput.Matches(runner.Output)) + Errors.Add(expectedOutput.Message); } + } - for (int i = actualAssemblies.Length; i < expectedAssemblies.Length; i++) - Errors.Add($" Assembly {expectedAssemblies[i].AssemblyName} was not found"); + public PackageTestReport(PackageTest test, int rc, IPackageTestRunner runner = null) + { + Test = test; + Result = null; + Runner = runner; - for (int i = expectedAssemblies.Length; i < actualAssemblies.Length; i++) - Errors.Add($" Found unexpected assembly {actualAssemblies[i].AssemblyName}"); + if (rc != test.ExpectedReturnCode) + Errors.Add($" Expected: rc = {test.ExpectedReturnCode} But was: {rc}"); + else if (test.OutputCheck != null) + { + if (!test.OutputCheck.Matches(runner.Output)) + Errors.Add(test.OutputCheck.Message); + } } - public PackageTestReport(PackageTest test, Exception ex, ITestRunner runner = null) + public PackageTestReport(PackageTest test, Exception ex, IPackageTestRunner runner = null) { Test = test; Result = null; Runner = runner; - - Errors.Add($" {ex.Message}"); - } - public void Display(int index, TextWriter writer) + Errors.Add($" {ex.Message}"); + } + + public void Display(int index, TextWriter writer) { writer.WriteLine(); writer.WriteLine($"{index}. {Test.Description}"); diff --git a/recipe/test-results.cake b/recipe/test-results.cake index 41ba053..28e255b 100644 --- a/recipe/test-results.cake +++ b/recipe/test-results.cake @@ -3,7 +3,7 @@ using System.Xml; -public abstract class ResultSummary +public abstract class TestResultSummary { public string OverallResult { get; set; } public int Total { get; set; } @@ -14,7 +14,7 @@ public abstract class ResultSummary public int Skipped { get; set; } } -public class ExpectedResult : ResultSummary +public class ExpectedResult : TestResultSummary { public ExpectedResult(string overallResult) { @@ -43,7 +43,7 @@ public class ExpectedAssemblyResult public string Runtime { get; } } -public class ActualResult : ResultSummary +public class ActualResult : TestResultSummary { public ActualResult(string resultFile) { diff --git a/recipe/test-runners.cake b/recipe/test-runners.cake index ae16003..d97339f 100644 --- a/recipe/test-runners.cake +++ b/recipe/test-runners.cake @@ -2,70 +2,95 @@ // TEST RUNNER INTERFACES ///////////////////////////////////////////////////////////////////////////// -/// -/// Common interface for all test runners -/// -public interface ITestRunner -{ - string PackageId { get; } - string Version { get; } -} - /// /// A runner capable of running unit tests /// -public interface IUnitTestRunner : ITestRunner +public interface IUnitTestRunner { + string PackageId { get; } + string Version { get; } + int RunUnitTest(FilePath testPath); } /// /// A runner capable of running package tests /// -public interface IPackageTestRunner : ITestRunner +public interface IPackageTestRunner { - int RunPackageTest(string arguments); + string PackageId { get; } + string Version { get; } + + IEnumerable Output { get; } + + int RunPackageTest(string arguments, bool redirectOutput = false); } ///////////////////////////////////////////////////////////////////////////// -// ABSTRACT TEST RUNNER +// TEST RUNNER BASE CLASS ///////////////////////////////////////////////////////////////////////////// /// /// The TestRunner class is the abstract base for all TestRunners used to run unit- -/// or package-tests. A TestRunner knows how to run a test assembly and provide a result. -/// All base functionality is implemented in this class. Derived classes make that -/// functionality available selectively by implementing specific interfaces. +/// or package-tests. A TestRunner knows how to run both types of tests. The reason +/// for this design is that some derived runners are used for unit tests, others for +/// package tests and still others for both types. Derived classes implement one or +/// both interfaces to indicate what they support. /// -public abstract class TestRunner : ITestRunner +public abstract class TestRunner { protected ICakeContext Context => BuildSettings.Context; public string PackageId { get; protected set; } public string Version { get; protected set; } - protected int RunTest(FilePath executablePath, string arguments = null) - { - return RunTest(executablePath, new ProcessSettings { Arguments = arguments }); - } + public IEnumerable Output { get; protected set; } + + protected int RunUnitTest(FilePath executablePath, ProcessSettings processSettings) + { + if (executablePath == null) + throw new ArgumentNullException(nameof(executablePath)); - protected int RunTest(FilePath executablePath, ProcessSettings processSettings=null) + if (processSettings == null) + throw new ArgumentNullException(nameof(processSettings)); + + // Add default values to settings if not present + if (processSettings.WorkingDirectory == null) + processSettings.WorkingDirectory = BuildSettings.OutputDirectory; + + return Context.StartProcess(executablePath, processSettings); + } + + protected int RunPackageTest(FilePath executablePath, ProcessSettings processSettings) { if (executablePath == null) throw new ArgumentNullException(nameof(executablePath)); - if (processSettings == null) - processSettings = new ProcessSettings(); - + if (processSettings == null) + throw new ArgumentNullException(nameof(processSettings)); + // Add default values to settings if not present - if (processSettings.WorkingDirectory == null) + if (processSettings.WorkingDirectory == null) processSettings.WorkingDirectory = BuildSettings.OutputDirectory; - if (executablePath.GetExtension() == ".dll") - return Context.StartProcess("dotnet", processSettings); - else - return Context.StartProcess(executablePath, processSettings); + // Was Output Requested? + if (processSettings.RedirectStandardOutput) + processSettings.RedirectedStandardOutputHandler = OutputHandler; + + IEnumerable output; + // If Redirected Output was not requested, output will be null + int rc = Context.StartProcess(executablePath, processSettings, out output); + Output = output; + return rc; } + + internal string OutputHandler(string output) + { + // Ensure that package test output displays and is also re-directed. + // If the derive class doesn't need the output, it doesn't retrieve it. + Console.WriteLine(output); + return output; + } } /// @@ -79,7 +104,7 @@ public abstract class InstallableTestRunner : TestRunner Version = version; } - protected abstract FilePath ExecutableRelativePath { get; } + protected FilePath ExecutableRelativePath { get; set; } // Path under tools directory where package would be installed by Cake #tool directive. // NOTE: When used to run unit tests, a #tool directive is required. If derived package @@ -117,23 +142,13 @@ public abstract class InstallableTestRunner : TestRunner /// public class TestRunnerSource { - public TestRunnerSource(TestRunner runner1, params TestRunner[] moreRunners) + public TestRunnerSource(IPackageTestRunner runner1, params IPackageTestRunner[] moreRunners) { - AllRunners.Add(runner1); - AllRunners.AddRange(moreRunners); + Runners.Add(runner1); + Runners.AddRange(moreRunners); } - public List AllRunners { get; } = new List(); - - public IEnumerable UnitTestRunners - { - get { foreach(var runner in AllRunners.Where(r => r is IUnitTestRunner)) yield return (IUnitTestRunner)runner; } - } - - public IEnumerable PackageTestRunners - { - get { foreach(var runner in AllRunners.Where(r => r is IPackageTestRunner)) yield return (IPackageTestRunner)runner; } - } + public List Runners = new List(); } ///////////////////////////////////////////////////////////////////////////// @@ -152,7 +167,7 @@ public class NUnitLiteRunner : TestRunner, IUnitTestRunner { "NUNIT_INTERNAL_TRACE_LEVEL", CommandLineOptions.TraceLevel.Value } }; - return RunTest(testPath, processSettings); + return base.RunUnitTest(testPath, processSettings); } } @@ -164,43 +179,64 @@ public class NUnitLiteRunner : TestRunner, IUnitTestRunner // in the tools directory by use of a #tools directive. public class NUnitConsoleRunner : InstallableTestRunner, IUnitTestRunner, IPackageTestRunner { - protected override FilePath ExecutableRelativePath => "tools/nunit4-console.exe"; - - public NUnitConsoleRunner(string version) : base("NUnit.ConsoleRunner", version) { } + public NUnitConsoleRunner(string version) : base("NUnit.ConsoleRunner", version) + { + ExecutableRelativePath = version[0] == '3' ? "tools/nunit3-console.exe" : "tools/nunit-console.exe"; + } - // Run a unit test - public int RunUnitTest(FilePath testPath) => RunTest(ToolInstallDirectory.CombineWithFilePath(ExecutableRelativePath), $"\"{testPath}\" {BuildSettings.UnitTestArguments}"); + // Run a unit test + public int RunUnitTest(FilePath testPath) => + base.RunUnitTest( + ToolInstallDirectory.CombineWithFilePath(ExecutableRelativePath), + new ProcessSettings { Arguments = $"\"{testPath}\" {BuildSettings.UnitTestArguments}" }); - // Run a package test - public int RunPackageTest(string arguments) => RunTest(ExecutablePath, arguments); + // Run a package test + public int RunPackageTest(string arguments, bool redirectStandardOutput = false) => + base.RunPackageTest(ExecutablePath, new ProcessSettings { Arguments = arguments, RedirectStandardOutput = redirectStandardOutput }); } public class NUnitNetCoreConsoleRunner : InstallableTestRunner, IUnitTestRunner, IPackageTestRunner { - protected override FilePath ExecutableRelativePath => "tools/net6.0/nunit4-console.exe"; - - public NUnitNetCoreConsoleRunner(string version) : base("NUnit.ConsoleRunner.NetCore", version) { } + public NUnitNetCoreConsoleRunner(string version) : base("NUnit.ConsoleRunner.NetCore", version) + { + ExecutableRelativePath = version[0] == '3' ? "tools/net8.0/nunit3-console.exe" : "tools/nunit-netcore-console.exe"; + } - // Run a unit test - public int RunUnitTest(FilePath testPath) => RunTest(ExecutablePath, $"\"{testPath}\" {BuildSettings.UnitTestArguments}"); + // Run a unit test + public int RunUnitTest(FilePath testPath) => base.RunUnitTest( + ExecutablePath, + new ProcessSettings { Arguments = $"\"{testPath}\" {BuildSettings.UnitTestArguments}" }); - // Run a package test - public int RunPackageTest(string arguments) => RunTest(ExecutablePath, arguments); + // Run a package test + public int RunPackageTest(string arguments, bool redirectOutput) => base.RunPackageTest( + ExecutablePath, + new ProcessSettings { Arguments = arguments, RedirectStandardOutput = redirectOutput }); } -public class EngineExtensionTestRunner : TestRunner, IPackageTestRunner -{ - private IPackageTestRunner[] _runners = new IPackageTestRunner[] { - new NUnitConsoleRunner("3.17.0"), - new NUnitConsoleRunner("3.15.5") - }; +//public class EngineExtensionTestRunner : TestRunner, IPackageTestRunner +//{ +// private IPackageTestRunner[] _runners = new IPackageTestRunner[] { +// new NUnitConsoleRunner("3.17.0"), +// new NUnitConsoleRunner("3.15.5") +// }; - public int RunPackageTest(string arguments) - { - - return _runners[0].RunPackageTest(arguments); - } -} +// public int RunPackageTest(string arguments) +// { + +// return _runners[0].RunPackageTest(arguments); +// } + +// public int RunPackageTest(string arguments, out string output) +// { +// var settings = new ProcessSettings +// { +// Arguments = arguments, +// RedirectStandardOutput = true +// }; + +// return RunTest(ExecutablePath, settings, out output); +// } +//} ///////////////////////////////////////////////////////////////////////////// // AGENT RUNNER @@ -222,12 +258,14 @@ public class AgentRunner : TestRunner, IPackageTestRunner _x86Executable = x86Executable; } - public int RunPackageTest(string arguments) + public int RunPackageTest(string arguments, bool redirectOutput = false) { - _executablePath = arguments.Contains("--x86") + _executablePath = arguments.Contains("--x86") ? _x86Executable : _stdExecutable; - return base.RunTest(_executablePath, arguments.Replace("--x86", string.Empty)); - } + return base.RunPackageTest( + _executablePath, + new ProcessSettings { Arguments = arguments.Replace("--x86", string.Empty), RedirectStandardOutput = redirectOutput }); + } } diff --git a/recipe/zip-package.cake b/recipe/zip-package.cake index 15e249f..9abad2f 100644 --- a/recipe/zip-package.cake +++ b/recipe/zip-package.cake @@ -4,7 +4,7 @@ public class ZipPackage : PackageDefinition string id, string source, IPackageTestRunner testRunner = null, - TestRunnerSource testRunnerSource = null, + IPackageTestRunner[] testRunners = null, PackageCheck[] checks = null, IEnumerable tests = null, PackageReference[] bundledExtensions = null ) @@ -13,17 +13,13 @@ public class ZipPackage : PackageDefinition id, source, testRunner: testRunner, - testRunnerSource: testRunnerSource, + testRunners: testRunners, checks: checks, tests: tests) { BundledExtensions = bundledExtensions; } - // ZIP package supports bundling of extensions - public PackageReference[] BundledExtensions { get; } - public bool HasBundledExtensions => BundledExtensions != null; - // The file name of this package, including extension public override string PackageFileName => $"{PackageId}-{PackageVersion}.zip"; // The directory into which this package is installed