Skip to content

Commit

Permalink
Enhanced Package Testing
Browse files Browse the repository at this point in the history
  • Loading branch information
CharliePoole committed Feb 22, 2025
1 parent 5091c5d commit b75a304
Show file tree
Hide file tree
Showing 13 changed files with 392 additions and 219 deletions.
2 changes: 1 addition & 1 deletion GitVersion.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This copy of GitVersion.yml is used in building the recipe package itself.
next-version: 1.3.0
next-version: 1.4.0
mode: ContinuousDelivery
legacy-semver-padding: 5
build-metadata-padding: 5
Expand Down
13 changes: 7 additions & 6 deletions recipe/build-settings.cake
Original file line number Diff line number Diff line change
Expand Up @@ -365,16 +365,17 @@ public static class BuildSettings
foreach (var package in Packages)
{
Console.WriteLine(package.PackageId);
Console.WriteLine(" PackageType: " + package.PackageType);
Console.WriteLine(" PackageFileName: " + package.PackageFileName);
Console.WriteLine(" PackageInstallDirectory: " + package.PackageInstallDirectory);
Console.WriteLine(" PackageTestDirectory: " + package.PackageTestDirectory);
Console.WriteLine(" PackageType: " + package.PackageType);
Console.WriteLine(" PackageFileName: " + package.PackageFileName);
Console.WriteLine(" PackageInstallDirectory: " + package.PackageInstallDirectory);
Console.WriteLine(" PackageTestDirectory: " + package.PackageTestDirectory);
Console.WriteLine(" ExtensionInstallDirectory: " + package.ExtensionInstallDirectory);
}
var selected = SelectedPackages.Select(p => p.PackageId);
if (CommandLineOptions.PackageSelector.Exists)
Console.WriteLine(" SelectedPackages: " + string.Join(", ", selected.ToArray()));
Console.WriteLine(" SelectedPackages: " + string.Join(", ", selected.ToArray()));
else
Console.WriteLine(" SelectedPackages: NO SELECTOR SPECIFIED");
Console.WriteLine(" SelectedPackages: NO SELECTOR SPECIFIED");

Console.WriteLine("\nPUBLISHING");
Console.WriteLine("ShouldPublishToMyGet: " + ShouldPublishToMyGet);
Expand Down
6 changes: 3 additions & 3 deletions recipe/chocolatey-package.cake
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ public class ChocolateyPackage : PackageDefinition
string id,
string source,
IPackageTestRunner testRunner = null,
TestRunnerSource testRunnerSource = null,
IPackageTestRunner[] testRunners = null,
PackageCheck[] checks = null,
IEnumerable<PackageTest> tests = null)
: base(
PackageType.Chocolatey,
id,
source,
testRunner: testRunner,
testRunnerSource: testRunnerSource,
testRunners: testRunners,
checks: checks,
tests: tests)
{
Expand All @@ -29,7 +29,7 @@ public class ChocolateyPackage : PackageDefinition
// The directory used to contain results of package tests for this package
public override string PackageResultDirectory => BuildSettings.ChocolateyResultDirectory + PackageId + "/";
// The directory into which extensions to the test runner are installed
public override string ExtensionInstallDirectory => BuildSettings.PackageTestDirectory;
public override string ExtensionInstallDirectory => BuildSettings.ChocolateyTestDirectory;

public override void BuildPackage()
{
Expand Down
8 changes: 5 additions & 3 deletions recipe/dotnet-tool-package.cake
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ public class DotNetToolPackage : NuGetPackage
string source,
string basePath = null,
IPackageTestRunner testRunner = null,
TestRunnerSource testRunnerSource = null,
IPackageTestRunner[] testRunners = null,
PackageCheck[] checks = null,
PackageCheck[] symbols = null,
IEnumerable<PackageTest> tests = null)
Expand All @@ -15,17 +15,19 @@ public class DotNetToolPackage : NuGetPackage
source,
basePath: basePath,
testRunner: testRunner,
testRunnerSource: testRunnerSource,
testRunners: testRunners,
checks: checks,
symbols: symbols,
tests: tests)
{
}

public override string PackageTestDirectory => PackageInstallDirectory;

public override void InstallPackage()
{
var arguments = $"tool install {PackageId} --version {BuildSettings.PackageVersion} " +
$"--add-source \"{BuildSettings.PackageDirectory}\" --tool-path \"{PackageTestDirectory}\"";
$"--add-source \"{BuildSettings.PackageDirectory}\" --tool-path \"{PackageInstallDirectory}\"";
Console.WriteLine($"Executing dotnet {arguments}");
_context.StartProcess("dotnet", arguments);
}
Expand Down
6 changes: 3 additions & 3 deletions recipe/nuget-package.cake
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ public class NuGetPackage : PackageDefinition
string source,
string basePath = null,
IPackageTestRunner testRunner = null,
TestRunnerSource testRunnerSource = null,
IPackageTestRunner[] testRunners = null,
PackageCheck[] checks = null,
PackageCheck[] symbols = null,
IEnumerable<PackageTest> tests = null)
Expand All @@ -15,7 +15,7 @@ public class NuGetPackage : PackageDefinition
source,
basePath: basePath,
testRunner: testRunner,
testRunnerSource: testRunnerSource,
testRunners: testRunners,
checks: checks,
symbols: symbols,
tests: tests)
Expand All @@ -39,7 +39,7 @@ public class NuGetPackage : PackageDefinition
// The directory used to contain results of package tests for this package
public override string PackageResultDirectory => BuildSettings.NuGetResultDirectory + PackageId + "/";
// The directory into which extensions to the test runner are installed
public override string ExtensionInstallDirectory => BuildSettings.PackageTestDirectory;
public override string ExtensionInstallDirectory => BuildSettings.NuGetTestDirectory;

public override void BuildPackage()
{
Expand Down
115 changes: 115 additions & 0 deletions recipe/output-checks.cake
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
//////////////////////////////////////////////////////////////////////
// SYNTAX FOR EXPRESSING A STRING CONSTRAINT
//////////////////////////////////////////////////////////////////////

//public static class StringConstraints
//{
// private string _outputText;

// public static bool Contains(this string output)
// {
// return new StringCOnt();
// }

// public OutputCheck(string outputText)
// {
// _outputText = outputText;
// }

// public bool Contains(string content)
// {
// return output.Contains(content);
// }
//}

// OutputCheck is used to check content of redirected package test output
public abstract class OutputCheck
{
protected string _expectedText;
protected int _atleast;
protected int _exactly;

public OutputCheck(string expectedText, int atleast = 1, int exactly = -1)
{
_expectedText = expectedText;
_atleast = atleast;
_exactly = exactly;
}

public bool Matches(IEnumerable<string> output) => Matches(string.Join("\r\n", output));

public abstract bool Matches(string output);

public string Message { get; protected set; }
}

public class OutputContains : OutputCheck
{
public OutputContains(string expectedText, int atleast = 1, int exactly = -1) : base(expectedText, atleast, exactly) { }

public override bool Matches(string output)
{
int found = 0;

int index = output.IndexOf(_expectedText);
int textLength = _expectedText.Length;
int outputLength = output.Length;
while (index >= 0 && index < output.Length - textLength)
{
++found;
index += textLength;
index = output.IndexOf(_expectedText, index);
}

if (_atleast > 0 && found >= _atleast || _exactly > 0 && found == _exactly)
return true;

var sb = new StringBuilder(" Expected: ");
if (_atleast > 0)
{
sb.Append($"at least {_atleast} ");
sb.Append(_atleast == 1 ? "line " : "lines ");
sb.Append($"containing \"{_expectedText}\" but found {found}");
}
else
{
sb.Append($"exactly {_exactly} ");
sb.Append(_exactly == 1 ? "line " : "lines ");
sb.Append($"containing \"{_expectedText}\" but found {found}");
}
//sb.Append(_atleast > 0 ? "at least " : "exactly ");
//sb.Append()
//else
// {
// Message = $"at least {_atleast} {_atleast = 1 ? "line" : "lines"} containing \"{_expectedText}\" but found {found}";
// return false;
// }

if (_exactly > 0)
if (found == _exactly)
return true;
else
{
Message = $" Expected: at least one line containing \"{_expectedText}\" But none were found";
return false;
}

return false;
}
}

public class OutputDoesNotContain : OutputCheck
{
public OutputDoesNotContain(string expectedText) : base(expectedText) { }

public override bool Matches(string output)
{
if (output.Contains(_expectedText))
{
Message = $" Expected: no lines containing \"{_expectedText}\" But at least one was found";
return false;
}

return true;
}
}
84 changes: 56 additions & 28 deletions recipe/package-definition.cake
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@ public abstract class PackageDefinition
string source,
string basePath = null, // Defaults to OutputDirectory
IPackageTestRunner testRunner = null,
TestRunnerSource testRunnerSource = null,
IPackageTestRunner[] testRunners = null,
string extraTestArguments = null,
PackageCheck[] checks = null,
PackageCheck[] symbols = null,
IEnumerable<PackageTest> tests = null)
{
if (testRunner == null && testRunnerSource == null && tests != null)
throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: TestRunner or TestRunnerSource must be provided if there are tests.");
if (testRunner != null && testRunnerSource != null)
throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: Either TestRunner or TestRunnerSource must be provided, but not both.");
if (testRunner == null && testRunners == null && tests != null)
throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: TestRunner or TestRunners must be provided if there are tests.");
if (testRunner != null && testRunners != null)
throw new System.InvalidOperationException($"Unable to create {packageType} package {id}: Either TestRunner or TestRunners must be provided, but not both.");

_context = BuildSettings.Context;

Expand All @@ -44,7 +44,7 @@ public abstract class PackageDefinition
PackageSource = source;
BasePath = basePath ?? BuildSettings.OutputDirectory;
TestRunner = testRunner;
TestRunnerSource = testRunnerSource;
TestRunners = testRunners;
ExtraTestArguments = extraTestArguments;
PackageChecks = checks;
SymbolChecks = symbols;
Expand All @@ -56,8 +56,13 @@ public abstract class PackageDefinition
public string PackageVersion { get; protected set; }
public string PackageSource { get; }
public string BasePath { get; }

// Defaults to null unless the package sets it.
public PackageReference[] BundledExtensions { get; protected set; } = null;
public bool HasBundledExtensions => BundledExtensions != null;

public IPackageTestRunner TestRunner { get; }
public TestRunnerSource TestRunnerSource { get; }
public IPackageTestRunner[] TestRunners { get; }
public string ExtraTestArguments { get; }
public PackageCheck[] PackageChecks { get; }
public PackageCheck[] SymbolChecks { get; protected set; }
Expand All @@ -74,9 +79,10 @@ public abstract class PackageDefinition
public abstract string PackageResultDirectory { get; }
// The directory into which extensions to the test runner are installed
public abstract string ExtensionInstallDirectory { get; }

// The directory containing the package executable after installation
public virtual string PackageTestDirectory => $"{PackageInstallDirectory}{PackageId}.{PackageVersion}/";

public string PackageFilePath => BuildSettings.PackageDirectory + PackageFileName;
public string PackageTestDirectory => $"{PackageInstallDirectory}{PackageId}.{PackageVersion}/";

public bool IsSelectedBy(string selectionExpression)
{
Expand Down Expand Up @@ -200,21 +206,17 @@ public abstract class PackageDefinition

_context.CleanDirectory(PackageResultDirectory);

// Ensure we start out each package with no extensions installed.
// If any package test installs an extension, it remains available
// for subsequent tests of the same package only.
//foreach (DirectoryPath dirPath in _context.GetDirectories(ExtensionInstallDirectory + "*"))
//{
// _context.DeleteDirectory(dirPath, new DeleteDirectorySettings() { Recursive = true });
// _context.Information("Deleted directory " + dirPath.GetDirectoryName());
//}
// Ensure we start out each package with no extensions installed.
// If any package test installs an extension, it remains available
// for subsequent tests of the same package only.
RemoveExtensions();

// Package was defined with either a TestRunnerSource or a single TestRunner. In either
// case, these will all be package test runners and may or may not require installation.
var defaultRunners = TestRunnerSource ?? new TestRunnerSource((TestRunner)TestRunner);
// Package was defined with one or more TestRunners. These
// may or may not require installation.
var defaultRunners = TestRunners ?? new[] { TestRunner };

// Preinstall all runners requiring installation
InstallRunners(defaultRunners.PackageTestRunners);
InstallRunners(defaultRunners);

foreach (var packageTest in PackageTests)
{
Expand All @@ -225,27 +227,29 @@ public abstract class PackageDefinition
InstallRunners(packageTest.TestRunners);

// Use runners from the test if provided, otherwise the default runners
var runners = packageTest.TestRunners.Length > 0 ? packageTest.TestRunners : defaultRunners.PackageTestRunners;
var runners = packageTest.TestRunners.Length > 0 ? packageTest.TestRunners : defaultRunners;

foreach (var runner in runners)
{
Console.WriteLine(runner.Version);
var testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/";
var resultFile = testResultDir + "TestResult.xml";
string testResultDir = $"{PackageResultDirectory}/{packageTest.Name}/";
string resultFile = testResultDir + "TestResult.xml";

Banner.Display(packageTest.Description);

_context.CreateDirectory(testResultDir);
string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}";
if (CommandLineOptions.TraceLevel.Value != "Off")
arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}";
bool redirectOutput = packageTest.OutputCheck != null;

int rc = runner.RunPackageTest(arguments);
int rc = runner.RunPackageTest(arguments, redirectOutput);

var actualResult = packageTest.ExpectedResult != null ? new ActualResult(resultFile) : null;

try
{
var result = new ActualResult(resultFile);
var report = new PackageTestReport(packageTest, result, runner);
var report = new PackageTestReport(packageTest, actualResult, runner);
reporter.AddReport(report);

Console.WriteLine(report.Errors.Count == 0
Expand All @@ -258,6 +262,15 @@ public abstract class PackageDefinition

Console.WriteLine("\nERROR: No result found!");
}

//else
//{
// var report = new PackageTestReport(packageTest, rc, runner);
// reporter.AddReport(report);

// if (rc != packageTest.ExpectedReturnCode)
// Console.WriteLine($"\nERROR: Expected rc = {packageTest.ExpectedReturnCode} but got {rc}!");
//}
}
}

Expand All @@ -283,6 +296,21 @@ public abstract class PackageDefinition
extension.InstallExtension(this);
}

// Remove all extensions prior to starting a run. Note that we avoid removing the the
// package being developed, which may actually be an extension itself.
protected void RemoveExtensions()
{
foreach (DirectoryPath dirPath in _context.GetDirectories(ExtensionInstallDirectory + "*"))
{
string dirName = dirPath.Segments.Last();
if ((dirName.StartsWith("NUnit.Extension.") || dirName.StartsWith("nunit-extension-")) && !dirName.StartsWith(PackageId))
{
_context.DeleteDirectory(dirPath, new DeleteDirectorySettings() { Recursive = true });
_context.Information("Deleted directory " + dirPath.GetDirectoryName());
}
}
}

private void InstallRunners(IEnumerable<IPackageTestRunner> runners)
{
// Install any runners needing installation
Expand Down
Loading

0 comments on commit b75a304

Please sign in to comment.