From c17344cf248c0565e8855c7d04cdf2956077ad06 Mon Sep 17 00:00:00 2001 From: Garrett Moseke Date: Mon, 8 Jul 2024 16:27:22 +0200 Subject: [PATCH 1/4] fix: properly clean up output on macos Because we can't hive nice standardized output across OS --- Sources/TestRunner.swift | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Sources/TestRunner.swift b/Sources/TestRunner.swift index a4cb326..3accc5c 100644 --- a/Sources/TestRunner.swift +++ b/Sources/TestRunner.swift @@ -157,7 +157,7 @@ class PeregrineRunner: TestRunner { throw TestParseError.unexpectedLineFormat("Could not parse test definition from \(line)") } let suiteAndName = remainder.split(separator: "/") - guard let testSuite = suiteAndName.first, let testName = suiteAndName.last else { + guard let testSuite = suiteAndName.first?.trimmingPrefix("-["), let testName = suiteAndName.last else { throw TestParseError.unexpectedLineFormat("Could not parse test definition from \(line)") } let test = Test(suite: String(testSuite), name: String(testName)) @@ -401,10 +401,26 @@ class PeregrineRunner: TestRunner { } private func parseTestFromName(_ testName: String, line: String) throws -> Test { - let nameComponents = testName.split(separator: ".") - guard let testSuite = nameComponents.first, let testName = nameComponents.last else { - throw TestParseError.unexpectedLineFormat("could not parse test name from line: \(line)") - } + // because of course the output is subtly different on macos vs linux + #if os(macOS) + // example line: + // Test Case '-[PeregrineTests.PeregrineTests testRunSingleFail]' passed (0.739 seconds). + let nameComponents = testName.split(separator: " ") + guard + let testSuite = nameComponents.first?.split(separator: ".").last, + var testName = nameComponents.last + else { + throw TestParseError.unexpectedLineFormat("could not parse test name from line: \(line)") + } + testName.removeLast() + #elseif os(Linux) + // example line: + // Test Case 'PeregrineTests.testRunSingleFail' passed (0.459 seconds) + let nameComponents = testName.split(separator: ".") + guard var testSuite = nameComponents.first, var testName = nameComponents.last else { + throw TestParseError.unexpectedLineFormat("could not parse test name from line: \(line)") + } + #endif return Test(suite: String(testSuite), name: String(testName)) } From 08090286dfb0e86db6182f00dce70e20e48b398b Mon Sep 17 00:00:00 2001 From: Garrett Moseke Date: Mon, 8 Jul 2024 16:47:06 +0200 Subject: [PATCH 2/4] test: add test coverage for skipped test parsing --- .../TestPackageTests/TestPackageTests.swift | 8 ++++ Tests/PeregrineTests/PeregrineTests.swift | 47 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/TestPackage/Tests/TestPackageTests/TestPackageTests.swift b/TestPackage/Tests/TestPackageTests/TestPackageTests.swift index c15df79..94d8efc 100644 --- a/TestPackage/Tests/TestPackageTests/TestPackageTests.swift +++ b/TestPackage/Tests/TestPackageTests/TestPackageTests.swift @@ -18,6 +18,14 @@ final class SuiteOne: XCTestCase { func testCustomFailMessage() { XCTAssertEqual("Hosea Matthews", "Dutch Van Der Linde", "Always listen to Hosea") } + + func testSkippedNoReason() throws { + throw XCTSkip() + } + + func testSkippedWithReason() throws { + throw XCTSkip("Lernie is hard") + } } final class SuiteTwo: XCTestCase { diff --git a/Tests/PeregrineTests/PeregrineTests.swift b/Tests/PeregrineTests/PeregrineTests.swift index 3e0acd7..568751d 100644 --- a/Tests/PeregrineTests/PeregrineTests.swift +++ b/Tests/PeregrineTests/PeregrineTests.swift @@ -28,13 +28,14 @@ class PeregrineTests: XCTestCase { } func testParseList() async throws { - try XCTSkipIf(true, "fdjfldskfjslk") let listedTests = try Set(await runner.listTests()) let expected = Set([ Test(suite: "SuiteOne", name: "testSuccess"), Test(suite: "SuiteOne", name: "testSingleFail"), Test(suite: "SuiteOne", name: "testThreeFail"), Test(suite: "SuiteOne", name: "testCustomFailMessage"), + Test(suite: "SuiteOne", name: "testSkippedNoReason"), + Test(suite: "SuiteOne", name: "testSkippedWithReason"), Test(suite: "SuiteTwo", name: "testSuccess"), Test(suite: "SuiteTwo", name: "testSingleFail"), Test(suite: "SuiteTwo", name: "testThreeFail"), @@ -57,8 +58,6 @@ class PeregrineTests: XCTestCase { XCTAssertTrue(output.results.map { $0.errors }.reduce([], +).isEmpty) } - // TODO: add test for skipped test parsing - func testRunSingleFail() async throws { // Test normal single failed XCT* runner.options = TestOptions( @@ -126,7 +125,16 @@ class PeregrineTests: XCTestCase { packagePath: testPackagePath, plaintextOutput: false, quietOutput: true, - additionalSwiftFlags: ["--filter", "SuiteOne", "--filter", "SuiteTwo"] + additionalSwiftFlags: [ + "--filter", + "SuiteOne", + "--filter", + "SuiteTwo", + "--skip", + "testSkippedNoReason", + "--skip", + "testSkippedWithReason", + ] ) let output = try await runner.runTests(tests: []) XCTAssertFalse(output.success) @@ -166,4 +174,35 @@ class PeregrineTests: XCTestCase { XCTAssertFalse(output.success) XCTAssertNotNil(output.backtraceLines) } + + func testSkippedOutput() async throws { + runner.options = TestOptions( + toolchainPath: nil, + packagePath: testPackagePath, + plaintextOutput: false, + quietOutput: true, + additionalSwiftFlags: [ + "--filter", + "SuiteOne/testSuccess", + "--filter", + "SuiteOne/testSkippedNoReason", + "--filter", + "SuiteOne/testSkippedWithReason", + ] + ) + let output = try await runner.runTests(tests: []) + XCTAssertTrue(output.success) + let expectedErrors = Set([ + "Test skipped", + "Test skipped - Lernie is hard", + ]) + XCTAssertEqual(Set(output.results.map { $0.errors }.reduce([], +).map { $0.1 }), expectedErrors) + XCTAssertEqual( + Set(output.results.filter { $0.skipped }.map { $0.test }), + Set([ + Test(suite: "SuiteOne", name: "testSkippedNoReason"), + Test(suite: "SuiteOne", name: "testSkippedWithReason"), + ]) + ) + } } From 881803fb944bf8757b003a5c43c8f9f816d2ae12 Mon Sep 17 00:00:00 2001 From: Garrett Moseke Date: Mon, 8 Jul 2024 17:03:00 +0200 Subject: [PATCH 3/4] chore: bump tool version to 0.4.2, add discussion help block --- Sources/Peregrine.swift | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Sources/Peregrine.swift b/Sources/Peregrine.swift index 22d7982..77f68af 100644 --- a/Sources/Peregrine.swift +++ b/Sources/Peregrine.swift @@ -12,7 +12,14 @@ import SwiftCommand struct Peregrine: AsyncParsableCommand { static let configuration = CommandConfiguration( abstract: "A utility for clearer swift test output.", - version: "0.4.1", + discussion: """ + peregrine is a tool intended to clean up the often noisy output of swift-package-manager's `swift test` command. + It is meant as a development conveneince tool to more quickly and easily find failures and pull some simple test + statistics for large test suites. It is **NOT** a drop-in replacement for `swift test` - when debugging, it is still + generally favorable to `swift test --filter fooTest` where applicable. peregrine is meant to help you find that + `fooTest` is having issues in the first place. + """, + version: "0.4.2", subcommands: [Run.self, CountTests.self], defaultSubcommand: Run.self ) @@ -33,7 +40,7 @@ struct Peregrine: AsyncParsableCommand { @Flag(help: "Supress toolchain information & progress output") var quiet: Bool = false - @Option(help: "Control Peregrine's log level.") + @Option(help: "Control Peregrine's log level (1-7, with 1 being the most granular)") var logLevel: LogLevel = .debug } } @@ -49,7 +56,7 @@ extension Peregrine { var longestTestCount: Int? = nil // Again, this should be handled with xunit, but the spm xunit output is severely lacking - @Option(help: "Control the output format for long tests") + @Option(help: "Control the output format for long tests (stdout, csv)") var longTestOutputFormat: LongTestOutputFormat = .stdout @Option(help: "Output path for longest test file. Ignored if output is set to stdout.") From 4ae9176d991239e2f04e23ead55604ca56de8e12 Mon Sep 17 00:00:00 2001 From: Garrett Moseke Date: Mon, 8 Jul 2024 19:16:42 +0200 Subject: [PATCH 4/4] chore: remove unnecessary processing during test listing --- Sources/TestRunner.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/TestRunner.swift b/Sources/TestRunner.swift index 3accc5c..0a7dd95 100644 --- a/Sources/TestRunner.swift +++ b/Sources/TestRunner.swift @@ -152,12 +152,13 @@ class PeregrineRunner: TestRunner { var tests = [Test]() for try await line in listProcess.stdout.lines { + // `swift test list` output is standard across OS - thankfully logger.trace("swift test list stdout: \(line)") guard let remainder = line.split(separator: ".").last else { throw TestParseError.unexpectedLineFormat("Could not parse test definition from \(line)") } let suiteAndName = remainder.split(separator: "/") - guard let testSuite = suiteAndName.first?.trimmingPrefix("-["), let testName = suiteAndName.last else { + guard let testSuite = suiteAndName.first, let testName = suiteAndName.last else { throw TestParseError.unexpectedLineFormat("Could not parse test definition from \(line)") } let test = Test(suite: String(testSuite), name: String(testName))