Merge pull request #13 from GNMoseke/feat/cleanup-macos-output

Feat/cleanup macos output

Sources/Peregrine.swift

@@ -12,7 +12,14 @@ import SwiftCommand
 struct Peregrine: AsyncParsableCommand {
     static let configuration = CommandConfiguration(
         abstract: "A utility for clearer swift test output.",
-        version: "0.4.1",
+        discussion: """
+        peregrine is a tool intended to clean up the often noisy output of swift-package-manager's `swift test` command.
+        It is meant as a development conveneince tool to more quickly and easily find failures and pull some simple test 
+        statistics for large test suites. It is **NOT** a drop-in replacement for `swift test` - when debugging, it is still
+        generally favorable to `swift test --filter fooTest` where applicable. peregrine is meant to help you find that
+        `fooTest` is having issues in the first place.
+        """,
+        version: "0.4.2",
         subcommands: [Run.self, CountTests.self],
         defaultSubcommand: Run.self
     )
@@ -33,7 +40,7 @@ struct Peregrine: AsyncParsableCommand {
         @Flag(help: "Supress toolchain information & progress output")
         var quiet: Bool = false
 
-        @Option(help: "Control Peregrine's log level.")
+        @Option(help: "Control Peregrine's log level (1-7, with 1 being the most granular)")
         var logLevel: LogLevel = .debug
     }
 }
@@ -49,7 +56,7 @@ extension Peregrine {
         var longestTestCount: Int? = nil
 
         // Again, this should be handled with xunit, but the spm xunit output is severely lacking
-        @Option(help: "Control the output format for long tests")
+        @Option(help: "Control the output format for long tests (stdout, csv)")
         var longTestOutputFormat: LongTestOutputFormat = .stdout
 
         @Option(help: "Output path for longest test file. Ignored if output is set to stdout.")

Sources/TestRunner.swift

@@ -152,6 +152,7 @@ class PeregrineRunner: TestRunner {
 
         var tests = [Test]()
         for try await line in listProcess.stdout.lines {
+            // `swift test list` output is standard across OS - thankfully
             logger.trace("swift test list stdout: \(line)")
             guard let remainder = line.split(separator: ".").last else {
                 throw TestParseError.unexpectedLineFormat("Could not parse test definition from \(line)")
@@ -401,10 +402,26 @@ class PeregrineRunner: TestRunner {
 }
 
 private func parseTestFromName(_ testName: String, line: String) throws -> Test {
-    let nameComponents = testName.split(separator: ".")
-    guard let testSuite = nameComponents.first, let testName = nameComponents.last else {
-        throw TestParseError.unexpectedLineFormat("could not parse test name from line: \(line)")
-    }
+    // because of course the output is subtly different on macos vs linux
+    #if os(macOS)
+        // example line:
+        // Test Case '-[PeregrineTests.PeregrineTests testRunSingleFail]' passed (0.739 seconds).
+        let nameComponents = testName.split(separator: " ")
+        guard
+            let testSuite = nameComponents.first?.split(separator: ".").last,
+            var testName = nameComponents.last
+        else {
+            throw TestParseError.unexpectedLineFormat("could not parse test name from line: \(line)")
+        }
+        testName.removeLast()
+    #elseif os(Linux)
+        // example line:
+        // Test Case 'PeregrineTests.testRunSingleFail' passed (0.459 seconds)
+        let nameComponents = testName.split(separator: ".")
+        guard var testSuite = nameComponents.first, var testName = nameComponents.last else {
+            throw TestParseError.unexpectedLineFormat("could not parse test name from line: \(line)")
+        }
+    #endif
     return Test(suite: String(testSuite), name: String(testName))
 }
 

TestPackage/Tests/TestPackageTests/TestPackageTests.swift

@@ -18,6 +18,14 @@ final class SuiteOne: XCTestCase {
     func testCustomFailMessage() {
        XCTAssertEqual("Hosea Matthews", "Dutch Van Der Linde", "Always listen to Hosea") 
     }
+
+    func testSkippedNoReason() throws {
+        throw XCTSkip()
+    }
+
+    func testSkippedWithReason() throws {
+        throw XCTSkip("Lernie is hard")
+    }
 }
 
 final class SuiteTwo: XCTestCase {

Tests/PeregrineTests/PeregrineTests.swift

@@ -28,13 +28,14 @@ class PeregrineTests: XCTestCase {
     }
 
     func testParseList() async throws {
-        try XCTSkipIf(true, "fdjfldskfjslk")
         let listedTests = try Set(await runner.listTests())
         let expected = Set([
             Test(suite: "SuiteOne", name: "testSuccess"),
             Test(suite: "SuiteOne", name: "testSingleFail"),
             Test(suite: "SuiteOne", name: "testThreeFail"),
             Test(suite: "SuiteOne", name: "testCustomFailMessage"),
+            Test(suite: "SuiteOne", name: "testSkippedNoReason"),
+            Test(suite: "SuiteOne", name: "testSkippedWithReason"),
             Test(suite: "SuiteTwo", name: "testSuccess"),
             Test(suite: "SuiteTwo", name: "testSingleFail"),
             Test(suite: "SuiteTwo", name: "testThreeFail"),
@@ -57,8 +58,6 @@ class PeregrineTests: XCTestCase {
         XCTAssertTrue(output.results.map { $0.errors }.reduce([], +).isEmpty)
     }
 
-    // TODO: add test for skipped test parsing
-
     func testRunSingleFail() async throws {
         // Test normal single failed XCT*
         runner.options = TestOptions(
@@ -126,7 +125,16 @@ class PeregrineTests: XCTestCase {
             packagePath: testPackagePath,
             plaintextOutput: false,
             quietOutput: true,
-            additionalSwiftFlags: ["--filter", "SuiteOne", "--filter", "SuiteTwo"]
+            additionalSwiftFlags: [
+                "--filter",
+                "SuiteOne",
+                "--filter",
+                "SuiteTwo",
+                "--skip",
+                "testSkippedNoReason",
+                "--skip",
+                "testSkippedWithReason",
+            ]
         )
         let output = try await runner.runTests(tests: [])
         XCTAssertFalse(output.success)
@@ -166,4 +174,35 @@ class PeregrineTests: XCTestCase {
         XCTAssertFalse(output.success)
         XCTAssertNotNil(output.backtraceLines)
     }
+
+    func testSkippedOutput() async throws {
+        runner.options = TestOptions(
+            toolchainPath: nil,
+            packagePath: testPackagePath,
+            plaintextOutput: false,
+            quietOutput: true,
+            additionalSwiftFlags: [
+                "--filter",
+                "SuiteOne/testSuccess",
+                "--filter",
+                "SuiteOne/testSkippedNoReason",
+                "--filter",
+                "SuiteOne/testSkippedWithReason",
+            ]
+        )
+        let output = try await runner.runTests(tests: [])
+        XCTAssertTrue(output.success)
+        let expectedErrors = Set([
+            "Test skipped",
+            "Test skipped - Lernie is hard",
+        ])
+        XCTAssertEqual(Set(output.results.map { $0.errors }.reduce([], +).map { $0.1 }), expectedErrors)
+        XCTAssertEqual(
+            Set(output.results.filter { $0.skipped }.map { $0.test }),
+            Set([
+                Test(suite: "SuiteOne", name: "testSkippedNoReason"),
+                Test(suite: "SuiteOne", name: "testSkippedWithReason"),
+            ])
+        )
+    }
 }