From c5ef5651ea6c8495b87ccc3999923c5ea53c5b29 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 22 May 2025 22:00:59 -0500 Subject: [PATCH 1/2] Output console message for test case ended events in verbose mode, with status and issue counts Fixes #1021 Fixes rdar://146863942 --- .../Event.HumanReadableOutputRecorder.swift | 97 ++++++++++++++----- Tests/TestingTests/EventRecorderTests.swift | 19 ++-- 2 files changed, 87 insertions(+), 29 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 06d12de6e..829b481d8 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -36,7 +36,7 @@ extension Event { /// A type that contains mutable context for /// ``Event/ConsoleOutputRecorder``. - private struct _Context { + fileprivate struct Context { /// The instant at which the run started. var runStartInstant: Test.Clock.Instant? @@ -51,6 +51,17 @@ extension Event { /// The number of test suites started or skipped during the run. var suiteCount = 0 + /// An enumeration describing the various keys which can be used in a test + /// data graph for an output recorder. + enum TestDataKey: Hashable { + /// A string key, typically containing one key from the key path + /// representation of a ``Test/ID`` instance. + case string(String) + + /// A test case ID. + case testCaseID(Test.Case.ID) + } + /// A type describing data tracked on a per-test basis. struct TestData { /// The instant at which the test started. @@ -62,18 +73,15 @@ extension Event { /// The number of known issues recorded for the test. var knownIssueCount = 0 - - /// The number of test cases for the test. - var testCasesCount = 0 } /// Data tracked on a per-test basis. - var testData = Graph() + var testData = Graph() } /// This event recorder's mutable context about events it has received, /// which may be used to inform how subsequent events are written. - private var _context = Locked(rawValue: _Context()) + private var _context = Locked(rawValue: Context()) /// Initialize a new human-readable event recorder. /// @@ -128,7 +136,9 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts( + in graph: Graph? + ) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { return (0, 0, 0, 0, "") } @@ -241,6 +251,7 @@ extension Event.HumanReadableOutputRecorder { 0 } let test = eventContext.test + let keyPath = eventContext.keyPath let testName = if let test { if let displayName = test.displayName { if verbosity > 0 { @@ -271,7 +282,7 @@ extension Event.HumanReadableOutputRecorder { case .testStarted: let test = test! - context.testData[test.id.keyPathRepresentation] = .init(startInstant: instant) + context.testData[keyPath] = .init(startInstant: instant) if test.isSuite { context.suiteCount += 1 } else { @@ -287,23 +298,17 @@ extension Event.HumanReadableOutputRecorder { } case let .issueRecorded(issue): - let id: [String] = if let test { - test.id.keyPathRepresentation - } else { - [] - } - var testData = context.testData[id] ?? .init(startInstant: instant) + var testData = context.testData[keyPath] ?? .init(startInstant: instant) if issue.isKnown { testData.knownIssueCount += 1 } else { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } - context.testData[id] = testData + context.testData[keyPath] = testData case .testCaseStarted: - let test = test! - context.testData[test.id.keyPathRepresentation]?.testCasesCount += 1 + context.testData[keyPath] = .init(startInstant: instant) default: // These events do not manipulate the context structure. @@ -384,13 +389,12 @@ extension Event.HumanReadableOutputRecorder { case .testEnded: let test = test! - let id = test.id - let testDataGraph = context.testData.subgraph(at: id.keyPathRepresentation) + let testDataGraph = context.testData.subgraph(at: keyPath) let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - let testCasesCount = if test.isParameterized { - " with \(testData.testCasesCount.counting("test case"))" + let testCasesCount = if test.isParameterized, let testDataGraph { + " with \(testDataGraph.children.count.counting("test case"))" } else { "" } @@ -517,15 +521,37 @@ extension Event.HumanReadableOutputRecorder { break } + let status = verbosity > 0 ? " started" : "" + return [ Message( symbol: .default, - stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)" + stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)\(status)." ) ] case .testCaseEnded: - break + guard verbosity > 0, let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { + break + } + + let testDataGraph = context.testData.subgraph(at: keyPath) + let testData = testDataGraph?.value ?? .init(startInstant: instant) + let issues = _issueCounts(in: testDataGraph) + let duration = testData.startInstant.descriptionOfDuration(to: instant) + + let message = if issues.errorIssueCount > 0 { + Message( + symbol: .fail, + stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)." + ) + } else { + Message( + symbol: .pass(knownIssueCount: issues.knownIssueCount), + stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)." + ) + } + return [message] case let .iterationEnded(index): guard let iterationStartInstant = context.iterationStartInstant else { @@ -568,6 +594,31 @@ extension Event.HumanReadableOutputRecorder { } } +extension Test.ID { + /// The key path in a test data graph representing this test ID. + fileprivate var keyPath: some Collection { + keyPathRepresentation.map { .string($0) } + } +} + +extension Event.Context { + /// The key path in a test data graph representing this event this context is + /// associated with, including its test and/or test case IDs. + fileprivate var keyPath: some Collection { + var keyPath = [Event.HumanReadableOutputRecorder.Context.TestDataKey]() + + if let test { + keyPath.append(contentsOf: test.id.keyPath) + + if let testCase { + keyPath.append(.testCaseID(testCase.id)) + } + } + + return keyPath + } +} + // MARK: - Codable extension Event.HumanReadableOutputRecorder.Message: Codable {} diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index d06e12c7c..6b563d1ca 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -94,6 +94,7 @@ struct EventRecorderTests { } @Test("Verbose output") + @available(_regexAPI, *) func verboseOutput() async throws { let stream = Stream() @@ -112,6 +113,14 @@ struct EventRecorderTests { #expect(buffer.contains(#"\#(Event.Symbol.details.unicodeCharacter) lhs: Swift.String → "987""#)) #expect(buffer.contains(#""Animal Crackers" (aka 'WrittenTests')"#)) #expect(buffer.contains(#""Not A Lobster" (aka 'actuallyCrab()')"#)) + do { + let regex = try Regex(".* Passing 1 argument i → 0 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) passed after .*.") + #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) + } + do { + let regex = try Regex(".* Passing 1 argument i → 3 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) failed after .* with 1 issue.") + #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) + } if testsWithSignificantIOAreEnabled { print(buffer, terminator: "") @@ -203,17 +212,15 @@ struct EventRecorderTests { await runTest(for: PredictablyFailingTests.self, configuration: configuration) let buffer = stream.buffer.rawValue - if testsWithSignificantIOAreEnabled { - print(buffer, terminator: "") - } - let aurgmentRegex = try Regex(expectedPattern) + let argumentRegex = try Regex(expectedPattern) #expect( (try buffer .split(whereSeparator: \.isNewline) - .compactMap(aurgmentRegex.wholeMatch(in:)) - .first) != nil + .compactMap(argumentRegex.wholeMatch(in:)) + .first) != nil, + "buffer: \(buffer)" ) } From eb96e6f4a7dd54941c5e14b97fa38f6e6ee64b62 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 23 May 2025 14:20:31 -0500 Subject: [PATCH 2/2] Refine the message strings' grammar --- .../Recorder/Event.HumanReadableOutputRecorder.swift | 6 +++--- .../CustomTestStringConvertible.swift | 12 ++++++------ Tests/TestingTests/EventRecorderTests.swift | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 829b481d8..a3d121e09 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -526,7 +526,7 @@ extension Event.HumanReadableOutputRecorder { return [ Message( symbol: .default, - stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)\(status)." + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)\(status) started." ) ] @@ -543,12 +543,12 @@ extension Event.HumanReadableOutputRecorder { let message = if issues.errorIssueCount > 0 { Message( symbol: .fail, - stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)." + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)." ) } else { Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)." + stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)." ) } return [message] diff --git a/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift b/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift index 192dde5ad..6f0517468 100644 --- a/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift +++ b/Sources/Testing/SourceAttribution/CustomTestStringConvertible.swift @@ -42,9 +42,9 @@ /// the default description of a value may not be adequately descriptive: /// /// ``` -/// ◇ Passing argument food → .paella to isDelicious(_:) -/// ◇ Passing argument food → .oden to isDelicious(_:) -/// ◇ Passing argument food → .ragu to isDelicious(_:) +/// ◇ Test case passing 1 argument food → .paella to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → .oden to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → .ragu to isDelicious(_:) started. /// ``` /// /// By adopting ``CustomTestStringConvertible``, customized descriptions can be @@ -69,9 +69,9 @@ /// ``testDescription`` property: /// /// ``` -/// ◇ Passing argument food → paella valenciana to isDelicious(_:) -/// ◇ Passing argument food → おでん to isDelicious(_:) -/// ◇ Passing argument food → ragù alla bolognese to isDelicious(_:) +/// ◇ Test case passing 1 argument food → paella valenciana to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → おでん to isDelicious(_:) started. +/// ◇ Test case passing 1 argument food → ragù alla bolognese to isDelicious(_:) started. /// ``` /// /// ## See Also diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 6b563d1ca..ed7d765a0 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -114,11 +114,11 @@ struct EventRecorderTests { #expect(buffer.contains(#""Animal Crackers" (aka 'WrittenTests')"#)) #expect(buffer.contains(#""Not A Lobster" (aka 'actuallyCrab()')"#)) do { - let regex = try Regex(".* Passing 1 argument i → 0 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) passed after .*.") + let regex = try Regex(".* Test case passing 1 argument i → 0 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) passed after .*.") #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) } do { - let regex = try Regex(".* Passing 1 argument i → 3 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) failed after .* with 1 issue.") + let regex = try Regex(".* Test case passing 1 argument i → 3 \\(Swift.Int\\) to multitudeOcelot\\(i:\\) failed after .* with 1 issue.") #expect(try buffer.split(whereSeparator: \.isNewline).compactMap(regex.wholeMatch(in:)).first != nil) }