diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 876647af3..e5f5d2237 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -31,16 +31,6 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha let exitCode = Locked(rawValue: EXIT_SUCCESS) do { - let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments) - if args.listTests ?? false { - for testID in await listTestsForEntryPoint(Test.all) { -#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO - try? FileHandle.stdout.write("\(testID)\n") -#else - print(testID) -#endif - } - } else { #if !SWT_NO_EXIT_TESTS // If an exit test was specified, run it. `exitTest` returns `Never`. if let exitTest = ExitTest.findInEnvironmentForEntryPoint() { @@ -48,41 +38,59 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha } #endif - // Configure the test runner. - var configuration = try configurationForEntryPoint(from: args) + let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments) + // Configure the test runner. + var configuration = try configurationForEntryPoint(from: args) - // Set up the event handler. - configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown { - exitCode.withLock { exitCode in - exitCode = EXIT_FAILURE - } + // Set up the event handler. + configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in + if case let .issueRecorded(issue) = event.kind, !issue.isKnown { + exitCode.withLock { exitCode in + exitCode = EXIT_FAILURE } - oldEventHandler(event, context) } + oldEventHandler(event, context) + } #if !SWT_NO_FILE_IO - // Configure the event recorder to write events to stderr. - var options = Event.ConsoleOutputRecorder.Options() - options = .for(.stderr) - options.verbosity = args.verbosity - let eventRecorder = Event.ConsoleOutputRecorder(options: options) { string in - try? FileHandle.stderr.write(string) - } + // Configure the event recorder to write events to stderr. + var options = Event.ConsoleOutputRecorder.Options() + options = .for(.stderr) + options.verbosity = args.verbosity + let eventRecorder = Event.ConsoleOutputRecorder(options: options) { string in + try? FileHandle.stderr.write(string) + } + configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in + eventRecorder.record(event, in: context) + oldEventHandler(event, context) + } +#endif + + // If the caller specified an alternate event handler, hook it up too. + if let eventHandler { configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - eventRecorder.record(event, in: context) + eventHandler(event, context) oldEventHandler(event, context) } -#endif + } - // If the caller specified an alternate event handler, hook it up too. - if let eventHandler { - configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - eventHandler(event, context) - oldEventHandler(event, context) - } + if args.listTests ?? false { + let tests = await Test.all + for testID in listTestsForEntryPoint(tests) { + // Print the test ID to stdout (classical CLI behavior.) +#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO + try? FileHandle.stdout.write("\(testID)\n") +#else + print(testID) +#endif } + // Post an event for every discovered test. These events are turned into + // JSON objects if JSON output is enabled. + for test in tests { + Event.post(.testDiscovered, for: test, testCase: nil, configuration: configuration) + } + } else { // Run the tests. let runner = await Runner(configuration: configuration) await runner.run() diff --git a/Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift b/Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift index 93f14ce89..5b0266604 100644 --- a/Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift +++ b/Sources/Testing/ABI/v0/ABIv0.Record+Streaming.swift @@ -31,18 +31,15 @@ extension ABIv0.Record { ) -> Event.Handler { let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder() return { event, context in - if case let .runStarted(plan) = event.kind { - // Gather all tests in the run and forward records for them. - for test in plan.steps.lazy.map(\.test) { - try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in - eventHandler(testJSON) - } + if case .testDiscovered = event.kind, let test = context.test { + try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in + eventHandler(testJSON) + } + } else { + let messages = humanReadableOutputRecorder.record(event, in: context) + if let eventRecord = Self(encoding: event, in: context, messages: messages) { + try? JSON.withEncoding(of: eventRecord, eventHandler) } - } - - let messages = humanReadableOutputRecorder.record(event, in: context) - if let eventRecord = Self(encoding: event, in: context, messages: messages) { - try? JSON.withEncoding(of: eventRecord, eventHandler) } } } @@ -92,6 +89,12 @@ func eventHandlerForStreamingEventSnapshots( to eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void ) -> Event.Handler { return { event, context in + if case .testDiscovered = event.kind { + // Discard events of this kind rather than forwarding them to avoid a + // crash in Xcode 16 Beta 1 (which does not expect any events to occur + // before .runStarted.) + return + } let snapshot = EventAndContextSnapshot( event: Event.Snapshot(snapshotting: event), eventContext: Event.Context.Snapshot(snapshotting: context) diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 0c671247b..ac0c49a49 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -13,13 +13,23 @@ public struct Event: Sendable { /// An enumeration describing the various kinds of event that can be observed. public enum Kind: Sendable { - /// A test run started. + /// A test was discovered during test run planning. /// - /// - Parameters: - /// - plan: The test plan of the run that started. + /// This event is recorded once per discovered test when ``Runner/run()`` is + /// called. It does not indicate whether or not a test will run or be + /// skipped—only that the test was found by the testing library and is part + /// of the runner's plan. /// - /// This event is the first event posted after ``Runner/run()`` is called. - indirect case runStarted(_ plan: Runner.Plan) + /// This event is also posted once per test when `swift test list` is + /// called. In that case, events are posted for all discovered tests + /// regardless of whether or not they would run. + case testDiscovered + + /// A test run started. + /// + /// This event is posted when ``Runner/run()`` is called after + /// ``testDiscovered`` has been posted for all tests in the runner's plan. + case runStarted /// An iteration of the test run started. /// @@ -318,9 +328,22 @@ extension Event { extension Event.Kind { /// A serializable enumeration describing the various kinds of event that can be observed. public enum Snapshot: Sendable, Codable { + /// A test was discovered during test run planning. + /// + /// This event is recorded once per discovered test when ``Runner/run()`` is + /// called. It does not indicate whether or not a test will run or be + /// skipped—only that the test was found by the testing library and is part + /// of the runner's plan. + /// + /// This event is also posted once per test when `swift test list` is + /// called. In that case, events are posted for all discovered tests + /// regardless of whether or not they would run. + case testDiscovered + /// A test run started. /// - /// This is the first event posted after ``Runner/run()`` is called. + /// This event is posted when ``Runner/run()`` is called after + /// ``testDiscovered`` has been posted for all tests in the runner's plan. case runStarted /// An iteration of the test run started. @@ -420,6 +443,8 @@ extension Event.Kind { /// - Parameter kind: The original ``Event.Kind`` to snapshot. public init(snapshotting kind: Event.Kind) { switch kind { + case .testDiscovered: + self = .testDiscovered case .runStarted: self = .runStarted case let .iterationStarted(index): diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index a0e1de92a..60789b66e 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -296,6 +296,11 @@ extension Event.HumanReadableOutputRecorder { // Finally, produce any messages for the event. switch event.kind { + case .testDiscovered: + // Suppress events of this kind from output as they are not generally + // interesting in human-readable output. + break + case .runStarted: var comments = [Comment]() if verbosity > 0 { diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 55749685a..6c6163a00 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -27,8 +27,8 @@ public struct Runner: Sendable { /// - tests: The tests to run. /// - configuration: The configuration to use for running. public init(testing tests: [Test], configuration: Configuration = .init()) async { - self.plan = await Plan(tests: tests, configuration: configuration) - self.configuration = configuration + let plan = await Plan(tests: tests, configuration: configuration) + self.init(plan: plan, configuration: configuration) } /// Initialize an instance of this type that runs the tests in the specified @@ -343,7 +343,13 @@ extension Runner { } await Configuration.withCurrent(runner.configuration) { - Event.post(.runStarted(runner.plan), for: nil, testCase: nil, configuration: runner.configuration) + // Post an event for every test in the test plan being run. These events + // are turned into JSON objects if JSON output is enabled. + for test in runner.plan.steps.lazy.map(\.test) { + Event.post(.testDiscovered, for: test, testCase: nil, configuration: runner.configuration) + } + + Event.post(.runStarted, for: nil, testCase: nil, configuration: runner.configuration) defer { Event.post(.runEnded, for: nil, testCase: nil, configuration: runner.configuration) } diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index 90711dfba..cec9d278f 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -109,6 +109,25 @@ struct ABIEntryPointTests { #expect(result) } + @Test("v0 entry point listing tests only") + func v0_listingTestsOnly() async throws { + var arguments = __CommandLineArguments_v0() + arguments.listTests = true + arguments.eventStreamVersion = 0 + arguments.verbosity = .min + + try await confirmation("Test matched", expectedCount: 1...) { testMatched in + _ = try await _invokeEntryPointV0(passing: arguments) { recordJSON in + let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON) + if case .test = record.kind { + testMatched() + } else { + Issue.record("Unexpected record \(record)") + } + } + } + } + private func _invokeEntryPointV0( passing arguments: __CommandLineArguments_v0, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void = { _ in } diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index 1dd226583..4e77046fd 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -48,7 +48,7 @@ struct EventTests { sourceLocation: nil) ) ), - Event.Kind.runStarted(Runner.Plan(steps: [])), + Event.Kind.runStarted, Event.Kind.runEnded, Event.Kind.testCaseStarted, Event.Kind.testCaseEnded, diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 0eae7a9e7..4c4547179 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -155,7 +155,7 @@ struct SwiftPMTests { do { let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--xunit-output", temporaryFilePath]) let eventContext = Event.Context() - configuration.eventHandler(Event(.runStarted(Runner.Plan(steps: [])), testID: nil, testCaseID: nil), eventContext) + configuration.eventHandler(Event(.runStarted, testID: nil, testCaseID: nil), eventContext) configuration.eventHandler(Event(.runEnded, testID: nil, testCaseID: nil), eventContext) } @@ -222,15 +222,11 @@ struct SwiftPMTests { } do { let configuration = try configurationForEntryPoint(withArguments: ["PATH", outputArgumentName, temporaryFilePath, versionArgumentName, version]) - let eventContext = Event.Context() - let test = Test {} - let plan = Runner.Plan( - steps: [ - Runner.Plan.Step(test: test, action: .run(options: .init(isParallelizationEnabled: true))) - ] - ) - configuration.handleEvent(Event(.runStarted(plan), testID: nil, testCaseID: nil), in: eventContext) + let eventContext = Event.Context(test: test) + + configuration.handleEvent(Event(.testDiscovered, testID: test.id, testCaseID: nil), in: eventContext) + configuration.handleEvent(Event(.runStarted, testID: nil, testCaseID: nil), in: eventContext) do { let eventContext = Event.Context(test: test) configuration.handleEvent(Event(.testStarted, testID: test.id, testCaseID: nil), in: eventContext)