Emit ABIv0 JSON objects for tests when listing them.

This PR changes the behavior of `swift test list` and its various synonyms to allow reporting the list of tests via the ABI-stable JSON mechanism described in #479.

As it is not currently possible to directly call `swift test list --experimental-event-stream-output ... --experimental-event-stream-version 0`, it's a bit hard to test this code. However, it is possible to opt into this mode using `--experimental-configuration-path` and passing a path to a JSON file that includes `"listTests": true` (as noted by @allevato.)

Resolves #506.
Resolves rdar://130627856.

Author: grynspan

Sources/Testing/EntryPoints/ABIv0/ABIv0.Record+Streaming.swift

@@ -31,18 +31,15 @@ extension ABIv0.Record {
   ) -> Event.Handler {
     let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder()
     return { event, context in
-      if case let .runStarted(plan) = event.kind {
-        // Gather all tests in the run and forward records for them.
-        for test in plan.steps.lazy.map(\.test) {
-          try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in
-            eventHandler(testJSON)
-          }
+      if case .testDiscovered = event.kind, let test = context.test {
+        try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in
+          eventHandler(testJSON)
+        }
+      } else {
+        let messages = humanReadableOutputRecorder.record(event, in: context)
+        if let eventRecord = Self(encoding: event, in: context, messages: messages) {
+          try? JSON.withEncoding(of: eventRecord, eventHandler)
         }
-      }
-
-      let messages = humanReadableOutputRecorder.record(event, in: context)
-      if let eventRecord = Self(encoding: event, in: context, messages: messages) {
-        try? JSON.withEncoding(of: eventRecord, eventHandler)
       }
     }
   }
@@ -92,6 +89,12 @@ func eventHandlerForStreamingEventSnapshots(
   to eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void
 ) -> Event.Handler {
   return { event, context in
+    if case .testDiscovered = event.kind {
+      // Discard events of this kind rather than forwarding them to avoid a
+      // crash in Xcode 16 Beta 1 (which does not expect any events to occur
+      // before .runStarted.)
+      return
+    }
     let snapshot = EventAndContextSnapshot(
       event: Event.Snapshot(snapshotting: event),
       eventContext: Event.Context.Snapshot(snapshotting: context)

Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedTest.swift

@@ -81,7 +81,7 @@ extension ABIv0 {
         let testIsParameterized = test.isParameterized
         isParameterized = testIsParameterized
         if testIsParameterized {
-          _testCases = test.testCases?.map(EncodedTestCase.init(encoding:))
+          _testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:))
         }
       }
       name = test.name

Sources/Testing/EntryPoints/EntryPoint.swift

@@ -31,58 +31,66 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
   let exitCode = Locked(rawValue: EXIT_SUCCESS)
 
   do {
-    let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments)
-    if args.listTests ?? false {
-      for testID in await listTestsForEntryPoint(Test.all) {
-#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO
-        try? FileHandle.stdout.write("\(testID)\n")
-#else
-        print(testID)
-#endif
-      }
-    } else {
 #if !SWT_NO_EXIT_TESTS
       // If an exit test was specified, run it. `exitTest` returns `Never`.
       if let exitTest = ExitTest.findInEnvironmentForEntryPoint() {
         await exitTest()
       }
 #endif
 
-      // Configure the test runner.
-      var configuration = try configurationForEntryPoint(from: args)
+    let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments)
+    // Configure the test runner.
+    var configuration = try configurationForEntryPoint(from: args)
 
-      // Set up the event handler.
-      configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-        if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
-          exitCode.withLock { exitCode in
-            exitCode = EXIT_FAILURE
-          }
+    // Set up the event handler.
+    configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
+      if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+        exitCode.withLock { exitCode in
+          exitCode = EXIT_FAILURE
         }
-        oldEventHandler(event, context)
       }
+      oldEventHandler(event, context)
+    }
 
 #if !SWT_NO_FILE_IO
-      // Configure the event recorder to write events to stderr.
-      var options = Event.ConsoleOutputRecorder.Options()
-      options = .for(.stderr)
-      options.verbosity = args.verbosity
-      let eventRecorder = Event.ConsoleOutputRecorder(options: options) { string in
-        try? FileHandle.stderr.write(string)
-      }
+    // Configure the event recorder to write events to stderr.
+    var options = Event.ConsoleOutputRecorder.Options()
+    options = .for(.stderr)
+    options.verbosity = args.verbosity
+    let eventRecorder = Event.ConsoleOutputRecorder(options: options) { string in
+      try? FileHandle.stderr.write(string)
+    }
+    configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
+      eventRecorder.record(event, in: context)
+      oldEventHandler(event, context)
+    }
+#endif
+
+    // If the caller specified an alternate event handler, hook it up too.
+    if let eventHandler {
       configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-        eventRecorder.record(event, in: context)
+        eventHandler(event, context)
         oldEventHandler(event, context)
       }
-#endif
+    }
 
-      // If the caller specified an alternate event handler, hook it up too.
-      if let eventHandler {
-        configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-          eventHandler(event, context)
-          oldEventHandler(event, context)
-        }
+    if args.listTests ?? false {
+      let tests = await Test.all
+      for testID in listTestsForEntryPoint(tests) {
+        // Print the test ID to stdout (classical CLI behavior.)
+#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO
+        try? FileHandle.stdout.write("\(testID)\n")
+#else
+        print(testID)
+#endif
       }
 
+      // Post an event for every discovered test. These events are turned into
+      // JSON objects if JSON output is enabled.
+      for test in tests {
+        Event.post(.testDiscovered, for: test, testCase: nil, configuration: configuration)
+      }
+    } else {
       // Run the tests.
       let runner = await Runner(configuration: configuration)
       await runner.run()

Sources/Testing/Events/Event.swift

@@ -13,13 +13,18 @@
 public struct Event: Sendable {
   /// An enumeration describing the various kinds of event that can be observed.
   public enum Kind: Sendable {
-    /// A test run started.
+    /// A test was discovered during test run planning.
     ///
-    /// - Parameters:
-    ///   - plan: The test plan of the run that started.
+    /// This event is recorded once per discovered test when an instance of
+    /// ``Runner`` is initialized, _prior to_ ``Runner/run()`` being called. It
+    /// does not indicate whether or not a test will run—only that the test was
+    /// found by the testing library.
+    case testDiscovered
+
+    /// A test run started.
     ///
     /// This event is the first event posted after ``Runner/run()`` is called.
-    indirect case runStarted(_ plan: Runner.Plan)
+    case runStarted
 
     /// An iteration of the test run started.
     ///
@@ -318,6 +323,14 @@ extension Event {
 extension Event.Kind {
   /// A serializable enumeration describing the various kinds of event that can be observed.
   public enum Snapshot: Sendable, Codable {
+    /// A test was discovered during test run planning.
+    ///
+    /// This event is recorded once per discovered test when an instance of
+    /// ``Runner`` is initialized, _prior to_ ``Runner/run()`` being called. It
+    /// does not indicate whether or not a test will run—only that the test was
+    /// found by the testing library.
+    case testDiscovered
+
     /// A test run started.
     ///
     /// This is the first event posted after ``Runner/run()`` is called.
@@ -420,6 +433,8 @@ extension Event.Kind {
     /// - Parameter kind: The original ``Event.Kind`` to snapshot.
     public init(snapshotting kind: Event.Kind) {
       switch kind {
+      case .testDiscovered:
+        self = .testDiscovered
       case .runStarted:
         self = .runStarted
       case let .iterationStarted(index):

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -296,6 +296,11 @@ extension Event.HumanReadableOutputRecorder {
 
     // Finally, produce any messages for the event.
     switch event.kind {
+    case .testDiscovered:
+      // Suppress events of this kind from output as they are not generally
+      // interesting in human-readable output.
+      break
+
     case .runStarted:
       var comments = [Comment]()
       if verbosity > 0 {

Sources/Testing/Issues/Confirmation.swift

@@ -96,13 +96,92 @@ public func confirmation<R>(
   expectedCount: Int = 1,
   sourceLocation: SourceLocation = #_sourceLocation,
   _ body: (Confirmation) async throws -> R
+) async rethrows -> R {
+  try await confirmation(
+    comment,
+    expectedCount: expectedCount ... expectedCount,
+    sourceLocation: sourceLocation,
+    body
+  )
+}
+
+/// Confirm that some event occurs during the invocation of a function.
+///
+/// - Parameters:
+///   - comment: An optional comment to apply to any issues generated by this
+///     function.
+///   - expectedCount: A range of integers indicating the number of times the
+///   	expected event should occur when `body` is invoked.
+///   - sourceLocation: The source location to which any recorded issues should
+///     be attributed.
+///   - body: The function to invoke.
+///
+/// - Returns: Whatever is returned by `body`.
+///
+/// - Throws: Whatever is thrown by `body`.
+///
+/// Use confirmations to check that an event occurs while a test is running in
+/// complex scenarios where `#expect()` and `#require()` are insufficient. For
+/// example, a confirmation may be useful when an expected event occurs:
+///
+/// - In a context that cannot be awaited by the calling function such as an
+///   event handler or delegate callback;
+/// - More than once, or never; or
+/// - As a callback that is invoked as part of a larger operation.
+///
+/// To use a confirmation, pass a closure containing the work to be performed.
+/// The testing library will then pass an instance of ``Confirmation`` to the
+/// closure. Every time the event in question occurs, the closure should call
+/// the confirmation:
+///
+/// ```swift
+/// let minBuns = 5
+/// let maxBuns = 10
+/// await confirmation(
+/// 	"Baked between \(minBuns) and \(maxBuns) buns",
+///   expectedCount: minBuns ... maxBuns
+/// ) { bunBaked in
+///   foodTruck.eventHandler = { event in
+///     if event == .baked(.cinnamonBun) {
+///       bunBaked()
+///     }
+///   }
+///   await foodTruck.bakeTray(of: .cinnamonBun)
+/// }
+/// ```
+///
+/// When the closure returns, the testing library checks if the confirmation's
+/// preconditions have been met, and records an issue if they have not.
+///
+/// If an exact count is expected, use
+/// ``confirmation(_:expectedCount:sourceLocation:_:)-7kfko`` instead.
+@_spi(Experimental)
+public func confirmation<R>(
+  _ comment: Comment? = nil,
+  expectedCount: some RangeExpression<Int>,
+  sourceLocation: SourceLocation = #_sourceLocation,
+  _ body: (Confirmation) async throws -> R
 ) async rethrows -> R {
   let confirmation = Confirmation()
   defer {
     let actualCount = confirmation.count.rawValue
-    if actualCount != expectedCount {
+    if !expectedCount.contains(actualCount) {
+      var comment = comment
+      let issueKind: Issue.Kind
+      if let expectedCount = expectedCount as? ClosedRange<Int>,
+         expectedCount.lowerBound == expectedCount.upperBound {
+        issueKind = .confirmationMiscounted(actual: actualCount, expected: expectedCount.lowerBound)
+      } else {
+        // TODO: define an issue kind for out-of-range confirmation failures
+        issueKind = .unconditional
+        comment = if let comment {
+          "\(comment) - expected \(expectedCount) confirmations"
+        } else {
+          "expected \(expectedCount) confirmations"
+        }
+      }
       Issue.record(
-        .confirmationMiscounted(actual: actualCount, expected: expectedCount),
+        issueKind,
         comments: Array(comment),
         backtrace: .current(),
         sourceLocation: sourceLocation

Sources/Testing/Running/Runner.swift

@@ -27,8 +27,8 @@ public struct Runner: Sendable {
   ///   - tests: The tests to run.
   ///   - configuration: The configuration to use for running.
   public init(testing tests: [Test], configuration: Configuration = .init()) async {
-    self.plan = await Plan(tests: tests, configuration: configuration)
-    self.configuration = configuration
+    let plan = await Plan(tests: tests, configuration: configuration)
+    self.init(plan: plan, configuration: configuration)
   }
 
   /// Initialize an instance of this type that runs the tests in the specified
@@ -40,6 +40,12 @@ public struct Runner: Sendable {
   public init(plan: Plan, configuration: Configuration = .init()) {
     self.plan = plan
     self.configuration = configuration
+
+    // Post an event for every test in the resulting test plan. These events are
+    // turned into JSON objects if JSON output is enabled.
+    for test in plan.steps.lazy.map(\.test) {
+      Event.post(.testDiscovered, for: test, testCase: nil, configuration: configuration)
+    }
   }
 
   /// Initialize an instance of this type that runs all tests found in the
@@ -321,7 +327,7 @@ extension Runner {
     }
 
     await Configuration.withCurrent(runner.configuration) {
-      Event.post(.runStarted(runner.plan), for: nil, testCase: nil, configuration: runner.configuration)
+      Event.post(.runStarted, for: nil, testCase: nil, configuration: runner.configuration)
       defer {
         Event.post(.runEnded, for: nil, testCase: nil, configuration: runner.configuration)
       }

Sources/Testing/Test.swift

@@ -127,6 +127,23 @@ public struct Test: Sendable {
       return testCases
     }
   }
+  
+  /// Equivalent to ``testCases``, but without requiring that the test cases be
+  /// evaluated first.
+  ///
+  /// Most callers should not use this property and should prefer ``testCases``
+  /// since it will help catch logic errors in the testing library. Use this
+  /// property if you are interested in the test's test cases, but the test has
+  /// not been evaluated by an instance of ``Runner/Plan`` (e.g. if you are
+  /// implementing `swift test list`.)
+  var uncheckedTestCases: (some Sequence<Test.Case>)? {
+    testCasesState.flatMap { testCasesState in
+      if case let .evaluated(testCases) = testCasesState {
+        return testCases
+      }
+      return nil
+    }
+  }
 
   /// Evaluate this test's cases if they have not been evaluated yet.
   ///

Tests/TestingTests/ABIEntryPointTests.swift

@@ -109,6 +109,25 @@ struct ABIEntryPointTests {
     #expect(result)
   }
 
+  @Test("v0 entry point listing tests only")
+  func v0_listingTestsOnly() async throws {
+    var arguments = __CommandLineArguments_v0()
+    arguments.listTests = true
+    arguments.eventStreamVersion = 0
+    arguments.verbosity = .min
+
+    try await confirmation("Test matched", expectedCount: 1...) { testMatched in
+      _ = try await _invokeEntryPointV0(passing: arguments) { recordJSON in
+        let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON)
+        if case .test = record.kind {
+          testMatched()
+        } else {
+          Issue.record("Unexpected record \(record)")
+        }
+      }
+    }
+  }
+
   private func _invokeEntryPointV0(
     passing arguments: __CommandLineArguments_v0,
     recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void = { _ in }

Tests/TestingTests/EventTests.swift

@@ -48,7 +48,7 @@ struct EventTests {
                 sourceLocation: nil)
             )
           ),
-          Event.Kind.runStarted(Runner.Plan(steps: [])),
+          Event.Kind.runStarted,
           Event.Kind.runEnded,
           Event.Kind.testCaseStarted,
           Event.Kind.testCaseEnded,