[WIP] Add an experimental .serialized(.globally) trait.

Known issue: suites don't report they've ended until after all tests have run.

Author: grynspan

Diff for: Sources/Testing/Running/Runner.Plan.swift

@@ -73,6 +73,24 @@ extension Runner {
       }
     }
 
+    /// Stages of a test run.
+    ///
+    /// This enumeration conforms to `CaseIterable`, so callers can iterate over
+    /// all stages by looping over `Stage.allCases`. `Stage.allCases.first` and
+    /// `Stage.allCases.last` are respectively the first and last stages to run.
+    enum Stage: Sendable, CaseIterable {
+      /// Tests that might run in parallel (globally or locally) are being run.
+      case parallelizationAllowed
+
+      /// Tests that are globally serialized are being run.
+      case globallySerialized
+
+      /// The default stage to run tests in.
+      static var `default`: Self {
+        .parallelizationAllowed
+      }
+    }
+
     /// A type describing a step in a runner plan.
     ///
     /// An instance of this type contains a test and the corresponding action an
@@ -83,6 +101,49 @@ extension Runner {
 
       /// The action to perform with ``test``.
       public var action: Action
+
+      /// The stage at which this step should be performed.
+      var stage: Stage = .default
+
+      /// Whether or not this step may perform work over multiple stages of a
+      /// test run.
+      var isMultistaged: Bool {
+        test.isSuite
+      }
+
+      /// Whether or not this step performs its first work in the given test
+      /// run stage.
+      ///
+      /// - Parameters:
+      ///   - stage: The stage of interest.
+      ///
+      /// - Returns: Whether or not `stage` is the first stage in which this
+      ///   step performs some work.
+      func starts(in stage: Stage) -> Bool {
+        let firstStage = if isMultistaged {
+          Stage.allCases.first
+        } else {
+          self.stage
+        }
+        return stage == firstStage
+      }
+
+      /// Whether or not this step performs its final work in the given test
+      /// run stage.
+      ///
+      /// - Parameters:
+      ///   - stage: The stage of interest.
+      ///
+      /// - Returns: Whether or not `stage` is the last stage in which this
+      ///   step performs some work.
+      func ends(in stage: Stage) -> Bool {
+        let lastStage = if isMultistaged {
+          Stage.allCases.last
+        } else {
+          self.stage
+        }
+        return stage == lastStage
+      }
     }
 
     /// The graph of the steps in the runner plan.
@@ -318,9 +379,26 @@ extension Runner.Plan {
       (action, recursivelyApply: action.isRecursive)
     }
 
-    // Zip the tests and actions together and return them.
-    return zip(testGraph, actionGraph).mapValues { _, pair in
-      pair.0.map { Step(test: $0, action: pair.1) }
+    // Figure out what stage each test should operate in.
+    let stageGraph: Graph<String, Runner.Plan.Stage> = testGraph.mapValues { _, test in
+      switch test?.isGloballySerialized {
+      case nil:
+        .default
+      case .some(false):
+        .parallelizationAllowed
+      case .some(true):
+        .globallySerialized
+      }
+    }
+
+    // Zip the tests, actions, and stages together and return them.
+    return zip(zip(testGraph, actionGraph), stageGraph).mapValues { _, tuple in
+      let test = tuple.0.0
+      let action = tuple.0.1
+      let stage = tuple.1
+      return test.map { test in
+        Step(test: test, action: action, stage: stage)
+      }
     }
   }
 

Diff for: Sources/Testing/Running/Runner.swift

@@ -113,11 +113,13 @@ extension Runner {
   ///
   /// - Parameters:
   ///   - sequence: The sequence to enumerate.
+  ///   - stage: The stage at which the runner is operating.
   ///   - body: The function to invoke.
   ///
   /// - Throws: Whatever is thrown by `body`.
   private static func _forEach<E>(
     in sequence: some Sequence<E>,
+    stage: Plan.Stage,
     _ body: @Sendable @escaping (E) async throws -> Void
   ) async throws where E: Sendable {
     try await withThrowingTaskGroup(of: Void.self) { taskGroup in
@@ -128,7 +130,7 @@ extension Runner {
         }
 
         // If not parallelizing, wait after each task.
-        if !_configuration.isParallelizationEnabled {
+        if stage == .globallySerialized || !_configuration.isParallelizationEnabled {
           try await taskGroup.waitForAll()
         }
       }
@@ -139,6 +141,7 @@ extension Runner {
   ///
   /// - Parameters:
   ///   - stepGraph: The subgraph whose root value, a step, is to be run.
+  ///   - stage: The stage at which the runner is operating.
   ///
   /// - Throws: Whatever is thrown from the test body. Thrown errors are
   ///   normally reported as test failures.
@@ -153,63 +156,53 @@ extension Runner {
   /// ## See Also
   ///
   /// - ``Runner/run()``
-  private static func _runStep(atRootOf stepGraph: Graph<String, Plan.Step?>) async throws {
+  private static func _runStep(atRootOf stepGraph: Graph<String, Plan.Step?>, stage: Plan.Stage) async throws {
     // Exit early if the task has already been cancelled.
     try Task.checkCancellation()
 
-    // Whether to send a `.testEnded` event at the end of running this step.
-    // Some steps' actions may not require a final event to be sent — for
-    // example, a skip event only sends `.testSkipped`.
-    let shouldSendTestEnded: Bool
-
     let configuration = _configuration
 
     // Determine what action to take for this step.
-    if let step = stepGraph.value {
+    if let step = stepGraph.value, step.starts(in: stage) {
       Event.post(.planStepStarted(step), for: (step.test, nil), configuration: configuration)
 
       // Determine what kind of event to send for this step based on its action.
       switch step.action {
       case .run:
         Event.post(.testStarted, for: (step.test, nil), configuration: configuration)
-        shouldSendTestEnded = true
       case let .skip(skipInfo):
         Event.post(.testSkipped(skipInfo), for: (step.test, nil), configuration: configuration)
-        shouldSendTestEnded = false
       case let .recordIssue(issue):
         Event.post(.issueRecorded(issue), for: (step.test, nil), configuration: configuration)
-        shouldSendTestEnded = false
       }
-    } else {
-      shouldSendTestEnded = false
     }
     defer {
-      if let step = stepGraph.value {
-        if shouldSendTestEnded {
+      if let step = stepGraph.value, step.ends(in: stage) {
+        if case .run = step.action {
           Event.post(.testEnded, for: (step.test, nil), configuration: configuration)
         }
         Event.post(.planStepEnded(step), for: (step.test, nil), configuration: configuration)
       }
     }
 
-    if let step = stepGraph.value, case .run = step.action {
+    if let step = stepGraph.value, case .run = step.action, step.stage == stage {
       await Test.withCurrent(step.test) {
         _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) {
           try await _applyScopingTraits(for: step.test, testCase: nil) {
             // Run the test function at this step (if one is present.)
             if let testCases = step.test.testCases {
-              try await _runTestCases(testCases, within: step)
+              try await _runTestCases(testCases, within: step, stage: stage)
             }
 
             // Run the children of this test (i.e. the tests in this suite.)
-            try await _runChildren(of: stepGraph)
+            try await _runChildren(of: stepGraph, stage: stage)
           }
         }
       }
     } else {
       // There is no test at this node in the graph, so just skip down to the
       // child nodes.
-      try await _runChildren(of: stepGraph)
+      try await _runChildren(of: stepGraph, stage: stage)
     }
   }
 
@@ -234,10 +227,11 @@ extension Runner {
   /// - Parameters:
   ///   - stepGraph: The subgraph whose root value, a step, will be used to
   ///     find children to run.
+  ///   - stage: The stage at which the runner is operating.
   ///
   /// - Throws: Whatever is thrown from the test body. Thrown errors are
   ///   normally reported as test failures.
-  private static func _runChildren(of stepGraph: Graph<String, Plan.Step?>) async throws {
+  private static func _runChildren(of stepGraph: Graph<String, Plan.Step?>, stage: Plan.Stage) async throws {
     let childGraphs = if _configuration.isParallelizationEnabled {
       // Explicitly shuffle the steps to help detect accidental dependencies
       // between tests due to their ordering.
@@ -267,8 +261,8 @@ extension Runner {
     }
 
     // Run the child nodes.
-    try await _forEach(in: childGraphs) { _, childGraph in
-      try await _runStep(atRootOf: childGraph)
+    try await _forEach(in: childGraphs, stage: stage) { _, childGraph in
+      try await _runStep(atRootOf: childGraph, stage: stage)
     }
   }
 
@@ -277,20 +271,21 @@ extension Runner {
   /// - Parameters:
   ///   - testCases: The test cases to be run.
   ///   - step: The runner plan step associated with this test case.
+  ///   - stage: The stage at which the runner is operating.
   ///
   /// - Throws: Whatever is thrown from a test case's body. Thrown errors are
   ///   normally reported as test failures.
   ///
   /// If parallelization is supported and enabled, the generated test cases will
   /// be run in parallel using a task group.
-  private static func _runTestCases(_ testCases: some Sequence<Test.Case>, within step: Plan.Step) async throws {
+  private static func _runTestCases(_ testCases: some Sequence<Test.Case>, within step: Plan.Step, stage: Plan.Stage) async throws {
     // Apply the configuration's test case filter.
     let testCaseFilter = _configuration.testCaseFilter
     let testCases = testCases.lazy.filter { testCase in
       testCaseFilter(testCase, step.test)
     }
 
-    try await _forEach(in: testCases) { testCase in
+    try await _forEach(in: testCases, stage: stage) { testCase in
       try await _runTestCase(testCase, within: step)
     }
   }
@@ -384,7 +379,9 @@ extension Runner {
 
         await withTaskGroup(of: Void.self) { [runner] taskGroup in
           _ = taskGroup.addTaskUnlessCancelled {
-            try? await _runStep(atRootOf: runner.plan.stepGraph)
+            for stage in Plan.Stage.allCases {
+              try? await _runStep(atRootOf: runner.plan.stepGraph, stage: stage)
+            }
           }
           await taskGroup.waitForAll()
         }

Diff for: Sources/Testing/Testing.docc/Traits.md

@@ -36,6 +36,7 @@ behavior of test functions.
 
 - <doc:Parallelization>
 - ``Trait/serialized``
+<!-- - ``Trait/serialized(_:)`` -->
 
 ### Annotating tests
 

Diff for: Sources/Testing/Testing.docc/Traits/Trait.md

@@ -27,6 +27,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
 ### Running tests serially or in parallel
 
 - ``Trait/serialized``
+<!-- - ``Trait/serialized(_:)`` -->
 
 ### Categorizing tests
 

Diff for: Sources/Testing/Traits/ParallelizationTrait.swift

@@ -24,8 +24,30 @@
 /// globally disabled (by, for example, passing `--no-parallel` to the
 /// `swift test` command.)
 ///
-/// To add this trait to a test, use ``Trait/serialized``.
-public struct ParallelizationTrait: TestTrait, SuiteTrait {}
+/// To add this trait to a test, use ``Trait/serialized`` or
+/// ``Trait/serialized(_:)``.
+public struct ParallelizationTrait: TestTrait, SuiteTrait {
+  /// Scopes in which suites and test functions can be serialized using the
+  /// ``serialized(_:)`` trait.
+  @_spi(Experimental)
+  public enum Scope: Sendable {
+    /// Parallelization is applied locally.
+    ///
+    /// TODO: More blurb.
+    case locally
+
+    /// Parallelization is applied globally.
+    ///
+    /// TODO: More blurb.
+    case globally
+  }
+
+  var scope: Scope
+
+  public var isRecursive: Bool {
+    scope == .globally
+  }
+}
 
 // MARK: - TestScoping
 
@@ -45,10 +67,38 @@ extension ParallelizationTrait: TestScoping {
 extension Trait where Self == ParallelizationTrait {
   /// A trait that serializes the test to which it is applied.
   ///
+  /// This value is equivalent to ``serialized(_:)`` with the argument
+  /// ``ParallelizationTrait/Scope/locally``.
+  ///
   /// ## See Also
   ///
   /// - ``ParallelizationTrait``
   public static var serialized: Self {
-    Self()
+    Self(scope: .locally)
+  }
+
+  /// A trait that serializes the test to which it is applied.
+  ///
+  /// - Parameters:
+  ///   - scope: The scope in which parallelization is enforced.
+  ///
+  /// ## See Also
+  ///
+  /// - ``ParallelizationTrait``
+  @_spi(Experimental)
+  public static func serialized(_ scope: ParallelizationTrait.Scope) -> Self {
+    Self(scope: scope)
+  }
+}
+
+// MARK: -
+
+extension Test {
+  /// Whether or not this test has been globally serialized.
+  var isGloballySerialized: Bool {
+    traits.lazy
+      .compactMap { $0 as? ParallelizationTrait }
+      .map(\.scope)
+      .contains(.globally)
   }
 }