Avoid constructing a Test for an unavailable parameterized test function. (swiftlang#146)

* Revert "Do not evaluate unavailable test function parameters. (swiftlang#133)"

This reverts commit 9e105df.

* Avoid constructing a `Test` for an unavailable parameterized test function.

This PR rethinks the work in swiftlang#133. Instead of lazily evaluating individual arguments, we can instead lazily evaluate the entire test. In the event the test is unavailable, we substitute a zero-argument no-op function so that there is no opportunity for us to touch unavailable type metadata. 🤞🏻

Resolves rdar://118996437.

Author: grynspan

Sources/Testing/Parameterization/Test.Case.Generator.swift

@@ -19,18 +19,14 @@ extension Test.Case {
   ///     ([103416861](rdar://103416861))
   /// }
   struct Generator<S>: Sendable where S: Sequence & Sendable, S.Element: Sendable {
-    /// A closure that produces the underlying sequence of argument values.
+    /// The underlying sequence of argument values.
     ///
-    /// The resulting sequence _must_ be iterable multiple times. Hence,
-    /// initializers accept only _collections_, not sequences. The constraint
-    /// here is only to `Sequence` to allow the storage of computed sequences
-    /// over collections (such as `CartesianProduct` or `Zip2Sequence`) that are
-    /// safe to iterate multiple times.
-    ///
-    /// This property is a closure rather than an instance of `S` so that it can
-    /// be lazily evaluated rather than requiring evaluation as soon as the
-    /// owning instance of ``Test`` is initialized.
-    private var _sequence: @Sendable () async -> S
+    /// The sequence _must_ be iterable multiple times. Hence, initializers
+    /// accept only _collections_, not sequences. The constraint here is only to
+    /// `Sequence` to allow the storage of computed sequences over collections
+    /// (such as `CartesianProduct` or `Zip2Sequence`) that are safe to iterate
+    /// multiple times.
+    private var _sequence: S
 
     /// A closure that maps an element from `_sequence` to a test case instance.
     ///
@@ -43,12 +39,12 @@ extension Test.Case {
     /// Initialize an instance of this type.
     ///
     /// - Parameters:
-    ///   - sequence: A closure that produces the sequence of argument values
-    ///     for which test cases should be generated.
-    ///   - mapElement: A function that maps each element in the result of
-    ///     `sequence` to a corresponding instance of ``Test/Case``.
+    ///   - sequence: The sequence of argument values for which test cases
+    ///     should be generated.
+    ///   - mapElement: A function that maps each element in `sequence` to a
+    ///     corresponding instance of ``Test/Case``.
     private init(
-      sequence: @escaping @Sendable () async -> S,
+      sequence: S,
       mapElement: @escaping @Sendable (_ element: S.Element) -> Test.Case
     ) {
       _sequence = sequence
@@ -65,9 +61,7 @@ extension Test.Case {
     ) where S == CollectionOfOne<Void> {
       // A beautiful hack to give us the right number of cases: iterate over a
       // collection containing a single Void value.
-      self.init {
-        CollectionOfOne(())
-      } mapElement: { _ in
+      self.init(sequence: CollectionOfOne(())) { _ in
         Test.Case(arguments: [], body: testFunction)
       }
     }
@@ -89,7 +83,7 @@ extension Test.Case {
     /// be preferred.
     @_disfavoredOverload
     init(
-      arguments collection: @escaping @Sendable () async -> S,
+      arguments collection: S,
       parameters: [Test.ParameterInfo],
       testFunction: @escaping @Sendable (S.Element) async throws -> Void
     ) where S: Collection {
@@ -128,13 +122,11 @@ extension Test.Case {
     ///   - testFunction: The test function to which each generated test case
     ///     passes an argument value from `collection`.
     init<C1, C2>(
-      arguments collection1: @escaping @Sendable () async -> C1, _ collection2: @escaping @Sendable () async -> C2,
+      arguments collection1: C1, _ collection2: C2,
       parameters: [Test.ParameterInfo],
       testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void
     ) where S == CartesianProduct<C1, C2> {
-      self.init {
-        await cartesianProduct(collection1(), collection2())
-      } mapElement: { element in
+      self.init(sequence: cartesianProduct(collection1, collection2)) { element in
         Test.Case(values: [element.0, element.1], parameters: parameters) {
           try await testFunction(element.0, element.1)
         }
@@ -160,7 +152,7 @@ extension Test.Case {
     ///     ([103416861](rdar://103416861))
     /// }
     private init<E1, E2>(
-      sequence: @escaping @Sendable () async -> S,
+      sequence: S,
       parameters: [Test.ParameterInfo],
       testFunction: @escaping @Sendable ((E1, E2)) async throws -> Void
     ) where S.Element == (E1, E2), E1: Sendable, E2: Sendable {
@@ -198,7 +190,7 @@ extension Test.Case {
     ///     ([103416861](rdar://103416861))
     /// }
     init<E1, E2>(
-      arguments collection: @escaping @Sendable () async -> S,
+      arguments collection: S,
       parameters: [Test.ParameterInfo],
       testFunction: @escaping @Sendable ((E1, E2)) async throws -> Void
     ) where S: Collection, S.Element == (E1, E2) {
@@ -216,7 +208,7 @@ extension Test.Case {
     ///   - testFunction: The test function to which each generated test case
     ///     passes an argument value from `zippedCollections`.
     init<C1, C2>(
-      arguments zippedCollections: @escaping @Sendable () async -> Zip2Sequence<C1, C2>,
+      arguments zippedCollections: Zip2Sequence<C1, C2>,
       parameters: [Test.ParameterInfo],
       testFunction: @escaping @Sendable ((C1.Element, C2.Element)) async throws -> Void
     ) where S == Zip2Sequence<C1, C2>, C1: Collection, C2: Collection {
@@ -240,7 +232,7 @@ extension Test.Case {
     /// collections of 2-tuples because the `Element` tuple type for
     /// `Dictionary` includes labels (`(key: Key, value: Value)`).
     init<Key, Value>(
-      arguments dictionary: @escaping @Sendable () async -> Dictionary<Key, Value>,
+      arguments dictionary: Dictionary<Key, Value>,
       parameters: [Test.ParameterInfo],
       testFunction: @escaping @Sendable ((Key, Value)) async throws -> Void
     ) where S == Dictionary<Key, Value> {
@@ -261,17 +253,14 @@ extension Test.Case {
   }
 }
 
-// MARK: - Sequence generation
+// MARK: - Sequence
 
-extension Test.Case.Generator {
-  /// Generate a sequence of test cases corresponding to the sequence of
-  /// elements passed to this instance during instantiation.
-  ///
-  /// - Returns:
-  ///   A sequence of ``Test/Case`` instances.
-  ///
-  /// Each call to this function generates a new sequence.
-  func generate() async -> some Sequence<Test.Case> {
-    await _sequence().lazy.map(_mapElement)
+extension Test.Case.Generator: Sequence {
+  func makeIterator() -> some IteratorProtocol<Test.Case> {
+    _sequence.lazy.map(_mapElement).makeIterator()
+  }
+
+  var underestimatedCount: Int {
+    _sequence.underestimatedCount
   }
 }

Sources/Testing/Running/Runner.swift

@@ -171,7 +171,7 @@ extension Runner {
       }
     }
 
-    if let step = stepGraph.value, case .run = step.action, let testCases = await step.test.testCases {
+    if let step = stepGraph.value, case .run = step.action, let testCases = step.test.testCases {
       try await Test.withCurrent(step.test) {
         try await _withErrorHandling(for: step, sourceLocation: step.test.sourceLocation) {
           try await _runTestCases(testCases, within: step)

Sources/Testing/Test+Macro.swift

@@ -235,7 +235,7 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments collection: @escaping @Sendable () async -> C,
+    arguments collection: C,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__ParameterInfo],
     testFunction: @escaping @Sendable (C.Element) async throws -> Void
@@ -363,7 +363,7 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments collection1: @escaping @Sendable () async -> C1, _ collection2: @escaping @Sendable () async -> C2,
+    arguments collection1: C1, _ collection2: C2,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__ParameterInfo],
     testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void
@@ -386,7 +386,7 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments collection: @escaping @Sendable () async -> C,
+    arguments collection: C,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__ParameterInfo],
     testFunction: @escaping @Sendable ((E1, E2)) async throws -> Void
@@ -412,7 +412,7 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments dictionary: @escaping @Sendable () async -> Dictionary<Key, Value>,
+    arguments dictionary: Dictionary<Key, Value>,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__ParameterInfo],
     testFunction: @escaping @Sendable ((Key, Value)) async throws -> Void
@@ -432,7 +432,7 @@ extension Test {
     xcTestCompatibleSelector: __XCTestCompatibleSelector?,
     displayName: String? = nil,
     traits: [any TestTrait],
-    arguments zippedCollections: @escaping @Sendable () async -> Zip2Sequence<C1, C2>,
+    arguments zippedCollections: Zip2Sequence<C1, C2>,
     sourceLocation: SourceLocation,
     parameters paramTuples: [__ParameterInfo],
     testFunction: @escaping @Sendable (C1.Element, C2.Element) async throws -> Void

Sources/Testing/Test.swift

@@ -53,6 +53,32 @@ public struct Test: Sendable {
   /// The source location of this test.
   public var sourceLocation: SourceLocation
 
+  /// The (underestimated) number of iterations that will need to occur during
+  /// testing.
+  ///
+  /// The value of this property is inherently capped at `Int.max`. In practice,
+  /// the number of iterations that can run in a reasonable timespan will be
+  /// significantly lower.
+  ///
+  /// For instances of ``Test`` that represent non-parameterized test functions
+  /// (that is, test functions that do not iterate over a sequence of inputs),
+  /// the value of this property is always `1`. For instances of ``Test`` that
+  /// represent test suite types, the value of this property is always `nil`.
+  ///
+  /// For more information about underestimated counts, see the documentation
+  /// for [`Sequence`](https://developer.apple.com/documentation/swift/array/underestimatedcount-4ggqp).
+  @_spi(ExperimentalParameterizedTesting)
+  public var underestimatedCaseCount: Int? {
+    // NOTE: it is important that we only expose an _underestimated_ count for
+    // two reasons:
+    // 1. If the total number of cases exceeds `.max` due to combinatoric
+    //    complexity, `count` would be too low; and
+    // 2. We reserve the right to support async sequences as input in the
+    //    future, and async sequences do not have `count` properties (but an
+    //    underestimated count of `0` is still technically correct.)
+    testCases?.underestimatedCount
+  }
+
   /// The type containing this test, if any.
   ///
   /// If a test is associated with a free function or static function, the value
@@ -70,29 +96,22 @@ public struct Test: Sendable {
 
   /// Storage for the ``testCases`` property.
   ///
-  /// This use of `AnySequence` is necessary because it is not currently
-  /// possible to express `Sequence<Test.Case> & Sendable` as an existential
-  /// (`any`) ([96960993](rdar://96960993)). It is also not possible to have a
-  /// value of an underlying generic sequence type without specifying its
-  /// generic parameters.
-  private var _testCases: (@Sendable () async -> AnySequence<Test.Case>)?
+  /// This use of `UncheckedSendable` and of `AnySequence` is necessary because
+  /// it is not currently possible to express `Sequence<Test.Case> & Sendable`
+  /// as an existential (`any`) ([96960993](rdar://96960993)). It is also not
+  /// possible to have a value of an underlying generic sequence type without
+  /// specifying its generic parameters.
+  private var _testCases: UncheckedSendable<AnySequence<Test.Case>>?
 
   /// The set of test cases associated with this test, if any.
   ///
   /// For parameterized tests, each test case is associated with a single
   /// combination of parameterized inputs. For non-parameterized tests, a single
   /// test case is synthesized. For test suite types (as opposed to test
   /// functions), the value of this property is `nil`.
-  ///
-  /// - Warning: The parameterized inputs to a test may have limited
-  ///   availability if the test has the `@available` attribute applied to it.
-  ///   This property does not evaluate availability, and the effect of reading
-  ///   it on a platform where the inputs are unavailable is undefined.
   @_spi(ExperimentalParameterizedTesting)
   public var testCases: (some Sequence<Test.Case> & Sendable)? {
-    get async {
-      await _testCases?()
-    }
+    _testCases?.rawValue
   }
 
   /// Whether or not this test is parameterized.
@@ -121,7 +140,7 @@ public struct Test: Sendable {
   ///
   /// A test suite can be declared using the ``Suite(_:_:)`` macro.
   public var isSuite: Bool {
-    containingType != nil && _testCases == nil
+    containingType != nil && testCases == nil
   }
 
   /// Initialize an instance of this type representing a test suite type.
@@ -156,7 +175,7 @@ public struct Test: Sendable {
     self.sourceLocation = sourceLocation
     self.containingType = containingType
     self.xcTestCompatibleSelector = xcTestCompatibleSelector
-    self._testCases = { await .init(testCases.generate()) }
+    self._testCases = .init(rawValue: .init(testCases))
     self.parameters = parameters
   }
 }

Sources/TestingMacros/Support/AttributeDiscovery.swift

@@ -218,10 +218,7 @@ struct AttributeInfo {
         ArrayElementSyntax(expression: traitExpr)
       }
     }))
-    // TODO: extract arguments: ... here, rather than assuming all "other" arguments are parameterized inputs
-    arguments += otherArguments.map { argument in
-      Argument(label: argument.label, expression: "{ \(argument.expression.trimmed) }")
-    }
+    arguments += otherArguments
     arguments.append(Argument(label: "sourceLocation", expression: sourceLocation))
 
     return LabeledExprListSyntax(arguments)