Skip to content

Commit 94e7873

Browse files
authored
[6.1] [SWT-0007] Introduce API allowing traits to customize test execution (#907)
- **Explanation**: Include the Test Scoping Traits feature and proposal in Swift 6.1. - **Scope**: Adds an important and frequently requested new feature to the 6.1 release. Removes an SPI which may be in use, but was always intended to be refined/replaced. - **Issues**: n/a - **Original PRs**: #733, #900 - **Risk**: Low, new functionality - **Testing**: New APIs have unit tests. - **Reviewers**: @grynspan, @briancroom
1 parent 4a09505 commit 94e7873

File tree

7 files changed

+883
-161
lines changed

7 files changed

+883
-161
lines changed

Diff for: Documentation/Proposals/0007-test-scoping-traits.md

+510
Large diffs are not rendered by default.

Diff for: Sources/Testing/Running/Runner.swift

+19-22
Original file line numberDiff line numberDiff line change
@@ -56,45 +56,42 @@ public struct Runner: Sendable {
5656
// MARK: - Running tests
5757

5858
extension Runner {
59-
/// Execute the ``CustomExecutionTrait/execute(_:for:testCase:)`` functions
60-
/// associated with the test in a plan step.
59+
/// Apply the custom scope for any test scope providers of the traits
60+
/// associated with a specified test by calling their
61+
/// ``TestScoping/provideScope(for:testCase:performing:)`` function.
6162
///
6263
/// - Parameters:
63-
/// - step: The step being performed.
64-
/// - testCase: The test case, if applicable, for which to execute the
65-
/// custom trait.
64+
/// - test: The test being run, for which to provide custom scope.
65+
/// - testCase: The test case, if applicable, for which to provide custom
66+
/// scope.
6667
/// - body: A function to execute from within the
67-
/// ``CustomExecutionTrait/execute(_:for:testCase:)`` functions of each
68-
/// trait applied to `step.test`.
68+
/// ``TestScoping/provideScope(for:testCase:performing:)`` function of
69+
/// each non-`nil` scope provider of the traits applied to `test`.
6970
///
7071
/// - Throws: Whatever is thrown by `body` or by any of the
71-
/// ``CustomExecutionTrait/execute(_:for:testCase:)`` functions.
72-
private func _executeTraits(
73-
for step: Plan.Step,
72+
/// ``TestScoping/provideScope(for:testCase:performing:)`` function calls.
73+
private func _applyScopingTraits(
74+
for test: Test,
7475
testCase: Test.Case?,
7576
_ body: @escaping @Sendable () async throws -> Void
7677
) async throws {
7778
// If the test does not have any traits, exit early to avoid unnecessary
7879
// heap allocations below.
79-
if step.test.traits.isEmpty {
80-
return try await body()
81-
}
82-
83-
if case .skip = step.action {
80+
if test.traits.isEmpty {
8481
return try await body()
8582
}
8683

8784
// Construct a recursive function that invokes each trait's ``execute(_:for:testCase:)``
8885
// function. The order of the sequence is reversed so that the last trait is
8986
// the one that invokes body, then the second-to-last invokes the last, etc.
9087
// and ultimately the first trait is the first one to be invoked.
91-
let executeAllTraits = step.test.traits.lazy
88+
let executeAllTraits = test.traits.lazy
9289
.reversed()
93-
.compactMap { $0 as? any CustomExecutionTrait }
94-
.compactMap { $0.execute(_:for:testCase:) }
95-
.reduce(body) { executeAllTraits, traitExecutor in
90+
.compactMap { $0.scopeProvider(for: test, testCase: testCase) }
91+
.map { $0.provideScope(for:testCase:performing:) }
92+
.reduce(body) { executeAllTraits, provideScope in
9693
{
97-
try await traitExecutor(executeAllTraits, step.test, testCase)
94+
try await provideScope(test, testCase, executeAllTraits)
9895
}
9996
}
10097

@@ -200,7 +197,7 @@ extension Runner {
200197
if let step = stepGraph.value, case .run = step.action {
201198
await Test.withCurrent(step.test) {
202199
_ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) {
203-
try await _executeTraits(for: step, testCase: nil) {
200+
try await _applyScopingTraits(for: step.test, testCase: nil) {
204201
// Run the test function at this step (if one is present.)
205202
if let testCases = step.test.testCases {
206203
try await _runTestCases(testCases, within: step)
@@ -336,7 +333,7 @@ extension Runner {
336333
let sourceLocation = step.test.sourceLocation
337334
await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) {
338335
try await withTimeLimit(for: step.test, configuration: configuration) {
339-
try await _executeTraits(for: step, testCase: testCase) {
336+
try await _applyScopingTraits(for: step.test, testCase: testCase) {
340337
try await testCase.body()
341338
}
342339
} timeoutHandler: { timeLimit in

Diff for: Sources/Testing/Testing.docc/Traits.md

+1
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ behavior of test functions.
5353
- ``Trait``
5454
- ``TestTrait``
5555
- ``SuiteTrait``
56+
- ``TestScoping``
5657

5758
### Supporting types
5859

Diff for: Sources/Testing/Testing.docc/Traits/Trait.md

+7
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,15 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
3939
- ``Trait/bug(_:id:_:)-3vtpl``
4040

4141
### Adding information to tests
42+
4243
- ``Trait/comments``
4344

4445
### Preparing internal state
4546

4647
- ``Trait/prepare(for:)-3s3zo``
48+
49+
### Providing custom execution scope for tests
50+
51+
- ``TestScoping``
52+
- ``Trait/scopeProvider(for:testCase:)-cjmg``
53+
- ``Trait/TestScopeProvider``

Diff for: Sources/Testing/Traits/Trait.swift

+162-35
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,150 @@ public protocol Trait: Sendable {
4141
///
4242
/// By default, the value of this property is an empty array.
4343
var comments: [Comment] { get }
44+
45+
/// The type of the test scope provider for this trait.
46+
///
47+
/// The default type is `Never`, which cannot be instantiated. The
48+
/// ``scopeProvider(for:testCase:)-cjmg`` method for any trait with this
49+
/// default type must return `nil`, meaning that trait will not provide a
50+
/// custom scope for the tests it's applied to.
51+
associatedtype TestScopeProvider: TestScoping = Never
52+
53+
/// Get this trait's scope provider for the specified test and/or test case,
54+
/// if any.
55+
///
56+
/// - Parameters:
57+
/// - test: The test for which a scope provider is being requested.
58+
/// - testCase: The test case for which a scope provider is being requested,
59+
/// if any. When `test` represents a suite, the value of this argument is
60+
/// `nil`.
61+
///
62+
/// - Returns: A value conforming to ``Trait/TestScopeProvider`` which may be
63+
/// used to provide custom scoping for `test` and/or `testCase`, or `nil` if
64+
/// they should not have any custom scope.
65+
///
66+
/// If this trait's type conforms to ``TestScoping``, the default value
67+
/// returned by this method depends on `test` and/or `testCase`:
68+
///
69+
/// - If `test` represents a suite, this trait must conform to ``SuiteTrait``.
70+
/// If the value of this suite trait's ``SuiteTrait/isRecursive`` property
71+
/// is `true`, then this method returns `nil`; otherwise, it returns `self`.
72+
/// This means that by default, a suite trait will _either_ provide its
73+
/// custom scope once for the entire suite, or once per-test function it
74+
/// contains.
75+
/// - Otherwise `test` represents a test function. If `testCase` is `nil`,
76+
/// this method returns `nil`; otherwise, it returns `self`. This means that
77+
/// by default, a trait which is applied to or inherited by a test function
78+
/// will provide its custom scope once for each of that function's cases.
79+
///
80+
/// A trait may explicitly implement this method to further customize the
81+
/// default behaviors above. For example, if a trait should provide custom
82+
/// test scope both once per-suite and once per-test function in that suite,
83+
/// it may implement the method and return a non-`nil` scope provider under
84+
/// those conditions.
85+
///
86+
/// A trait may also implement this method and return `nil` if it determines
87+
/// that it does not need to provide a custom scope for a particular test at
88+
/// runtime, even if the test has the trait applied. This can improve
89+
/// performance and make diagnostics clearer by avoiding an unnecessary call
90+
/// to ``TestScoping/provideScope(for:testCase:performing:)``.
91+
///
92+
/// If this trait's type does not conform to ``TestScoping`` and its
93+
/// associated ``Trait/TestScopeProvider`` type is the default `Never`, then
94+
/// this method returns `nil` by default. This means that instances of this
95+
/// trait will not provide a custom scope for tests to which they're applied.
96+
func scopeProvider(for test: Test, testCase: Test.Case?) -> TestScopeProvider?
97+
}
98+
99+
/// A protocol that allows providing a custom execution scope for a test
100+
/// function (and each of its cases) or a test suite by performing custom code
101+
/// before or after it runs.
102+
///
103+
/// Types conforming to this protocol may be used in conjunction with a
104+
/// ``Trait``-conforming type by implementing the
105+
/// ``Trait/scopeProvider(for:testCase:)-cjmg`` method, allowing custom traits
106+
/// to provide custom scope for tests. Consolidating common set-up and tear-down
107+
/// logic for tests which have similar needs allows each test function to be
108+
/// more succinct with less repetitive boilerplate so it can focus on what makes
109+
/// it unique.
110+
public protocol TestScoping: Sendable {
111+
/// Provide custom execution scope for a function call which is related to the
112+
/// specified test and/or test case.
113+
///
114+
/// - Parameters:
115+
/// - test: The test under which `function` is being performed.
116+
/// - testCase: The test case, if any, under which `function` is being
117+
/// performed. When invoked on a suite, the value of this argument is
118+
/// `nil`.
119+
/// - function: The function to perform. If `test` represents a test suite,
120+
/// this function encapsulates running all the tests in that suite. If
121+
/// `test` represents a test function, this function is the body of that
122+
/// test function (including all cases if it is parameterized.)
123+
///
124+
/// - Throws: Whatever is thrown by `function`, or an error preventing this
125+
/// type from providing a custom scope correctly. An error thrown from this
126+
/// method is recorded as an issue associated with `test`. If an error is
127+
/// thrown before `function` is called, the corresponding test will not run.
128+
///
129+
/// When the testing library is preparing to run a test, it starts by finding
130+
/// all traits applied to that test, including those inherited from containing
131+
/// suites. It begins with inherited suite traits, sorting them
132+
/// outermost-to-innermost, and if the test is a function, it then adds all
133+
/// traits applied directly to that functions in the order they were applied
134+
/// (left-to-right). It then asks each trait for its scope provider (if any)
135+
/// by calling ``Trait/scopeProvider(for:testCase:)-cjmg``. Finally, it calls
136+
/// this method on all non-`nil` scope providers, giving each an opportunity
137+
/// to perform arbitrary work before or after invoking `function`.
138+
///
139+
/// This method should either invoke `function` once before returning or throw
140+
/// an error if it is unable to provide a custom scope.
141+
///
142+
/// Issues recorded by this method are associated with `test`.
143+
func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws
144+
}
145+
146+
extension Trait where Self: TestScoping {
147+
/// Get this trait's scope provider for the specified test and/or test case,
148+
/// if any.
149+
///
150+
/// - Parameters:
151+
/// - test: The test for which a scope provider is being requested.
152+
/// - testCase: The test case for which a scope provider is being requested,
153+
/// if any. When `test` represents a suite, the value of this argument is
154+
/// `nil`.
155+
///
156+
/// This default implementation is used when this trait type conforms to
157+
/// ``TestScoping`` and its return value is discussed in
158+
/// ``Trait/scopeProvider(for:testCase:)-cjmg``.
159+
public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? {
160+
testCase == nil ? nil : self
161+
}
162+
}
163+
164+
extension SuiteTrait where Self: TestScoping {
165+
/// Get this trait's scope provider for the specified test and/or test case,
166+
/// if any.
167+
///
168+
/// - Parameters:
169+
/// - test: The test for which a scope provider is being requested.
170+
/// - testCase: The test case for which a scope provider is being requested,
171+
/// if any. When `test` represents a suite, the value of this argument is
172+
/// `nil`.
173+
///
174+
/// This default implementation is used when this trait type conforms to
175+
/// ``TestScoping`` and its return value is discussed in
176+
/// ``Trait/scopeProvider(for:testCase:)-cjmg``.
177+
public func scopeProvider(for test: Test, testCase: Test.Case?) -> Self? {
178+
if test.isSuite {
179+
isRecursive ? nil : self
180+
} else {
181+
testCase == nil ? nil : self
182+
}
183+
}
184+
}
185+
186+
extension Never: TestScoping {
187+
public func provideScope(for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void) async throws {}
44188
}
45189

46190
/// A protocol describing traits that can be added to a test function.
@@ -72,43 +216,26 @@ extension Trait {
72216
}
73217
}
74218

219+
extension Trait where TestScopeProvider == Never {
220+
/// Get this trait's scope provider for the specified test and/or test case,
221+
/// if any.
222+
///
223+
/// - Parameters:
224+
/// - test: The test for which a scope provider is being requested.
225+
/// - testCase: The test case for which a scope provider is being requested,
226+
/// if any. When `test` represents a suite, the value of this argument is
227+
/// `nil`.
228+
///
229+
/// This default implementation is used when this trait type's associated
230+
/// ``Trait/TestScopeProvider`` type is the default value of `Never`, and its
231+
/// return value is discussed in ``Trait/scopeProvider(for:testCase:)-cjmg``.
232+
public func scopeProvider(for test: Test, testCase: Test.Case?) -> Never? {
233+
nil
234+
}
235+
}
236+
75237
extension SuiteTrait {
76238
public var isRecursive: Bool {
77239
false
78240
}
79241
}
80-
81-
/// A protocol extending ``Trait`` that offers an additional customization point
82-
/// for trait authors to execute code before and after each test function (if
83-
/// added to the traits of a test function), or before and after each test suite
84-
/// (if added to the traits of a test suite).
85-
@_spi(Experimental)
86-
public protocol CustomExecutionTrait: Trait {
87-
88-
/// Execute a function with the effects of this trait applied.
89-
///
90-
/// - Parameters:
91-
/// - function: The function to perform. If `test` represents a test suite,
92-
/// this function encapsulates running all the tests in that suite. If
93-
/// `test` represents a test function, this function is the body of that
94-
/// test function (including all cases if it is parameterized.)
95-
/// - test: The test under which `function` is being performed.
96-
/// - testCase: The test case, if any, under which `function` is being
97-
/// performed. This is `nil` when invoked on a suite.
98-
///
99-
/// - Throws: Whatever is thrown by `function`, or an error preventing the
100-
/// trait from running correctly.
101-
///
102-
/// This function is called for each ``CustomExecutionTrait`` on a test suite
103-
/// or test function and allows additional work to be performed before and
104-
/// after the test runs.
105-
///
106-
/// This function is invoked once for the test it is applied to, and then once
107-
/// for each test case in that test, if applicable.
108-
///
109-
/// Issues recorded by this function are recorded against `test`.
110-
///
111-
/// - Note: If a test function or test suite is skipped, this function does
112-
/// not get invoked by the runner.
113-
func execute(_ function: @Sendable () async throws -> Void, for test: Test, testCase: Test.Case?) async throws
114-
}

0 commit comments

Comments
 (0)