Skip to content

Commit 6f87c49

Browse files
grynspangrynspan
committed
Capturing values in exit tests
This PR implements an experimental form of state capture in exit tests. If you specify a capture list on the test's body closure and explicitly write the type of each captured value, _and_ each value conforms to `Sendable`, `Codable`, and (implicitly) `Copyable`, we'll encode them and send them "over the wire" to the child process: ```swift let a = Int.random(in: 100 ..< 200) await #expect(exitsWith: .failure) { [a = a as Int] in assert(a > 500) } ``` This PR is incomplete. Among other details: - [] Need to properly transmit the data, not stuff it in an environment variable - [] Need to capture source location information correctly for error handling during value decoding - [] Need to implement diagnostics correctly We are ultimately constrained by the language here as we don't have real type information for the captured values, nor can we infer captures by inspecting the syntax of the exit test body (hence the need for an explicit capture list with types.) If we had something like `decltype()` we could apply during macro expansion, you wouldn't need to write `x = x as T` and could just write `x`.
1 parent 1eba9c0 commit 6f87c49

17 files changed

+1165
-127
lines changed

Diff for: ‎Package.swift

+8
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,14 @@ extension Array where Element == PackageDescription.SwiftSetting {
221221
.define("SWT_NO_PIPES", .when(platforms: [.wasi])),
222222
]
223223

224+
// Unconditionally enable 'ExperimentalExitTestValueCapture' when building
225+
// for development.
226+
if buildingForDevelopment {
227+
result += [
228+
.define("ExperimentalExitTestValueCapture")
229+
]
230+
}
231+
224232
return result
225233
}
226234

Diff for: ‎[email protected]

+310
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,310 @@
1+
// swift-tools-version: 6.1
2+
3+
//
4+
// This source file is part of the Swift.org open source project
5+
//
6+
// Copyright (c) 2023 Apple Inc. and the Swift project authors
7+
// Licensed under Apache License v2.0 with Runtime Library Exception
8+
//
9+
// See https://swift.org/LICENSE.txt for license information
10+
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
11+
//
12+
13+
import PackageDescription
14+
import CompilerPluginSupport
15+
16+
/// Information about the current state of the package's git repository.
17+
let git = Context.gitInformation
18+
19+
/// Whether or not this package is being built for development rather than
20+
/// distribution as a package dependency.
21+
let buildingForDevelopment = (git?.currentTag == nil)
22+
23+
let package = Package(
24+
name: "swift-testing",
25+
26+
platforms: [
27+
.macOS(.v10_15),
28+
.iOS(.v13),
29+
.watchOS(.v6),
30+
.tvOS(.v13),
31+
.macCatalyst(.v13),
32+
.visionOS(.v1),
33+
],
34+
35+
products: {
36+
var result = [Product]()
37+
38+
#if os(Windows)
39+
result.append(
40+
.library(
41+
name: "Testing",
42+
type: .dynamic, // needed so Windows exports ABI entry point symbols
43+
targets: ["Testing"]
44+
)
45+
)
46+
#else
47+
result.append(
48+
.library(
49+
name: "Testing",
50+
targets: ["Testing"]
51+
)
52+
)
53+
#endif
54+
55+
result.append(
56+
.library(
57+
name: "_TestDiscovery",
58+
type: .static,
59+
targets: ["_TestDiscovery"]
60+
)
61+
)
62+
63+
return result
64+
}(),
65+
66+
traits: [
67+
.trait(
68+
name: "ExperimentalExitTestValueCapture",
69+
description: "Enable experimental support for capturing values in exit tests"
70+
),
71+
],
72+
73+
dependencies: [
74+
.package(url: "https://github.com/swiftlang/swift-syntax.git", from: "601.0.0-latest"),
75+
],
76+
77+
targets: [
78+
.target(
79+
name: "Testing",
80+
dependencies: [
81+
"_TestDiscovery",
82+
"_TestingInternals",
83+
"TestingMacros",
84+
],
85+
exclude: ["CMakeLists.txt", "Testing.swiftcrossimport"],
86+
cxxSettings: .packageSettings,
87+
swiftSettings: .packageSettings + .enableLibraryEvolution(),
88+
linkerSettings: [
89+
.linkedLibrary("execinfo", .when(platforms: [.custom("freebsd"), .openbsd]))
90+
]
91+
),
92+
.testTarget(
93+
name: "TestingTests",
94+
dependencies: [
95+
"Testing",
96+
"_Testing_CoreGraphics",
97+
"_Testing_Foundation",
98+
],
99+
swiftSettings: .packageSettings
100+
),
101+
102+
.macro(
103+
name: "TestingMacros",
104+
dependencies: [
105+
.product(name: "SwiftDiagnostics", package: "swift-syntax"),
106+
.product(name: "SwiftSyntax", package: "swift-syntax"),
107+
.product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
108+
.product(name: "SwiftParser", package: "swift-syntax"),
109+
.product(name: "SwiftSyntaxMacros", package: "swift-syntax"),
110+
.product(name: "SwiftCompilerPlugin", package: "swift-syntax"),
111+
],
112+
exclude: ["CMakeLists.txt"],
113+
swiftSettings: .packageSettings + {
114+
var result = [PackageDescription.SwiftSetting]()
115+
116+
// The only target which needs the ability to import this macro
117+
// implementation target's module is its unit test target. Users of the
118+
// macros this target implements use them via their declarations in the
119+
// Testing module. This target's module is never distributed to users,
120+
// but as an additional guard against accidental misuse, this specifies
121+
// the unit test target as the only allowable client.
122+
if buildingForDevelopment {
123+
result.append(.unsafeFlags(["-Xfrontend", "-allowable-client", "-Xfrontend", "TestingMacrosTests"]))
124+
}
125+
126+
return result
127+
}()
128+
),
129+
130+
// "Support" targets: These targets are not meant to be used directly by
131+
// test authors.
132+
.target(
133+
name: "_TestingInternals",
134+
exclude: ["CMakeLists.txt"],
135+
cxxSettings: .packageSettings
136+
),
137+
.target(
138+
name: "_TestDiscovery",
139+
dependencies: ["_TestingInternals",],
140+
exclude: ["CMakeLists.txt"],
141+
cxxSettings: .packageSettings,
142+
swiftSettings: .packageSettings
143+
),
144+
145+
// Cross-import overlays (not supported by Swift Package Manager)
146+
.target(
147+
name: "_Testing_CoreGraphics",
148+
dependencies: [
149+
"Testing",
150+
],
151+
path: "Sources/Overlays/_Testing_CoreGraphics",
152+
swiftSettings: .packageSettings + .enableLibraryEvolution()
153+
),
154+
.target(
155+
name: "_Testing_Foundation",
156+
dependencies: [
157+
"Testing",
158+
],
159+
path: "Sources/Overlays/_Testing_Foundation",
160+
exclude: ["CMakeLists.txt"],
161+
// The Foundation module only has Library Evolution enabled on Apple
162+
// platforms, and since this target's module publicly imports Foundation,
163+
// it can only enable Library Evolution itself on those platforms.
164+
swiftSettings: .packageSettings + .enableLibraryEvolution(applePlatformsOnly: true)
165+
),
166+
167+
// Utility targets: These are utilities intended for use when developing
168+
// this package, not for distribution.
169+
.executableTarget(
170+
name: "SymbolShowcase",
171+
dependencies: [
172+
"Testing",
173+
],
174+
swiftSettings: .packageSettings
175+
),
176+
],
177+
178+
cxxLanguageStandard: .cxx20
179+
)
180+
181+
// BUG: swift-package-manager-#6367
182+
#if !os(Windows) && !os(FreeBSD) && !os(OpenBSD)
183+
package.targets.append(contentsOf: [
184+
.testTarget(
185+
name: "TestingMacrosTests",
186+
dependencies: [
187+
"Testing",
188+
"TestingMacros",
189+
],
190+
swiftSettings: .packageSettings
191+
)
192+
])
193+
#endif
194+
195+
extension Array where Element == PackageDescription.SwiftSetting {
196+
/// Settings intended to be applied to every Swift target in this package.
197+
/// Analogous to project-level build settings in an Xcode project.
198+
static var packageSettings: Self {
199+
var result = availabilityMacroSettings
200+
201+
if buildingForDevelopment {
202+
result.append(.unsafeFlags(["-require-explicit-sendable"]))
203+
}
204+
205+
result += [
206+
.enableUpcomingFeature("ExistentialAny"),
207+
208+
.enableExperimentalFeature("AccessLevelOnImport"),
209+
.enableUpcomingFeature("InternalImportsByDefault"),
210+
211+
.enableUpcomingFeature("MemberImportVisibility"),
212+
213+
// This setting is enabled in the package, but not in the toolchain build
214+
// (via CMake). Enabling it is dependent on acceptance of the @section
215+
// proposal via Swift Evolution.
216+
.enableExperimentalFeature("SymbolLinkageMarkers"),
217+
218+
// When building as a package, the macro plugin always builds as an
219+
// executable rather than a library.
220+
.define("SWT_NO_LIBRARY_MACRO_PLUGINS"),
221+
222+
.define("SWT_TARGET_OS_APPLE", .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS])),
223+
224+
.define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])),
225+
.define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])),
226+
.define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android])),
227+
.define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])),
228+
.define("SWT_NO_PIPES", .when(platforms: [.wasi])),
229+
]
230+
231+
// Unconditionally enable 'ExperimentalExitTestValueCapture' when building
232+
// for development.
233+
if buildingForDevelopment {
234+
result += [
235+
.define("ExperimentalExitTestValueCapture")
236+
]
237+
}
238+
239+
return result
240+
}
241+
242+
/// Settings which define commonly-used OS availability macros.
243+
///
244+
/// These leverage a pseudo-experimental feature in the Swift compiler for
245+
/// setting availability definitions, which was added in
246+
/// [swift#65218](https://github.com/swiftlang/swift/pull/65218).
247+
private static var availabilityMacroSettings: Self {
248+
[
249+
.enableExperimentalFeature("AvailabilityMacro=_mangledTypeNameAPI:macOS 11.0, iOS 14.0, watchOS 7.0, tvOS 14.0"),
250+
.enableExperimentalFeature("AvailabilityMacro=_uttypesAPI:macOS 11.0, iOS 14.0, watchOS 7.0, tvOS 14.0"),
251+
.enableExperimentalFeature("AvailabilityMacro=_backtraceAsyncAPI:macOS 12.0, iOS 15.0, watchOS 8.0, tvOS 15.0"),
252+
.enableExperimentalFeature("AvailabilityMacro=_clockAPI:macOS 13.0, iOS 16.0, watchOS 9.0, tvOS 16.0"),
253+
.enableExperimentalFeature("AvailabilityMacro=_regexAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0"),
254+
.enableExperimentalFeature("AvailabilityMacro=_swiftVersionAPI:macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0"),
255+
.enableExperimentalFeature("AvailabilityMacro=_typedThrowsAPI:macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0"),
256+
257+
.enableExperimentalFeature("AvailabilityMacro=_distantFuture:macOS 99.0, iOS 99.0, watchOS 99.0, tvOS 99.0, visionOS 99.0"),
258+
]
259+
}
260+
261+
/// Create a Swift setting which enables Library Evolution, optionally
262+
/// constraining it to only Apple platforms.
263+
///
264+
/// - Parameters:
265+
/// - applePlatformsOnly: Whether to constrain this setting to only Apple
266+
/// platforms.
267+
static func enableLibraryEvolution(applePlatformsOnly: Bool = false) -> Self {
268+
var result = [PackageDescription.SwiftSetting]()
269+
270+
if buildingForDevelopment {
271+
var condition: BuildSettingCondition?
272+
if applePlatformsOnly {
273+
condition = .when(platforms: [.macOS, .iOS, .macCatalyst, .watchOS, .tvOS, .visionOS])
274+
}
275+
result.append(.unsafeFlags(["-enable-library-evolution"], condition))
276+
}
277+
278+
return result
279+
}
280+
}
281+
282+
extension Array where Element == PackageDescription.CXXSetting {
283+
/// Settings intended to be applied to every C++ target in this package.
284+
/// Analogous to project-level build settings in an Xcode project.
285+
static var packageSettings: Self {
286+
var result = Self()
287+
288+
result += [
289+
.define("SWT_NO_EXIT_TESTS", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])),
290+
.define("SWT_NO_PROCESS_SPAWNING", .when(platforms: [.iOS, .watchOS, .tvOS, .visionOS, .wasi, .android])),
291+
.define("SWT_NO_SNAPSHOT_TYPES", .when(platforms: [.linux, .custom("freebsd"), .openbsd, .windows, .wasi, .android])),
292+
.define("SWT_NO_DYNAMIC_LINKING", .when(platforms: [.wasi])),
293+
.define("SWT_NO_PIPES", .when(platforms: [.wasi])),
294+
]
295+
296+
// Capture the testing library's version as a C++ string constant.
297+
if let git {
298+
let testingLibraryVersion = if let tag = git.currentTag {
299+
tag
300+
} else if git.hasUncommittedChanges {
301+
"\(git.currentCommit) (modified)"
302+
} else {
303+
git.currentCommit
304+
}
305+
result.append(.define("SWT_TESTING_LIBRARY_VERSION", to: #""\#(testingLibraryVersion)""#))
306+
}
307+
308+
return result
309+
}
310+
}

Diff for: ‎Sources/Testing/ABI/ABI.Record+Streaming.swift

+1-34
Original file line numberDiff line numberDiff line change
@@ -12,47 +12,14 @@
1212
private import Foundation
1313

1414
extension ABI.Version {
15-
/// Post-process encoded JSON and write it to a file.
16-
///
17-
/// - Parameters:
18-
/// - json: The JSON to write.
19-
/// - file: The file to write to.
20-
///
21-
/// - Throws: Whatever is thrown when writing to `file`.
22-
private static func _asJSONLine(_ json: UnsafeRawBufferPointer, _ eventHandler: (_ recordJSON: UnsafeRawBufferPointer) throws -> Void) rethrows {
23-
// We don't actually expect the JSON encoder to produce output containing
24-
// newline characters, so in debug builds we'll log a diagnostic message.
25-
if _slowPath(json.contains(where: \.isASCIINewline)) {
26-
#if DEBUG && !SWT_NO_FILE_IO
27-
let message = Event.ConsoleOutputRecorder.warning(
28-
"JSON encoder produced one or more newline characters while encoding an event to JSON. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new",
29-
options: .for(.stderr)
30-
)
31-
#if SWT_TARGET_OS_APPLE
32-
try? FileHandle.stderr.write(message)
33-
#else
34-
print(message)
35-
#endif
36-
#endif
37-
38-
// Remove the newline characters to conform to JSON lines specification.
39-
var json = Array(json)
40-
json.removeAll(where: \.isASCIINewline)
41-
try json.withUnsafeBytes(eventHandler)
42-
} else {
43-
// No newlines found, no need to copy the buffer.
44-
try eventHandler(json)
45-
}
46-
}
47-
4815
static func eventHandler(
4916
encodeAsJSONLines: Bool,
5017
forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void
5118
) -> Event.Handler {
5219
// Encode as JSON Lines if requested.
5320
var eventHandlerCopy = eventHandler
5421
if encodeAsJSONLines {
55-
eventHandlerCopy = { @Sendable in _asJSONLine($0, eventHandler) }
22+
eventHandlerCopy = { @Sendable in JSON.asJSONLine($0, eventHandler) }
5623
}
5724

5825
let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder()

Diff for: ‎Sources/Testing/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ add_library(Testing
3232
Events/Recorder/Event.Symbol.swift
3333
Events/TimeValue.swift
3434
ExitTests/ExitTest.swift
35+
ExitTests/ExitTest.CapturedValue.swift
3536
ExitTests/ExitTest.Condition.swift
3637
ExitTests/ExitTest.Result.swift
3738
ExitTests/SpawnProcess.swift

0 commit comments

Comments
 (0)