From 616dd879db5205720596a9da611e17d80f4ba689 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 6 Mar 2025 14:28:13 -0600 Subject: [PATCH 1/9] Represent non-encodable test argument values in Test.Case.ID Resolves #995 Resolves rdar://119522099 --- .../CustomTestArgumentEncodable.swift | 80 ++++++--- .../Test.Case.Generator.swift | 24 ++- .../Parameterization/Test.Case.ID.swift | 65 +++++--- .../Testing/Parameterization/Test.Case.swift | 156 ++++++++++++++---- Tests/TestingTests/EventTests.swift | 2 +- .../Test.Case.GeneratorTests.swift | 31 ++++ Tests/TestingTests/Test.CaseTests.swift | 61 +++++++ 7 files changed, 337 insertions(+), 82 deletions(-) create mode 100644 Tests/TestingTests/Test.Case.GeneratorTests.swift create mode 100644 Tests/TestingTests/Test.CaseTests.swift diff --git a/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift b/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift index 58d738f11..1201badc5 100644 --- a/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift +++ b/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift @@ -43,50 +43,76 @@ public protocol CustomTestArgumentEncodable: Sendable { func encodeTestArgument(to encoder: some Encoder) throws } +/// Get the best encodable representation of a test argument value, if any. +/// +/// - Parameters: +/// - value: The value for which an encodable representation is requested. +/// +/// - Returns: The best encodable representation of `value`, if one is available, +/// otherwise `nil`. +/// +/// For a description of the heuristics used to obtain an encodable +/// representation of an argument value, see . +func encodableArgumentValue(for value: some Sendable) -> (any Encodable)? { +#if canImport(Foundation) + // Helper for opening an existential. + func customArgumentWrapper(for value: some CustomTestArgumentEncodable) -> some Encodable { + _CustomArgumentWrapper(rawValue: value) + } + + return if let customEncodable = value as? any CustomTestArgumentEncodable { + customArgumentWrapper(for: customEncodable) + } else if let rawRepresentable = value as? any RawRepresentable, let encodableRawValue = rawRepresentable.rawValue as? any Encodable { + encodableRawValue + } else if let encodable = value as? any Encodable { + encodable + } else if let identifiable = value as? any Identifiable, let encodableID = identifiable.id as? any Encodable { + encodableID + } else { + nil + } +#else + return nil +#endif +} + extension Test.Case.Argument.ID { /// Initialize this instance with an ID for the specified test argument. /// /// - Parameters: /// - value: The value of a test argument for which to get an ID. + /// - encodableValue: An encodable representation of `value`, if any, with + /// which to attempt to encode a stable representation. /// - parameter: The parameter of the test function to which this argument /// value was passed. /// - /// - Returns: `nil` if an ID cannot be formed from the specified test - /// argument value. - /// - /// - Throws: Any error encountered while attempting to encode `value`. + /// If a representation of `value` can be successfully encoded, the value of + /// this instance's `bytes` property will be the the bytes of that encoded + /// JSON representation and the value of its `isStable` property will be + /// `true`. Otherwise, the value of its `bytes` property will be the bytes of + /// a textual description of `value` and the value of `isStable` will be + /// `false` to reflect that the representation is not considered stable. /// /// This function is not part of the public interface of the testing library. /// /// ## See Also /// /// - ``CustomTestArgumentEncodable`` - init?(identifying value: some Sendable, parameter: Test.Parameter) throws { + init(identifying value: some Sendable, encodableValue: (any Encodable)?, parameter: Test.Parameter) { #if canImport(Foundation) - func customArgumentWrapper(for value: some CustomTestArgumentEncodable) -> some Encodable { - _CustomArgumentWrapper(rawValue: value) - } - - let encodableValue: (any Encodable)? = if let customEncodable = value as? any CustomTestArgumentEncodable { - customArgumentWrapper(for: customEncodable) - } else if let rawRepresentable = value as? any RawRepresentable, let encodableRawValue = rawRepresentable.rawValue as? any Encodable { - encodableRawValue - } else if let encodable = value as? any Encodable { - encodable - } else if let identifiable = value as? any Identifiable, let encodableID = identifiable.id as? any Encodable { - encodableID - } else { - nil - } - - guard let encodableValue else { - return nil + if let encodableValue { + do { + self = .init(bytes: try Self._encode(encodableValue, parameter: parameter), isStable: true) + return + } catch { + // FIXME: Capture the error and propagate to the user, not as a test + // failure but as an advisory warning. A missing argument ID will + // prevent re-running the test case, but is not a blocking issue. + } } - - self = .init(bytes: try Self._encode(encodableValue, parameter: parameter)) -#else - nil #endif + + self = .init(bytes: String(describingForTest: value).utf8, isStable: false) } #if canImport(Foundation) diff --git a/Sources/Testing/Parameterization/Test.Case.Generator.swift b/Sources/Testing/Parameterization/Test.Case.Generator.swift index d4d583e48..1adf08d95 100644 --- a/Sources/Testing/Parameterization/Test.Case.Generator.swift +++ b/Sources/Testing/Parameterization/Test.Case.Generator.swift @@ -62,7 +62,7 @@ extension Test.Case { // A beautiful hack to give us the right number of cases: iterate over a // collection containing a single Void value. self.init(sequence: CollectionOfOne(())) { _ in - Test.Case(arguments: [], body: testFunction) + Test.Case(body: testFunction) } } @@ -257,7 +257,27 @@ extension Test.Case { extension Test.Case.Generator: Sequence { func makeIterator() -> some IteratorProtocol { - _sequence.lazy.map(_mapElement).makeIterator() + sequence(state: ( + iterator: _sequence.makeIterator(), + testCaseIDs: [Test.Case.ID: Int]() + )) { state in + guard let element = state.iterator.next() else { + return nil + } + + var testCase = _mapElement(element) + + // Store the original, unmodified test case ID. We're about to modify a + // property which affects it, and we want to update state based on the + // original one. + let testCaseID = testCase.id + + // Ensure test cases with identical IDs each have a unique discriminator. + testCase.discriminator = state.testCaseIDs[testCaseID, default: 0] + state.testCaseIDs[testCaseID] = testCase.discriminator + 1 + + return testCase + } } var underestimatedCount: Int { diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index 26b57fdf8..08637b968 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -15,27 +15,37 @@ extension Test.Case { /// parameterized test function. They are not necessarily unique across two /// different ``Test`` instances. @_spi(ForToolsIntegrationOnly) - public struct ID: Sendable, Equatable, Hashable { + public struct ID: Sendable { /// The IDs of the arguments of this instance's associated ``Test/Case``, in /// the order they appear in ``Test/Case/arguments``. + public var argumentIDs: [Argument.ID] + + /// A number used to distinguish this test case from others associated with + /// the same test function whose arguments have the same ID. + /// + /// ## See Also /// - /// The value of this property is `nil` if _any_ of the associated test - /// case's arguments has a `nil` ID. - public var argumentIDs: [Argument.ID]? + /// - ``Test/Case/discriminator`` + public var discriminator: Int - public init(argumentIDs: [Argument.ID]?) { + public init(argumentIDs: [Argument.ID], discriminator: Int) { self.argumentIDs = argumentIDs + self.discriminator = discriminator + } + + /// Whether or not this test case ID is considered stable across successive + /// runs. + /// + /// The value of this property is `true` if all of the argument IDs for this + /// instance are stable, otherwise it is `false`. + public var isStable: Bool { + argumentIDs.allSatisfy(\.isStable) } } @_spi(ForToolsIntegrationOnly) public var id: ID { - let argumentIDs = arguments.compactMap(\.id) - guard argumentIDs.count == arguments.count else { - return ID(argumentIDs: nil) - } - - return ID(argumentIDs: argumentIDs) + ID(argumentIDs: arguments.map(\.id), discriminator: discriminator) } } @@ -43,22 +53,31 @@ extension Test.Case { extension Test.Case.ID: CustomStringConvertible { public var description: String { - "argumentIDs: \(String(describing: argumentIDs))" + "argumentIDs: \(argumentIDs), discriminator: \(discriminator)" } } // MARK: - Codable -extension Test.Case.ID: Codable {} +extension Test.Case.ID: Codable { + public init(from decoder: some Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) -// MARK: - Equatable + // The `argumentIDs` property was optional when this type was first + // introduced, and a `nil` value represented a non-stable test case ID. + // To maintain previous behavior, if this value is absent when decoding, + // default to a single argument ID marked as non-stable. + let argumentIDs = try container.decodeIfPresent([Test.Case.Argument.ID].self, forKey: .argumentIDs) + ?? [Test.Case.Argument.ID(bytes: [], isStable: false)] -// We cannot safely implement Equatable for Test.Case because its values are -// type-erased. It does conform to `Identifiable`, but its ID type is composed -// of the IDs of its arguments, and those IDs are not always available (for -// example, if the type of an argument is not Codable). Thus, we cannot check -// for equality of test cases based on this, because if two test cases had -// different arguments, but the type of those arguments is not Codable, they -// both will have a `nil` ID and would incorrectly be considered equal. -// -// `Test.Case.ID` is Equatable, however. + // The `discriminator` property was added after this type was first + // introduced. It can safely default to zero when absent. + let discriminator = try container.decodeIfPresent(type(of: discriminator), forKey: .discriminator) ?? 0 + + self.init(argumentIDs: argumentIDs, discriminator: discriminator) + } +} + +// MARK: - Equatable, Hashable + +extension Test.Case.ID: Hashable {} diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 80ff101da..b4e55c7c8 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -26,38 +26,56 @@ extension Test { /// The raw bytes of this instance's identifier. public var bytes: [UInt8] - public init(bytes: [UInt8]) { - self.bytes = bytes + /// Whether or not this argument ID is considered stable across + /// successive runs. + /// + /// If the value of this property is `true`, the testing library can use + /// this ID to deterministically match against the original argument + /// it represents, and a user can selectively (re-)run that argument + /// of the associated parameterized test. If it is `false`, that + /// functionality is not supported for the argument this ID represents. + public var isStable: Bool + + public init(bytes: some Sequence, isStable: Bool) { + self.bytes = Array(bytes) + self.isStable = isStable } } - /// The ID of this parameterized test argument, if any. + /// The value of this parameterized test argument. + public var value: any Sendable + + /// The ID of this parameterized test argument. /// /// The uniqueness of this value is narrow: it is considered unique only /// within the scope of the parameter of the test function this argument /// was passed to. /// - /// The value of this property is `nil` when an ID cannot be formed. This - /// may occur if the type of ``value`` does not conform to one of the - /// protocols used for encoding a stable and unique representation of the - /// value. - /// /// ## See Also /// /// - ``CustomTestArgumentEncodable`` - @_spi(ForToolsIntegrationOnly) - public var id: ID? { - // FIXME: Capture the error and propagate to the user, not as a test - // failure but as an advisory warning. A missing argument ID will - // prevent re-running the test case, but is not a blocking issue. - try? Argument.ID(identifying: value, parameter: parameter) - } - - /// The value of this parameterized test argument. - public var value: any Sendable + public var id: ID /// The parameter of the test function to which this argument was passed. public var parameter: Parameter + + /// Initialize an instance of this type representing the specified + /// argument value. + /// + /// - Parameters: + /// - value: The value of this parameterized test argument. + /// - encodableValue: An encodable representation of `value`, if one is + /// available. When non-`nil`, this is used to attempt to form a + /// stable identifier. + /// - parameter: The parameter of the test function to which this + /// argument was passed. + /// + /// This forms an ``ID`` identifying `value` using `encodableValue`. + init(value: any Sendable, encodableValue: (any Encodable)?, parameter: Parameter) { + self.value = value + self.id = .init(identifying: value, encodableValue: encodableValue, parameter: parameter) + self.parameter = parameter + } } /// The arguments passed to this test case. @@ -75,11 +93,36 @@ extension Test { @_spi(Experimental) @_spi(ForToolsIntegrationOnly) public var arguments: [Argument] - init( - arguments: [Argument], - body: @escaping @Sendable () async throws -> Void - ) { - self.arguments = arguments + /// A number used to distinguish this test case from others associated with + /// the same test function whose arguments have the same ID. + /// + /// As an example, imagine the same argument is passed more than once to a + /// parameterized test: + /// + /// ```swift + /// @Test(arguments: [1, 1]) + /// func example(x: Int) { ... } + /// ``` + /// + /// There will be two ``Test/Case`` instances associated with this test + /// function. Each will represent one instance of the repeated argument `1`, + /// and each will have a different value for this property. + /// + /// The value of this property for successive runs of the same test are not + /// guaranteed to be the same. The value of this property may be equal for + /// two test cases associated with the same test if the IDs of their + /// arguments are different. + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) + public var discriminator: Int = 0 + + /// Initialize a test case for a non-parameterized test function. + /// + /// - Parameters: + /// - body: The body closure of this test case. + /// + /// The resulting test case will have zero arguments. + init(body: @escaping @Sendable () async throws -> Void) { + self.arguments = [] self.body = body } @@ -95,10 +138,29 @@ extension Test { parameters: [Parameter], body: @escaping @Sendable () async throws -> Void ) { - let arguments = zip(values, parameters).map { value, parameter in - Argument(value: value, parameter: parameter) + // Attempt to obtain an encodable representation of each value in order + // to construct a stable ID. + let encodingResult = values.reduce(into: ([any Encodable](), hasFailure: false)) { result, value in + // If we couldn't get an encodable representation of one of the values, + // give up and mark the overall attempt as a failure. This allows + // skipping unnecessary encoding work later: if any individual argument + // doesn't have a stable ID, the Test.Case.ID can't be considered stable, + // so there's no point encoding the values which _are_ encodable. + guard !result.hasFailure, let encodableValue = encodableArgumentValue(for: value) else { + return result.hasFailure = true + } + result.0.append(encodableValue) + } + let encodableValues: [any Encodable]? = if !encodingResult.hasFailure { + encodingResult.0 + } else { + nil } - self.init(arguments: arguments, body: body) + + self.arguments = zip(values.enumerated(), parameters).map { value, parameter in + Argument(value: value.1, encodableValue: encodableValues?[value.0], parameter: parameter) + } + self.body = body } /// Whether or not this test case is from a parameterized test. @@ -156,10 +218,32 @@ extension Test { // MARK: - Codable extension Test.Parameter: Codable {} -extension Test.Case.Argument.ID: Codable {} +extension Test.Case.Argument.ID: Codable { + public init(from decoder: some Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + // The `isStable` property was added after this type was introduced. + // Previously, only stable argument IDs were ever encoded, so if we're + // attempting to decode one, we can safely assume it is stable. + let isStable = try container.decodeIfPresent(type(of: isStable), forKey: .isStable) ?? true + + let bytes = try container.decode(type(of: bytes), forKey: .bytes) + self.init(bytes: bytes, isStable: isStable) + } +} // MARK: - Equatable, Hashable +extension Test.Case: Hashable { + public static func ==(lhs: Test.Case, rhs: Test.Case) -> Bool { + lhs.id == rhs.id + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(id) + } +} + extension Test.Parameter: Hashable {} extension Test.Case.Argument.ID: Hashable {} @@ -197,8 +281,8 @@ extension Test.Case.Argument { /// A serializable snapshot of a ``Test/Case/Argument`` instance. @_spi(ForToolsIntegrationOnly) public struct Snapshot: Sendable, Codable { - /// The ID of this parameterized test argument, if any. - public var id: Test.Case.Argument.ID? + /// The ID of this parameterized test argument. + public var id: Test.Case.Argument.ID /// A representation of this parameterized test argument's /// ``Test/Case/Argument/value`` property. @@ -217,6 +301,20 @@ extension Test.Case.Argument { value = Expression.Value(reflecting: argument.value) ?? .init(describing: argument.value) parameter = argument.parameter } + + public init(from decoder: some Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + // The `id` property was optional when this type was first introduced, + // and a `nil` value represented an argument whose ID was non-stable. + // To maintain previous behavior, if this value is absent when decoding, + // default to an argument ID marked as non-stable. + id = try container.decodeIfPresent(Test.Case.Argument.ID.self, forKey: .id) + ?? ID(bytes: [], isStable: false) + + value = try container.decode(type(of: value), forKey: .value) + parameter = try container.decode(type(of: parameter), forKey: .parameter) + } } } #endif diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index 941dcadb9..d6dd48971 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -57,7 +57,7 @@ struct EventTests { let testID = Test.ID(moduleName: "ModuleName", nameComponents: ["NameComponent1", "NameComponent2"], sourceLocation: #_sourceLocation) - let testCaseID = Test.Case.ID(argumentIDs: nil) + let testCaseID = Test.Case.ID(argumentIDs: [], discriminator: 0) let event = Event(kind, testID: testID, testCaseID: testCaseID, instant: .now) let eventSnapshot = Event.Snapshot(snapshotting: event) let decoded = try JSON.encodeAndDecode(eventSnapshot) diff --git a/Tests/TestingTests/Test.Case.GeneratorTests.swift b/Tests/TestingTests/Test.Case.GeneratorTests.swift new file mode 100644 index 000000000..0c2afd0ef --- /dev/null +++ b/Tests/TestingTests/Test.Case.GeneratorTests.swift @@ -0,0 +1,31 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Test.Case.Generator Tests") +struct Test_Case_GeneratorTests { + @Test func uniqueDiscriminators() throws { + let generator = Test.Case.Generator( + arguments: [1, 1, 1], + parameters: [Test.Parameter(index: 0, firstName: "x", type: Int.self)], + testFunction: { _ in } + ) + + let testCases = Array(generator) + #expect(testCases.count == 3) + + let firstCase = try #require(testCases.first) + #expect(firstCase.id.discriminator == 0) + + let discriminators = Set(testCases.map(\.id.discriminator)) + #expect(discriminators.count == 3) + } +} diff --git a/Tests/TestingTests/Test.CaseTests.swift b/Tests/TestingTests/Test.CaseTests.swift new file mode 100644 index 000000000..c757ab808 --- /dev/null +++ b/Tests/TestingTests/Test.CaseTests.swift @@ -0,0 +1,61 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Test.Case Tests") +struct Test_CaseTests { + @Test func singleStableArgument() throws { + let testCase = Test.Case( + values: [1], + parameters: [Test.Parameter(index: 0, firstName: "x", type: Int.self)], + body: {} + ) + #expect(testCase.id.isStable) + #expect(testCase.arguments.allSatisfy { $0.id.isStable }) + } + + @Test func twoStableArguments() throws { + let testCase = Test.Case( + values: [1, "a"], + parameters: [ + Test.Parameter(index: 0, firstName: "x", type: Int.self), + Test.Parameter(index: 1, firstName: "y", type: String.self), + ], + body: {} + ) + #expect(testCase.id.isStable) + #expect(testCase.arguments.allSatisfy { $0.id.isStable }) + } + + @Test("Two arguments: one non-stable, followed by one stable") + func nonStableAndStableArgument() throws { + let testCase = Test.Case( + values: [NonCodable(), IssueRecordingEncodable()], + parameters: [ + Test.Parameter(index: 0, firstName: "x", type: NonCodable.self), + Test.Parameter(index: 1, firstName: "y", type: IssueRecordingEncodable.self), + ], + body: {} + ) + #expect(!testCase.id.isStable) + #expect(testCase.arguments.allSatisfy { !$0.id.isStable }) + } +} + +// MARK: - Fixtures, helpers + +private struct NonCodable {} + +private struct IssueRecordingEncodable: Encodable { + func encode(to encoder: any Encoder) throws { + Issue.record("Unexpected attempt to encode an instance of \(Self.self)") + } +} From ae2d351300f4851e6e70d70aed54a89d0d1c6fe1 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Mon, 10 Mar 2025 17:29:06 -0500 Subject: [PATCH 2/9] Improve the modeling of test cases (and their IDs) for non-parameterized test functions by making arguments and discriminator optional --- .../Testing/ABI/Encoded/ABI.EncodedTest.swift | 6 +- .../Event.HumanReadableOutputRecorder.swift | 10 ++- .../Test.Case.Generator.swift | 17 ++-- .../Parameterization/Test.Case.ID.swift | 77 +++++++++++++---- .../Testing/Parameterization/Test.Case.swift | 85 ++++++++++++++++--- Tests/TestingTests/EventTests.swift | 2 +- .../Test.Case.Argument.IDTests.swift | 32 +++---- .../Test.Case.ArgumentTests.swift | 40 ++++----- Tests/TestingTests/Test.CaseTests.swift | 9 +- .../TestingTests/TestCaseSelectionTests.swift | 5 +- 10 files changed, 202 insertions(+), 81 deletions(-) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index 51d01781d..fa0eb17f9 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -124,9 +124,13 @@ extension ABI { var displayName: String init(encoding testCase: borrowing Test.Case) { + guard let arguments = testCase.arguments else { + preconditionFailure("Attempted to initialize an EncodedTestCase encoding a test case which is not parameterized: \(testCase)") + } + // TODO: define an encodable form of Test.Case.ID id = String(describing: testCase.id) - displayName = testCase.arguments.lazy + displayName = arguments.lazy .map(\.value) .map(String.init(describingForTest:)) .joined(separator: ", ") diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 0e856facf..e6f816004 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -171,8 +171,12 @@ extension Test.Case { /// - Parameters: /// - includeTypeNames: Whether the qualified type name of each argument's /// runtime type should be included. Defaults to `false`. + /// + /// - Returns: A string containing the arguments of this test case formatted + /// for presentation, or an empty string if this test cases is + /// non-parameterized. fileprivate func labeledArguments(includingQualifiedTypeNames includeTypeNames: Bool = false) -> String { - arguments.lazy + (arguments ?? []).lazy .map { argument in let valueDescription = String(describingForTest: argument.value) @@ -494,14 +498,14 @@ extension Event.HumanReadableOutputRecorder { return result case .testCaseStarted: - guard let testCase = eventContext.testCase, testCase.isParameterized else { + guard let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else { break } return [ Message( symbol: .default, - stringValue: "Passing \(testCase.arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)" + stringValue: "Passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName)" ) ] diff --git a/Sources/Testing/Parameterization/Test.Case.Generator.swift b/Sources/Testing/Parameterization/Test.Case.Generator.swift index 1adf08d95..f5a1125a8 100644 --- a/Sources/Testing/Parameterization/Test.Case.Generator.swift +++ b/Sources/Testing/Parameterization/Test.Case.Generator.swift @@ -267,14 +267,17 @@ extension Test.Case.Generator: Sequence { var testCase = _mapElement(element) - // Store the original, unmodified test case ID. We're about to modify a - // property which affects it, and we want to update state based on the - // original one. - let testCaseID = testCase.id + if testCase.isParameterized { + // Store the original, unmodified test case ID. We're about to modify a + // property which affects it, and we want to update state based on the + // original one. + let testCaseID = testCase.id - // Ensure test cases with identical IDs each have a unique discriminator. - testCase.discriminator = state.testCaseIDs[testCaseID, default: 0] - state.testCaseIDs[testCaseID] = testCase.discriminator + 1 + // Ensure test cases with identical IDs each have a unique discriminator. + let discriminator = state.testCaseIDs[testCaseID, default: 0] + testCase.discriminator = discriminator + state.testCaseIDs[testCaseID] = discriminator + 1 + } return testCase } diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index 08637b968..48756a5b7 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -18,17 +18,25 @@ extension Test.Case { public struct ID: Sendable { /// The IDs of the arguments of this instance's associated ``Test/Case``, in /// the order they appear in ``Test/Case/arguments``. - public var argumentIDs: [Argument.ID] + /// + /// The value of this property is `nil` for the ID of the single test case + /// associated with a non-parameterized test function. + public var argumentIDs: [Argument.ID]? /// A number used to distinguish this test case from others associated with - /// the same test function whose arguments have the same ID. + /// the same parameterized test function whose arguments have the same ID. + /// + /// The value of this property is `nil` for the ID of the single test case + /// associated with a non-parameterized test function. /// /// ## See Also /// /// - ``Test/Case/discriminator`` - public var discriminator: Int + public var discriminator: Int? + + init(argumentIDs: [Argument.ID]?, discriminator: Int?) { + precondition((argumentIDs == nil) == (discriminator == nil)) - public init(argumentIDs: [Argument.ID], discriminator: Int) { self.argumentIDs = argumentIDs self.discriminator = discriminator } @@ -39,13 +47,13 @@ extension Test.Case { /// The value of this property is `true` if all of the argument IDs for this /// instance are stable, otherwise it is `false`. public var isStable: Bool { - argumentIDs.allSatisfy(\.isStable) + (argumentIDs ?? []).allSatisfy(\.isStable) } } @_spi(ForToolsIntegrationOnly) public var id: ID { - ID(argumentIDs: arguments.map(\.id), discriminator: discriminator) + ID(argumentIDs: arguments.map { $0.map(\.id) }, discriminator: discriminator) } } @@ -53,7 +61,11 @@ extension Test.Case { extension Test.Case.ID: CustomStringConvertible { public var description: String { - "argumentIDs: \(argumentIDs), discriminator: \(discriminator)" + if let argumentIDs, let discriminator { + "argumentIDs: \(argumentIDs), discriminator: \(discriminator)" + } else { + "non-parameterized" + } } } @@ -63,19 +75,54 @@ extension Test.Case.ID: Codable { public init(from decoder: some Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) - // The `argumentIDs` property was optional when this type was first - // introduced, and a `nil` value represented a non-stable test case ID. - // To maintain previous behavior, if this value is absent when decoding, - // default to a single argument ID marked as non-stable. - let argumentIDs = try container.decodeIfPresent([Test.Case.Argument.ID].self, forKey: .argumentIDs) - ?? [Test.Case.Argument.ID(bytes: [], isStable: false)] + // The `argumentIDs` property is Optional but the meaning of `nil` has + // changed since this type was first introduced: it now identifies a + // non-parameterized test case, whereas it originally identified a + // parameterized test case for which one or more arguments could not be + // encoded. If it's present in the decoding container, accept whatever value + // is decoded (which may be `nil`). If it's absent, default to a single + // argument ID marked as non-stable to maintain previous behavior. + let argumentIDs: [Test.Case.Argument.ID]? = if container.contains(.argumentIDs) { + try container.decode(type(of: argumentIDs), forKey: .argumentIDs) + } else { + [Test.Case.Argument.ID(bytes: [], isStable: false)] + } // The `discriminator` property was added after this type was first - // introduced. It can safely default to zero when absent. - let discriminator = try container.decodeIfPresent(type(of: discriminator), forKey: .discriminator) ?? 0 + // introduced. If it's present in the decoding container, accept whatever + // value is decoded (which may be `nil`). If it's absent, default to `nil` + // if `argumentIDs` was interpreted as a non-parameterized test above, or + // else 0, to maintain previous behavior. + let discriminator: Int? = if container.contains(.discriminator) { + try container.decode(type(of: discriminator), forKey: .discriminator) + } else { + if let argumentIDs { + argumentIDs.isEmpty ? nil : 0 + } else { + nil + } + } self.init(argumentIDs: argumentIDs, discriminator: discriminator) } + + public func encode(to encoder: some Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + + // The `argumentIDs` property is Optional but the meaning of `nil` has + // changed since this type was first introduced: it now identifies a + // non-parameterized test case, whereas it originally identified a + // parameterized test case for which one or more arguments could not be + // encoded. Explicitly encode `nil` values here, rather than omitting them, + // so that when decoding we can distinguish these two scenarios. + try container.encode(argumentIDs, forKey: .argumentIDs) + + // The `discriminator` property was added after this type was first + // introduced. Explicitly encode `nil` values here, rather than omitting + // them, so that when decoding we can distinguish the older vs. newer + // implementations. + try container.encode(discriminator, forKey: .discriminator) + } } // MARK: - Equatable, Hashable diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index b4e55c7c8..c4f4b33a9 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -15,6 +15,29 @@ extension Test { /// Tests that are _not_ parameterized map to a single instance of /// ``Test/Case``. public struct Case: Sendable { + /// An enumeration describing the various kinds of test cases. + private enum _Kind: Sendable { + /// A test case associated with a non-parameterized test function. + /// + /// There is only one test case with this kind associated with each + /// non-parameterized test function. + case nonParameterized + + /// A test case associated with a parameterized test function, including + /// the argument(s) it was passed and a discriminator. + /// + /// - Parameters: + /// - arguments: The arguments passed to the parameterized test function + /// this test case is associated with. + /// - discriminator: A number used to distinguish this test case from + /// others associated with the same parameterized test function whose + /// arguments have the same ID. + case parameterized(arguments: [Argument], discriminator: Int) + } + + /// The kind of this test case. + private var _kind: _Kind + /// A type representing an argument passed to a parameter of a parameterized /// test function. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) @@ -78,7 +101,7 @@ extension Test { } } - /// The arguments passed to this test case. + /// The arguments passed to this test case, if any. /// /// If the argument was a tuple but its elements were passed to distinct /// parameters of the test function, each element of the tuple will be @@ -88,13 +111,19 @@ extension Test { /// represented as one ``Argument`` instance. /// /// Non-parameterized test functions will have a single test case instance, - /// and the value of this property will be an empty array for such test - /// cases. + /// and the value of this property will be `nil` for such test cases. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) - public var arguments: [Argument] + public var arguments: [Argument]? { + switch _kind { + case .nonParameterized: + nil + case let .parameterized(arguments, _): + arguments + } + } /// A number used to distinguish this test case from others associated with - /// the same test function whose arguments have the same ID. + /// the same parameterized test function whose arguments have the same ID. /// /// As an example, imagine the same argument is passed more than once to a /// parameterized test: @@ -111,9 +140,35 @@ extension Test { /// The value of this property for successive runs of the same test are not /// guaranteed to be the same. The value of this property may be equal for /// two test cases associated with the same test if the IDs of their - /// arguments are different. + /// arguments are different. The value of this property is `nil` for the + /// single test case associated with a non-parameterized test function. @_spi(Experimental) @_spi(ForToolsIntegrationOnly) - public var discriminator: Int = 0 + public internal(set) var discriminator: Int? { + get { + switch _kind { + case .nonParameterized: + nil + case let .parameterized(_, discriminator): + discriminator + } + } + set { + switch _kind { + case .nonParameterized: + precondition(newValue == nil, "A non-nil discriminator may only be set for a test case which is parameterized.") + case let .parameterized(arguments, _): + guard let newValue else { + preconditionFailure("A nil discriminator may only be set for a test case which is not parameterized.") + } + _kind = .parameterized(arguments: arguments, discriminator: newValue) + } + } + } + + private init(kind: _Kind, body: @escaping @Sendable () async throws -> Void) { + self._kind = kind + self.body = body + } /// Initialize a test case for a non-parameterized test function. /// @@ -122,8 +177,7 @@ extension Test { /// /// The resulting test case will have zero arguments. init(body: @escaping @Sendable () async throws -> Void) { - self.arguments = [] - self.body = body + self.init(kind: .nonParameterized, body: body) } /// Initialize a test case by pairing values with their corresponding @@ -157,15 +211,20 @@ extension Test { nil } - self.arguments = zip(values.enumerated(), parameters).map { value, parameter in + let arguments = zip(values.enumerated(), parameters).map { value, parameter in Argument(value: value.1, encodableValue: encodableValues?[value.0], parameter: parameter) } - self.body = body + self.init(kind: .parameterized(arguments: arguments, discriminator: 0), body: body) } /// Whether or not this test case is from a parameterized test. public var isParameterized: Bool { - !arguments.isEmpty + switch _kind { + case .nonParameterized: + false + case .parameterized: + true + } } /// The body closure of this test case. @@ -272,7 +331,7 @@ extension Test.Case { /// - testCase: The original test case to snapshot. public init(snapshotting testCase: borrowing Test.Case) { id = testCase.id - arguments = testCase.arguments.map(Test.Case.Argument.Snapshot.init) + arguments = (testCase.arguments ?? []).map(Test.Case.Argument.Snapshot.init) } } } diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index d6dd48971..317198f65 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -57,7 +57,7 @@ struct EventTests { let testID = Test.ID(moduleName: "ModuleName", nameComponents: ["NameComponent1", "NameComponent2"], sourceLocation: #_sourceLocation) - let testCaseID = Test.Case.ID(argumentIDs: [], discriminator: 0) + let testCaseID = Test.Case.ID(argumentIDs: nil, discriminator: nil) let event = Event(kind, testID: testID, testCaseID: testCaseID, instant: .now) let eventSnapshot = Event.Snapshot(snapshotting: event) let decoded = try JSON.encodeAndDecode(eventSnapshot) diff --git a/Tests/TestingTests/Test.Case.Argument.IDTests.swift b/Tests/TestingTests/Test.Case.Argument.IDTests.swift index ced76adac..052213912 100644 --- a/Tests/TestingTests/Test.Case.Argument.IDTests.swift +++ b/Tests/TestingTests/Test.Case.Argument.IDTests.swift @@ -20,10 +20,10 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) - #expect(String(decoding: argumentID.bytes, as: UTF8.self) == "123") + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) + #expect(String(decoding: argument.id.bytes, as: UTF8.self) == "123") } @Test("One CustomTestArgumentEncodable parameter") @@ -34,11 +34,11 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) #if canImport(Foundation) - let decodedArgument = try argumentID.bytes.withUnsafeBufferPointer { argumentID in + let decodedArgument = try argument.id.bytes.withUnsafeBufferPointer { argumentID in try JSON.decode(MyCustomTestArgument.self, from: .init(argumentID)) } #expect(decodedArgument == MyCustomTestArgument(x: 123, y: "abc")) @@ -53,10 +53,10 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) - #expect(String(decoding: argumentID.bytes, as: UTF8.self) == #""abc""#) + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) + #expect(String(decoding: argument.id.bytes, as: UTF8.self) == #""abc""#) } @Test("One RawRepresentable parameter") @@ -67,10 +67,10 @@ struct Test_Case_Argument_IDTests { ) { _ in } let testCases = try #require(test.testCases) let testCase = try #require(testCases.first { _ in true }) - #expect(testCase.arguments.count == 1) - let argument = try #require(testCase.arguments.first) - let argumentID = try #require(argument.id) - #expect(String(decoding: argumentID.bytes, as: UTF8.self) == #""abc""#) + let arguments = try #require(testCase.arguments) + #expect(arguments.count == 1) + let argument = try #require(arguments.first) + #expect(String(decoding: argument.id.bytes, as: UTF8.self) == #""abc""#) } } diff --git a/Tests/TestingTests/Test.Case.ArgumentTests.swift b/Tests/TestingTests/Test.Case.ArgumentTests.swift index a5c5e7462..4ea9925d6 100644 --- a/Tests/TestingTests/Test.Case.ArgumentTests.swift +++ b/Tests/TestingTests/Test.Case.ArgumentTests.swift @@ -19,10 +19,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? String == "value") #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") @@ -38,17 +38,17 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 2) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 2) do { - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? String == "value") #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") } do { - let argument = testCase.arguments[1] + let argument = arguments[1] #expect(argument.value as? Int == 123) #expect(argument.parameter.index == 1) #expect(argument.parameter.firstName == "y") @@ -65,10 +65,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? (String) == ("value")) #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") @@ -84,10 +84,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] let value = try #require(argument.value as? (String, Int)) #expect(value.0 == "value") #expect(value.1 == 123) @@ -105,17 +105,17 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 2) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 2) do { - let argument = testCase.arguments[0] + let argument = arguments[0] #expect(argument.value as? String == "value") #expect(argument.parameter.index == 0) #expect(argument.parameter.firstName == "x") } do { - let argument = testCase.arguments[1] + let argument = arguments[1] #expect(argument.value as? Int == 123) #expect(argument.parameter.index == 1) #expect(argument.parameter.firstName == "y") @@ -132,10 +132,10 @@ struct Test_Case_ArgumentTests { guard case .testCaseStarted = event.kind else { return } - let testCase = try #require(context.testCase) - try #require(testCase.arguments.count == 1) + let arguments = try #require(context.testCase?.arguments) + try #require(arguments.count == 1) - let argument = testCase.arguments[0] + let argument = arguments[0] let value = try #require(argument.value as? (String, Int)) #expect(value.0 == "value") #expect(value.1 == 123) diff --git a/Tests/TestingTests/Test.CaseTests.swift b/Tests/TestingTests/Test.CaseTests.swift index c757ab808..2530d03fb 100644 --- a/Tests/TestingTests/Test.CaseTests.swift +++ b/Tests/TestingTests/Test.CaseTests.swift @@ -19,7 +19,8 @@ struct Test_CaseTests { body: {} ) #expect(testCase.id.isStable) - #expect(testCase.arguments.allSatisfy { $0.id.isStable }) + let arguments = try #require(testCase.arguments) + #expect(arguments.allSatisfy { $0.id.isStable }) } @Test func twoStableArguments() throws { @@ -32,7 +33,8 @@ struct Test_CaseTests { body: {} ) #expect(testCase.id.isStable) - #expect(testCase.arguments.allSatisfy { $0.id.isStable }) + let arguments = try #require(testCase.arguments) + #expect(arguments.allSatisfy { $0.id.isStable }) } @Test("Two arguments: one non-stable, followed by one stable") @@ -46,7 +48,8 @@ struct Test_CaseTests { body: {} ) #expect(!testCase.id.isStable) - #expect(testCase.arguments.allSatisfy { !$0.id.isStable }) + let arguments = try #require(testCase.arguments) + #expect(arguments.allSatisfy { !$0.id.isStable }) } } diff --git a/Tests/TestingTests/TestCaseSelectionTests.swift b/Tests/TestingTests/TestCaseSelectionTests.swift index de8b10c66..d24ddf8f6 100644 --- a/Tests/TestingTests/TestCaseSelectionTests.swift +++ b/Tests/TestingTests/TestCaseSelectionTests.swift @@ -85,8 +85,9 @@ struct TestCaseSelectionTests { } let selectedTestCase = try #require(fixtureTest.testCases?.first { testCase in - guard let firstArg = testCase.arguments.first?.value as? String, - let secondArg = testCase.arguments.last?.value as? Int + guard let arguments = testCase.arguments, + let firstArg = arguments.first?.value as? String, + let secondArg = arguments.last?.value as? Int else { return false } From 0170819e40301ef16d5bb5c4177f807876577623 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 11 Mar 2025 15:46:36 -0500 Subject: [PATCH 3/9] Revert changes to Test.Case.Argument.Snapshot which aren't necessary --- .../Testing/Parameterization/Test.Case.swift | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index c4f4b33a9..9da55799f 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -340,8 +340,8 @@ extension Test.Case.Argument { /// A serializable snapshot of a ``Test/Case/Argument`` instance. @_spi(ForToolsIntegrationOnly) public struct Snapshot: Sendable, Codable { - /// The ID of this parameterized test argument. - public var id: Test.Case.Argument.ID + /// The ID of this parameterized test argument, if any. + public var id: Test.Case.Argument.ID? /// A representation of this parameterized test argument's /// ``Test/Case/Argument/value`` property. @@ -360,20 +360,6 @@ extension Test.Case.Argument { value = Expression.Value(reflecting: argument.value) ?? .init(describing: argument.value) parameter = argument.parameter } - - public init(from decoder: some Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - - // The `id` property was optional when this type was first introduced, - // and a `nil` value represented an argument whose ID was non-stable. - // To maintain previous behavior, if this value is absent when decoding, - // default to an argument ID marked as non-stable. - id = try container.decodeIfPresent(Test.Case.Argument.ID.self, forKey: .id) - ?? ID(bytes: [], isStable: false) - - value = try container.decode(type(of: value), forKey: .value) - parameter = try container.decode(type(of: parameter), forKey: .parameter) - } } } #endif From 9c5d98926c95e44358a261470be0d0380f607bad Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 11 Mar 2025 16:05:13 -0500 Subject: [PATCH 4/9] Incorporate various review feedback --- Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift | 2 +- .../Recorder/Event.HumanReadableOutputRecorder.swift | 4 +++- .../Testing/Parameterization/Test.Case.Generator.swift | 8 +++++--- Sources/Testing/Parameterization/Test.Case.ID.swift | 2 +- Sources/Testing/Parameterization/Test.Case.swift | 8 ++++++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift index fa0eb17f9..cda558f83 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift @@ -125,7 +125,7 @@ extension ABI { init(encoding testCase: borrowing Test.Case) { guard let arguments = testCase.arguments else { - preconditionFailure("Attempted to initialize an EncodedTestCase encoding a test case which is not parameterized: \(testCase)") + preconditionFailure("Attempted to initialize an EncodedTestCase encoding a test case which is not parameterized: \(testCase). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } // TODO: define an encodable form of Test.Case.ID diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index e6f816004..f585495a9 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -176,7 +176,9 @@ extension Test.Case { /// for presentation, or an empty string if this test cases is /// non-parameterized. fileprivate func labeledArguments(includingQualifiedTypeNames includeTypeNames: Bool = false) -> String { - (arguments ?? []).lazy + guard let arguments else { return "" } + + return arguments.lazy .map { argument in let valueDescription = String(describingForTest: argument.value) diff --git a/Sources/Testing/Parameterization/Test.Case.Generator.swift b/Sources/Testing/Parameterization/Test.Case.Generator.swift index f5a1125a8..d30e3a7d3 100644 --- a/Sources/Testing/Parameterization/Test.Case.Generator.swift +++ b/Sources/Testing/Parameterization/Test.Case.Generator.swift @@ -257,10 +257,12 @@ extension Test.Case { extension Test.Case.Generator: Sequence { func makeIterator() -> some IteratorProtocol { - sequence(state: ( + let state = ( iterator: _sequence.makeIterator(), - testCaseIDs: [Test.Case.ID: Int]() - )) { state in + testCaseIDs: [Test.Case.ID: Int](minimumCapacity: underestimatedCount) + ) + + return sequence(state: state) { state in guard let element = state.iterator.next() else { return nil } diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index 48756a5b7..09dd8f207 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -127,4 +127,4 @@ extension Test.Case.ID: Codable { // MARK: - Equatable, Hashable -extension Test.Case.ID: Hashable {} +extension Test.Case.ID: Equatable, Hashable {} diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 9da55799f..39732bb8f 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -293,7 +293,7 @@ extension Test.Case.Argument.ID: Codable { // MARK: - Equatable, Hashable -extension Test.Case: Hashable { +extension Test.Case: Equatable, Hashable { public static func ==(lhs: Test.Case, rhs: Test.Case) -> Bool { lhs.id == rhs.id } @@ -331,7 +331,11 @@ extension Test.Case { /// - testCase: The original test case to snapshot. public init(snapshotting testCase: borrowing Test.Case) { id = testCase.id - arguments = (testCase.arguments ?? []).map(Test.Case.Argument.Snapshot.init) + arguments = if let arguments = testCase.arguments { + arguments.map(Test.Case.Argument.Snapshot.init) + } else { + [] + } } } } From eb6f4ec7da657036a851551793db05f4406f50ef Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Mar 2025 11:42:46 -0500 Subject: [PATCH 5/9] Remove unnecessary conformances for Test.Case --- Sources/Testing/Parameterization/Test.Case.swift | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 39732bb8f..f95c91406 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -293,16 +293,6 @@ extension Test.Case.Argument.ID: Codable { // MARK: - Equatable, Hashable -extension Test.Case: Equatable, Hashable { - public static func ==(lhs: Test.Case, rhs: Test.Case) -> Bool { - lhs.id == rhs.id - } - - public func hash(into hasher: inout Hasher) { - hasher.combine(id) - } -} - extension Test.Parameter: Hashable {} extension Test.Case.Argument.ID: Hashable {} From d12399010574ca4586d5d947c7f642865fb2013e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Mar 2025 15:15:27 -0500 Subject: [PATCH 6/9] Use a for loop instead of reduce --- .../Testing/Parameterization/Test.Case.swift | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index f95c91406..409f57262 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -194,25 +194,24 @@ extension Test { ) { // Attempt to obtain an encodable representation of each value in order // to construct a stable ID. - let encodingResult = values.reduce(into: ([any Encodable](), hasFailure: false)) { result, value in + var encodableValues = [any Encodable]() + var hadFailure = false + for value in values { // If we couldn't get an encodable representation of one of the values, // give up and mark the overall attempt as a failure. This allows // skipping unnecessary encoding work later: if any individual argument // doesn't have a stable ID, the Test.Case.ID can't be considered stable, // so there's no point encoding the values which _are_ encodable. - guard !result.hasFailure, let encodableValue = encodableArgumentValue(for: value) else { - return result.hasFailure = true + guard let encodableValue = encodableArgumentValue(for: value) else { + hadFailure = true + break } - result.0.append(encodableValue) - } - let encodableValues: [any Encodable]? = if !encodingResult.hasFailure { - encodingResult.0 - } else { - nil + encodableValues.append(encodableValue) } let arguments = zip(values.enumerated(), parameters).map { value, parameter in - Argument(value: value.1, encodableValue: encodableValues?[value.0], parameter: parameter) + let encodableValue = hadFailure ? nil : encodableValues[value.0] + return Argument(value: value.1, encodableValue: encodableValue, parameter: parameter) } self.init(kind: .parameterized(arguments: arguments, discriminator: 0), body: body) } From 79caae8a52f8168e55bc3a12a286c89313c1c196 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 12 Mar 2025 16:53:48 -0500 Subject: [PATCH 7/9] Switch from some to any types for Codable implementations --- Sources/Testing/Parameterization/Test.Case.ID.swift | 2 +- Sources/Testing/Parameterization/Test.Case.swift | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index 09dd8f207..d6d6602bb 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -72,7 +72,7 @@ extension Test.Case.ID: CustomStringConvertible { // MARK: - Codable extension Test.Case.ID: Codable { - public init(from decoder: some Decoder) throws { + public init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) // The `argumentIDs` property is Optional but the meaning of `nil` has diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 409f57262..75a3be09f 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -276,8 +276,9 @@ extension Test { // MARK: - Codable extension Test.Parameter: Codable {} + extension Test.Case.Argument.ID: Codable { - public init(from decoder: some Decoder) throws { + public init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) // The `isStable` property was added after this type was introduced. From d710f467a44c2d31433dc683002e859232d09dd9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 13 Mar 2025 22:50:32 -0500 Subject: [PATCH 8/9] Simplify everything by storing `isStable` on Test.Case.ID instead of on each Test.Case.Argument, and reducing manual Codable implementations from 3 down to 1. Add more exhaustive serialization tests --- .../CustomTestArgumentEncodable.swift | 89 +++++-------- .../Parameterization/Test.Case.ID.swift | 96 ++++++-------- .../Testing/Parameterization/Test.Case.swift | 121 ++++++++---------- Tests/TestingTests/EventTests.swift | 2 +- Tests/TestingTests/Test.CaseTests.swift | 75 ++++++++++- .../TestSupport/TestingAdditions.swift | 23 ++++ 6 files changed, 221 insertions(+), 185 deletions(-) diff --git a/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift b/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift index 1201badc5..90b3ff292 100644 --- a/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift +++ b/Sources/Testing/Parameterization/CustomTestArgumentEncodable.swift @@ -43,76 +43,57 @@ public protocol CustomTestArgumentEncodable: Sendable { func encodeTestArgument(to encoder: some Encoder) throws } -/// Get the best encodable representation of a test argument value, if any. -/// -/// - Parameters: -/// - value: The value for which an encodable representation is requested. -/// -/// - Returns: The best encodable representation of `value`, if one is available, -/// otherwise `nil`. -/// -/// For a description of the heuristics used to obtain an encodable -/// representation of an argument value, see . -func encodableArgumentValue(for value: some Sendable) -> (any Encodable)? { -#if canImport(Foundation) - // Helper for opening an existential. - func customArgumentWrapper(for value: some CustomTestArgumentEncodable) -> some Encodable { - _CustomArgumentWrapper(rawValue: value) - } - - return if let customEncodable = value as? any CustomTestArgumentEncodable { - customArgumentWrapper(for: customEncodable) - } else if let rawRepresentable = value as? any RawRepresentable, let encodableRawValue = rawRepresentable.rawValue as? any Encodable { - encodableRawValue - } else if let encodable = value as? any Encodable { - encodable - } else if let identifiable = value as? any Identifiable, let encodableID = identifiable.id as? any Encodable { - encodableID - } else { - nil - } -#else - return nil -#endif -} - extension Test.Case.Argument.ID { - /// Initialize this instance with an ID for the specified test argument. + /// Initialize an ID instance with the specified test argument value. /// /// - Parameters: /// - value: The value of a test argument for which to get an ID. - /// - encodableValue: An encodable representation of `value`, if any, with - /// which to attempt to encode a stable representation. /// - parameter: The parameter of the test function to which this argument /// value was passed. /// - /// If a representation of `value` can be successfully encoded, the value of - /// this instance's `bytes` property will be the the bytes of that encoded - /// JSON representation and the value of its `isStable` property will be - /// `true`. Otherwise, the value of its `bytes` property will be the bytes of - /// a textual description of `value` and the value of `isStable` will be - /// `false` to reflect that the representation is not considered stable. + /// - Returns: `nil` if a stable ID cannot be formed from the specified test + /// argument value. + /// + /// - Throws: Any error encountered while attempting to encode `value`. + /// + /// If a stable representation of `value` can be encoded successfully, the + /// value of this instance's `bytes` property will be the the bytes of that + /// encoded JSON representation and this instance may be considered stable. If + /// no stable representation of `value` can be obtained, `nil` is returned. If + /// a stable representation was obtained but failed to encode, the error + /// resulting from the encoding attempt is thrown. /// /// This function is not part of the public interface of the testing library. /// /// ## See Also /// /// - ``CustomTestArgumentEncodable`` - init(identifying value: some Sendable, encodableValue: (any Encodable)?, parameter: Test.Parameter) { + init?(identifying value: some Sendable, parameter: Test.Parameter) throws { #if canImport(Foundation) - if let encodableValue { - do { - self = .init(bytes: try Self._encode(encodableValue, parameter: parameter), isStable: true) - return - } catch { - // FIXME: Capture the error and propagate to the user, not as a test - // failure but as an advisory warning. A missing argument ID will - // prevent re-running the test case, but is not a blocking issue. - } + func customArgumentWrapper(for value: some CustomTestArgumentEncodable) -> some Encodable { + _CustomArgumentWrapper(rawValue: value) + } + + let encodableValue: (any Encodable)? = if let customEncodable = value as? any CustomTestArgumentEncodable { + customArgumentWrapper(for: customEncodable) + } else if let rawRepresentable = value as? any RawRepresentable, let encodableRawValue = rawRepresentable.rawValue as? any Encodable { + encodableRawValue + } else if let encodable = value as? any Encodable { + encodable + } else if let identifiable = value as? any Identifiable, let encodableID = identifiable.id as? any Encodable { + encodableID + } else { + nil } -#endif - self = .init(bytes: String(describingForTest: value).utf8, isStable: false) + guard let encodableValue else { + return nil + } + + self.init(bytes: try Self._encode(encodableValue, parameter: parameter)) +#else + nil +#endif } #if canImport(Foundation) diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index d6d6602bb..1d8e6e5b5 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -34,26 +34,22 @@ extension Test.Case { /// - ``Test/Case/discriminator`` public var discriminator: Int? - init(argumentIDs: [Argument.ID]?, discriminator: Int?) { + /// Whether or not this test case ID is considered stable across successive + /// runs. + public var isStable: Bool + + init(argumentIDs: [Argument.ID]?, discriminator: Int?, isStable: Bool) { precondition((argumentIDs == nil) == (discriminator == nil)) self.argumentIDs = argumentIDs self.discriminator = discriminator - } - - /// Whether or not this test case ID is considered stable across successive - /// runs. - /// - /// The value of this property is `true` if all of the argument IDs for this - /// instance are stable, otherwise it is `false`. - public var isStable: Bool { - (argumentIDs ?? []).allSatisfy(\.isStable) + self.isStable = isStable } } @_spi(ForToolsIntegrationOnly) public var id: ID { - ID(argumentIDs: arguments.map { $0.map(\.id) }, discriminator: discriminator) + ID(argumentIDs: arguments.map { $0.map(\.id) }, discriminator: discriminator, isStable: isStable) } } @@ -62,9 +58,9 @@ extension Test.Case { extension Test.Case.ID: CustomStringConvertible { public var description: String { if let argumentIDs, let discriminator { - "argumentIDs: \(argumentIDs), discriminator: \(discriminator)" + "Parameterized test case ID: argumentIDs: \(argumentIDs), discriminator: \(discriminator), isStable: \(isStable)" } else { - "non-parameterized" + "Non-parameterized test case ID" } } } @@ -72,56 +68,40 @@ extension Test.Case.ID: CustomStringConvertible { // MARK: - Codable extension Test.Case.ID: Codable { + private enum CodingKeys: CodingKey { + case argumentIDs + case discriminator + case isStable + } + public init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) - // The `argumentIDs` property is Optional but the meaning of `nil` has - // changed since this type was first introduced: it now identifies a - // non-parameterized test case, whereas it originally identified a - // parameterized test case for which one or more arguments could not be - // encoded. If it's present in the decoding container, accept whatever value - // is decoded (which may be `nil`). If it's absent, default to a single - // argument ID marked as non-stable to maintain previous behavior. - let argumentIDs: [Test.Case.Argument.ID]? = if container.contains(.argumentIDs) { - try container.decode(type(of: argumentIDs), forKey: .argumentIDs) - } else { - [Test.Case.Argument.ID(bytes: [], isStable: false)] - } - - // The `discriminator` property was added after this type was first - // introduced. If it's present in the decoding container, accept whatever - // value is decoded (which may be `nil`). If it's absent, default to `nil` - // if `argumentIDs` was interpreted as a non-parameterized test above, or - // else 0, to maintain previous behavior. - let discriminator: Int? = if container.contains(.discriminator) { - try container.decode(type(of: discriminator), forKey: .discriminator) + if container.contains(.isStable) { + // `isStable` is present, so we're decoding an instance encoded using the + // newest style: every property can be decoded straightforwardly. + try self.init( + argumentIDs: container.decodeIfPresent([Test.Case.Argument.ID].self, forKey: .argumentIDs), + discriminator: container.decodeIfPresent(Int.self, forKey: .discriminator), + isStable: container.decode(Bool.self, forKey: .isStable) + ) + } else if container.contains(.argumentIDs) { + // `isStable` is absent, so we're decoding using the old style. Since + // `argumentIDs` is present, the representation should be considered + // stable. + let decodedArgumentIDs = try container.decode([Test.Case.Argument.ID].self, forKey: .argumentIDs) + let argumentIDs = decodedArgumentIDs.isEmpty ? nil : decodedArgumentIDs + + // Discriminator should be `nil` for the ID of a non-parameterized test + // case, but can default to 0 for the ID of a parameterized test case. + let discriminator = argumentIDs == nil ? nil : 0 + + self.init(argumentIDs: argumentIDs, discriminator: discriminator, isStable: true) } else { - if let argumentIDs { - argumentIDs.isEmpty ? nil : 0 - } else { - nil - } + // This is the old style, and since `argumentIDs` is absent, we know this + // ID represents a parameterized test case which is non-stable. + self.init(argumentIDs: [.init(bytes: [])], discriminator: 0, isStable: false) } - - self.init(argumentIDs: argumentIDs, discriminator: discriminator) - } - - public func encode(to encoder: some Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - - // The `argumentIDs` property is Optional but the meaning of `nil` has - // changed since this type was first introduced: it now identifies a - // non-parameterized test case, whereas it originally identified a - // parameterized test case for which one or more arguments could not be - // encoded. Explicitly encode `nil` values here, rather than omitting them, - // so that when decoding we can distinguish these two scenarios. - try container.encode(argumentIDs, forKey: .argumentIDs) - - // The `discriminator` property was added after this type was first - // introduced. Explicitly encode `nil` values here, rather than omitting - // them, so that when decoding we can distinguish the older vs. newer - // implementations. - try container.encode(discriminator, forKey: .discriminator) } } diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 75a3be09f..ab9183cf8 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -23,8 +23,7 @@ extension Test { /// non-parameterized test function. case nonParameterized - /// A test case associated with a parameterized test function, including - /// the argument(s) it was passed and a discriminator. + /// A test case associated with a parameterized test function. /// /// - Parameters: /// - arguments: The arguments passed to the parameterized test function @@ -32,7 +31,9 @@ extension Test { /// - discriminator: A number used to distinguish this test case from /// others associated with the same parameterized test function whose /// arguments have the same ID. - case parameterized(arguments: [Argument], discriminator: Int) + /// - isStable: Whether or not this test case is considered stable + /// across successive runs. + case parameterized(arguments: [Argument], discriminator: Int, isStable: Bool) } /// The kind of this test case. @@ -49,19 +50,8 @@ extension Test { /// The raw bytes of this instance's identifier. public var bytes: [UInt8] - /// Whether or not this argument ID is considered stable across - /// successive runs. - /// - /// If the value of this property is `true`, the testing library can use - /// this ID to deterministically match against the original argument - /// it represents, and a user can selectively (re-)run that argument - /// of the associated parameterized test. If it is `false`, that - /// functionality is not supported for the argument this ID represents. - public var isStable: Bool - - public init(bytes: some Sequence, isStable: Bool) { + init(bytes: some Sequence) { self.bytes = Array(bytes) - self.isStable = isStable } } @@ -82,21 +72,9 @@ extension Test { /// The parameter of the test function to which this argument was passed. public var parameter: Parameter - /// Initialize an instance of this type representing the specified - /// argument value. - /// - /// - Parameters: - /// - value: The value of this parameterized test argument. - /// - encodableValue: An encodable representation of `value`, if one is - /// available. When non-`nil`, this is used to attempt to form a - /// stable identifier. - /// - parameter: The parameter of the test function to which this - /// argument was passed. - /// - /// This forms an ``ID`` identifying `value` using `encodableValue`. - init(value: any Sendable, encodableValue: (any Encodable)?, parameter: Parameter) { + init(id: ID, value: any Sendable, parameter: Parameter) { + self.id = id self.value = value - self.id = .init(identifying: value, encodableValue: encodableValue, parameter: parameter) self.parameter = parameter } } @@ -117,7 +95,7 @@ extension Test { switch _kind { case .nonParameterized: nil - case let .parameterized(arguments, _): + case let .parameterized(arguments, _, _): arguments } } @@ -148,7 +126,7 @@ extension Test { switch _kind { case .nonParameterized: nil - case let .parameterized(_, discriminator): + case let .parameterized(_, discriminator, _): discriminator } } @@ -156,15 +134,27 @@ extension Test { switch _kind { case .nonParameterized: precondition(newValue == nil, "A non-nil discriminator may only be set for a test case which is parameterized.") - case let .parameterized(arguments, _): + case let .parameterized(arguments, _, isStable): guard let newValue else { preconditionFailure("A nil discriminator may only be set for a test case which is not parameterized.") } - _kind = .parameterized(arguments: arguments, discriminator: newValue) + _kind = .parameterized(arguments: arguments, discriminator: newValue, isStable: isStable) } } } + /// Whether or not this test case is considered stable across successive + /// runs. + @_spi(Experimental) @_spi(ForToolsIntegrationOnly) + public var isStable: Bool { + switch _kind { + case .nonParameterized: + true + case let .parameterized(_, _, isStable): + isStable + } + } + private init(kind: _Kind, body: @escaping @Sendable () async throws -> Void) { self._kind = kind self.body = body @@ -192,28 +182,40 @@ extension Test { parameters: [Parameter], body: @escaping @Sendable () async throws -> Void ) { - // Attempt to obtain an encodable representation of each value in order - // to construct a stable ID. - var encodableValues = [any Encodable]() - var hadFailure = false - for value in values { - // If we couldn't get an encodable representation of one of the values, - // give up and mark the overall attempt as a failure. This allows - // skipping unnecessary encoding work later: if any individual argument - // doesn't have a stable ID, the Test.Case.ID can't be considered stable, - // so there's no point encoding the values which _are_ encodable. - guard let encodableValue = encodableArgumentValue(for: value) else { - hadFailure = true - break + var isStable = true + + let arguments = zip(values, parameters).map { value, parameter in + var stableArgumentID: Argument.ID? + + // Attempt to get a stable, encoded representation of this value if no + // such attempts for previous values have failed. + if isStable { + do { + stableArgumentID = try .init(identifying: value, parameter: parameter) + } catch { + // FIXME: Capture the error and propagate to the user, not as a test + // failure but as an advisory warning. A missing stable argument ID + // will prevent re-running the test case, but isn't a blocking issue. + } } - encodableValues.append(encodableValue) - } - let arguments = zip(values.enumerated(), parameters).map { value, parameter in - let encodableValue = hadFailure ? nil : encodableValues[value.0] - return Argument(value: value.1, encodableValue: encodableValue, parameter: parameter) + let argumentID: Argument.ID + if let stableArgumentID { + argumentID = stableArgumentID + } else { + // If we couldn't get a stable representation of at least one value, + // give up and consider the overall test case non-stable. This allows + // skipping unnecessary work later: if any individual argument doesn't + // have a stable ID, there's no point encoding the values which _are_ + // encodable. + isStable = false + argumentID = .init(bytes: String(describingForTest: value).utf8) + } + + return Argument(id: argumentID, value: value, parameter: parameter) } - self.init(kind: .parameterized(arguments: arguments, discriminator: 0), body: body) + + self.init(kind: .parameterized(arguments: arguments, discriminator: 0, isStable: isStable), body: body) } /// Whether or not this test case is from a parameterized test. @@ -276,20 +278,7 @@ extension Test { // MARK: - Codable extension Test.Parameter: Codable {} - -extension Test.Case.Argument.ID: Codable { - public init(from decoder: any Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - - // The `isStable` property was added after this type was introduced. - // Previously, only stable argument IDs were ever encoded, so if we're - // attempting to decode one, we can safely assume it is stable. - let isStable = try container.decodeIfPresent(type(of: isStable), forKey: .isStable) ?? true - - let bytes = try container.decode(type(of: bytes), forKey: .bytes) - self.init(bytes: bytes, isStable: isStable) - } -} +extension Test.Case.Argument.ID: Codable {} // MARK: - Equatable, Hashable diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index 317198f65..653ad2a87 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -57,7 +57,7 @@ struct EventTests { let testID = Test.ID(moduleName: "ModuleName", nameComponents: ["NameComponent1", "NameComponent2"], sourceLocation: #_sourceLocation) - let testCaseID = Test.Case.ID(argumentIDs: nil, discriminator: nil) + let testCaseID = Test.Case.ID(argumentIDs: nil, discriminator: nil, isStable: true) let event = Event(kind, testID: testID, testCaseID: testCaseID, instant: .now) let eventSnapshot = Event.Snapshot(snapshotting: event) let decoded = try JSON.encodeAndDecode(eventSnapshot) diff --git a/Tests/TestingTests/Test.CaseTests.swift b/Tests/TestingTests/Test.CaseTests.swift index 2530d03fb..99a5bd879 100644 --- a/Tests/TestingTests/Test.CaseTests.swift +++ b/Tests/TestingTests/Test.CaseTests.swift @@ -10,8 +10,18 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +#if canImport(Foundation) +private import Foundation +#endif + @Suite("Test.Case Tests") struct Test_CaseTests { + @Test func nonParameterized() throws { + let testCase = Test.Case(body: {}) + #expect(testCase.id.argumentIDs == nil) + #expect(testCase.id.discriminator == nil) + } + @Test func singleStableArgument() throws { let testCase = Test.Case( values: [1], @@ -19,8 +29,6 @@ struct Test_CaseTests { body: {} ) #expect(testCase.id.isStable) - let arguments = try #require(testCase.arguments) - #expect(arguments.allSatisfy { $0.id.isStable }) } @Test func twoStableArguments() throws { @@ -33,8 +41,6 @@ struct Test_CaseTests { body: {} ) #expect(testCase.id.isStable) - let arguments = try #require(testCase.arguments) - #expect(arguments.allSatisfy { $0.id.isStable }) } @Test("Two arguments: one non-stable, followed by one stable") @@ -48,8 +54,65 @@ struct Test_CaseTests { body: {} ) #expect(!testCase.id.isStable) - let arguments = try #require(testCase.arguments) - #expect(arguments.allSatisfy { !$0.id.isStable }) + } + + @Suite("Test.Case.ID Tests") + struct IDTests { +#if canImport(Foundation) + @Test func legacyDecoding_stable() throws { + let encodedData = Data(""" + {"argumentIDs": [ + {"bytes": [1]} + ]} + """.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + + let argumentIDs = try #require(testCaseID.argumentIDs) + #expect(argumentIDs.count == 1) + } + + @Test func legacyDecoding_nonStable() throws { + let encodedData = Data("{}".utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(!testCaseID.isStable) + + let argumentIDs = try #require(testCaseID.argumentIDs) + #expect(argumentIDs.count == 1) + } + + @Test func legacyDecoding_nonParameterized() throws { + let encodedData = Data(#"{"argumentIDs": []}"#.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + #expect(testCaseID.argumentIDs == nil) + #expect(testCaseID.discriminator == nil) + } + + @Test func newDecoding_nonParameterized() throws { + let encodedData = Data(#"{"isStable": true}"#.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + #expect(testCaseID.argumentIDs == nil) + #expect(testCaseID.discriminator == nil) + } + + @Test func newDecoding_parameterizedStable() throws { + let encodedData = Data(""" + { + "isStable": true, + "argumentIDs": [ + {"bytes": [1]} + ], + "discriminator": 0 + } + """.utf8) + let testCaseID = try JSON.decode(Test.Case.ID.self, from: encodedData) + #expect(testCaseID.isStable) + #expect(testCaseID.argumentIDs?.count == 1) + #expect(testCaseID.discriminator == 0) + } +#endif } } diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 5a0121444..6807fd62a 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -9,10 +9,15 @@ // @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + #if canImport(XCTest) import XCTest #endif +#if canImport(Foundation) +import Foundation +#endif + extension Tag { /// A tag indicating that a test is related to a trait. @Tag static var traitRelated: Self @@ -348,6 +353,24 @@ extension JSON { try JSON.decode(T.self, from: data) } } + +#if canImport(Foundation) + /// Decode a value from JSON data. + /// + /// - Parameters: + /// - type: The type of value to decode. + /// - jsonRepresentation: Data of the JSON encoding of the value to decode. + /// + /// - Returns: An instance of `T` decoded from `jsonRepresentation`. + /// + /// - Throws: Whatever is thrown by the decoding process. + @_disfavoredOverload + static func decode(_ type: T.Type, from jsonRepresentation: Data) throws -> T where T: Decodable { + try jsonRepresentation.withUnsafeBytes { bytes in + try JSON.decode(type, from: bytes) + } + } +#endif } @available(_clockAPI, *) From 2a0827bca4f4513aedb69c555b5ea350f5cb77e1 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Fri, 14 Mar 2025 14:01:07 -0500 Subject: [PATCH 9/9] Refine Codable implementation of Test.Case.ID to maintain behavior with older versions of the testing library, validated by more tests --- .../Parameterization/Test.Case.ID.swift | 43 ++++++++++++--- Tests/TestingTests/Test.CaseTests.swift | 52 ++++++++++++++++++- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Parameterization/Test.Case.ID.swift b/Sources/Testing/Parameterization/Test.Case.ID.swift index 1d8e6e5b5..b29268104 100644 --- a/Sources/Testing/Parameterization/Test.Case.ID.swift +++ b/Sources/Testing/Parameterization/Test.Case.ID.swift @@ -68,10 +68,24 @@ extension Test.Case.ID: CustomStringConvertible { // MARK: - Codable extension Test.Case.ID: Codable { - private enum CodingKeys: CodingKey { - case argumentIDs + private enum CodingKeys: String, CodingKey { + /// A coding key for ``Test/Case/ID/argumentIDs``. + /// + /// This case's string value is non-standard because ``legacyArgumentIDs`` + /// already used "argumentIDs" and this needs to be different. + case argumentIDs = "argIDs" + + /// A coding key for ``Test/Case/ID/discriminator``. case discriminator + + /// A coding key for ``Test/Case/ID/isStable``. case isStable + + /// A coding key for the legacy representation of ``Test/Case/ID/argumentIDs``. + /// + /// This case's string value is non-standard in order to maintain + /// legacy compatibility with its original value. + case legacyArgumentIDs = "argumentIDs" } public init(from decoder: any Decoder) throws { @@ -85,11 +99,11 @@ extension Test.Case.ID: Codable { discriminator: container.decodeIfPresent(Int.self, forKey: .discriminator), isStable: container.decode(Bool.self, forKey: .isStable) ) - } else if container.contains(.argumentIDs) { - // `isStable` is absent, so we're decoding using the old style. Since - // `argumentIDs` is present, the representation should be considered - // stable. - let decodedArgumentIDs = try container.decode([Test.Case.Argument.ID].self, forKey: .argumentIDs) + } else if container.contains(.legacyArgumentIDs) { + // `isStable` is absent, so we're decoding using the old style. Since the + // legacy `argumentIDs` is present, the representation should be + // considered stable. + let decodedArgumentIDs = try container.decode([Test.Case.Argument.ID].self, forKey: .legacyArgumentIDs) let argumentIDs = decodedArgumentIDs.isEmpty ? nil : decodedArgumentIDs // Discriminator should be `nil` for the ID of a non-parameterized test @@ -103,6 +117,21 @@ extension Test.Case.ID: Codable { self.init(argumentIDs: [.init(bytes: [])], discriminator: 0, isStable: false) } } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + + try container.encode(isStable, forKey: .isStable) + try container.encodeIfPresent(discriminator, forKey: .discriminator) + try container.encodeIfPresent(argumentIDs, forKey: .argumentIDs) + + // Encode the legacy representation of `argumentIDs`. + if argumentIDs == nil { + try container.encode([Test.Case.Argument.ID](), forKey: .legacyArgumentIDs) + } else if isStable, let argumentIDs = argumentIDs { + try container.encode(argumentIDs, forKey: .legacyArgumentIDs) + } + } } // MARK: - Equatable, Hashable diff --git a/Tests/TestingTests/Test.CaseTests.swift b/Tests/TestingTests/Test.CaseTests.swift index 99a5bd879..f9e955fe9 100644 --- a/Tests/TestingTests/Test.CaseTests.swift +++ b/Tests/TestingTests/Test.CaseTests.swift @@ -59,6 +59,15 @@ struct Test_CaseTests { @Suite("Test.Case.ID Tests") struct IDTests { #if canImport(Foundation) + @Test(arguments: [ + Test.Case.ID(argumentIDs: nil, discriminator: nil, isStable: true), + Test.Case.ID(argumentIDs: [.init(bytes: "x".utf8)], discriminator: 0, isStable: false), + Test.Case.ID(argumentIDs: [.init(bytes: #""abc""#.utf8)], discriminator: 0, isStable: true), + ]) + func roundTripping(id: Test.Case.ID) throws { + #expect(try JSON.encodeAndDecode(id) == id) + } + @Test func legacyDecoding_stable() throws { let encodedData = Data(""" {"argumentIDs": [ @@ -101,7 +110,7 @@ struct Test_CaseTests { let encodedData = Data(""" { "isStable": true, - "argumentIDs": [ + "argIDs": [ {"bytes": [1]} ], "discriminator": 0 @@ -112,6 +121,42 @@ struct Test_CaseTests { #expect(testCaseID.argumentIDs?.count == 1) #expect(testCaseID.discriminator == 0) } + + @Test func newEncoding_nonParameterized() throws { + let id = Test.Case.ID(argumentIDs: nil, discriminator: nil, isStable: true) + let legacyID = try JSON.withEncoding(of: id) { data in + try JSON.decode(_LegacyTestCaseID.self, from: data) + } + let argumentIDs = try #require(legacyID.argumentIDs) + #expect(argumentIDs.isEmpty) + } + + @Test func newEncoding_parameterizedNonStable() throws { + let id = Test.Case.ID( + argumentIDs: [.init(bytes: "x".utf8)], + discriminator: 0, + isStable: false + ) + let legacyID = try JSON.withEncoding(of: id) { data in + try JSON.decode(_LegacyTestCaseID.self, from: data) + } + #expect(legacyID.argumentIDs == nil) + } + + @Test func newEncoding_parameterizedStable() throws { + let id = Test.Case.ID( + argumentIDs: [.init(bytes: #""abc""#.utf8)], + discriminator: 0, + isStable: true + ) + let legacyID = try JSON.withEncoding(of: id) { data in + try JSON.decode(_LegacyTestCaseID.self, from: data) + } + let argumentIDs = try #require(legacyID.argumentIDs) + #expect(argumentIDs.count == 1) + let argumentID = try #require(argumentIDs.first) + #expect(String(decoding: argumentID.bytes, as: UTF8.self) == #""abc""#) + } #endif } } @@ -125,3 +170,8 @@ private struct IssueRecordingEncodable: Encodable { Issue.record("Unexpected attempt to encode an instance of \(Self.self)") } } + +/// A fixture type which implements legacy decoding for ``Test/Case/ID``. +private struct _LegacyTestCaseID: Decodable { + var argumentIDs: [Test.Case.Argument.ID]? +}