From f0a42065d7fb733e35bf9d6494ddede22aa112ec Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 23 Apr 2024 09:31:55 -0400 Subject: [PATCH 1/5] Start defining and implementing a stable JSON schema for output. --- Documentation/ABI/JSON.md | 222 ++++++++++++++++++ .../Testing/EntryPoints/ABIEntryPoint.swift | 67 +----- .../ABIv0/ABIv0.Record+Streaming.swift | 106 +++++++++ .../EntryPoints/ABIv0/ABIv0.Record.swift | 85 +++++++ Sources/Testing/EntryPoints/ABIv0/ABIv0.swift | 12 + .../ABIv0/Encoded/ABIv0.EncodedEvent.swift | 103 ++++++++ .../ABIv0/Encoded/ABIv0.EncodedMessage.swift | 78 ++++++ .../ABIv0/Encoded/ABIv0.EncodedTest.swift | 122 ++++++++++ Sources/Testing/EntryPoints/EntryPoint.swift | 123 ++++++---- Sources/Testing/Events/Event.swift | 5 +- .../Event.HumanReadableOutputRecorder.swift | 11 +- Sources/Testing/Events/TimeValue.swift | 16 ++ Sources/Testing/Running/Runner.swift | 2 +- Sources/Testing/Support/JSON.swift | 4 + Tests/TestingTests/ABIEntryPointTests.swift | 7 +- Tests/TestingTests/EventTests.swift | 2 +- Tests/TestingTests/SwiftPMTests.swift | 39 ++- 17 files changed, 879 insertions(+), 125 deletions(-) create mode 100644 Documentation/ABI/JSON.md create mode 100644 Sources/Testing/EntryPoints/ABIv0/ABIv0.Record+Streaming.swift create mode 100644 Sources/Testing/EntryPoints/ABIv0/ABIv0.Record.swift create mode 100644 Sources/Testing/EntryPoints/ABIv0/ABIv0.swift create mode 100644 Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift create mode 100644 Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift create mode 100644 Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedTest.swift diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md new file mode 100644 index 000000000..2ba78dd63 --- /dev/null +++ b/Documentation/ABI/JSON.md @@ -0,0 +1,222 @@ +# JSON schema + + + +This document outlines the JSON schemas used by the testing library for its ABI +entry point and for the `--experimental-event-stream-output` command-line +argument. For more information about the ABI entry point, see the documentation +for [ABI.EntryPoint_v0](https://github.com/search?q=repo%3Aapple%2Fswift-testing%EntryPoint_v0&type=code). + +> [!WARNING] +> This JSON schema is still being developed and is subject to any and all +> changes including removal from the package. + +## Modified Backus-Naur form + +This schema is expressed using a modified Backus-Naur syntax. `{`, `}`, `:`, and +`,` represent their corresponding JSON tokens. `\n` represents an ASCII newline +character. + +The order of keys in JSON objects is not normative. Whitespace in this schema is +not normative; it is present to help the reader understand the content of the +various JSON objects in the schema. The event stream is output using the JSON +Lines format and does not include newline characters (except **one** at the end +of the `` rule.) + +Trailing commas in JSON objects and arrays are only to be included where +syntactically valid. + +### Common data types + +`` and `` are defined as in JSON. `` represents an +array (also defined as in JSON) whose elements all follow rule ``. + +``` + ::= true | false ; as in JSON + + ::= { + "fileID": , ; the Swift file ID of the file + "line": , + "column": , +} + + ::= "version": 0 ; will be incremented as the format changes + + ::= true | false ; boolean value as in JSON +``` + + + +### Streams + +A stream consists of a sequence of values encoded as [JSON Lines](https://jsonlines.org). +A single instance of `` is defined per test process and can be +accessed by passing `--experimental-event-stream-output` to the test executable +created by `swift build --build-tests`. + +``` + ::= \n | \n +``` + +### Records + +Records represent the values produced on a stream. Each record is encoded on a +single line and can be decoded independently of other lines. If a decoder +encounters a record whose `"kind"` field is unrecognized, the decoder should +ignore that line. + +``` + ::= | | + + ::= { + , + "kind": "metadata", + "payload": +} + + ::= { + , + "kind": "test", + "payload": +} + + ::= { + , + "kind": "event", + "payload": +} +``` + +### Metadata + +Metadata records are reserved for future use. + +``` + ::= { + ; unspecified JSON object content +} +``` + +### Tests + +Test records represent individual test functions and test suites. Test records +are passed through the record stream **before** most events. + + + +``` + ::= { + "kind": , + "name": , ; the unformatted function or non-qualified type name + ["displayName": ,] ; the user-supplied custom display name + "sourceLocation": , ; where the test is defined + "id": , +} + + ::= "suite" | "function" | "parameterizedFunction" + + ::= ; an opaque string representing the test case +``` + + + +### Events + +Event records represent things that can happen during testing. They include +information about the event such as when it occurred and where in the test +source it occurred. They also include a `"messages"` field that contains +sufficient information to display the event in a human-readable format. + +``` + ::= { + "kind": , + ["sourceLocation": ,] + "timestamp": , ; floating-point seconds since test epoch + "timestampSince1970": , ; floating-point seconds since UNIX epoch + "messages": , + ["testID": ,] +} + + ::= "runStarted" | "testStarted" | "testCaseStarted" | + "issueRecorded" | "knownIssueRecorded" | "testCaseEnded" | "testEnded" | + "testSkipped" | "runEnded" ; additional event kinds may be added in the future + + ::= { + "symbol": , + "text": , ; the human-readable text of this message + ["markdown": ] ; if available/desired/whatever, Markdown encoding + ; the same string as "text" +} + + ::= "default" | "skip" | "pass" | "passWithKnownIssue" | "fail" + "difference" | "warning" | "details" +``` + + diff --git a/Sources/Testing/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/EntryPoints/ABIEntryPoint.swift index 2ee2c22fb..696eabe6e 100644 --- a/Sources/Testing/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/EntryPoints/ABIEntryPoint.swift @@ -16,9 +16,8 @@ /// - argumentsJSON: A buffer to memory representing the JSON encoding of an /// instance of `__CommandLineArguments_v0`. If `nil`, a new instance is /// created from the command-line arguments to the current process. -/// - eventHandler: An event handler to which is passed a buffer to memory -/// representing each event and its context, as with ``Event/Handler``, but -/// encoded as JSON. +/// - recordHandler: A JSON record handler to which is passed a buffer to +/// memory representing each record as described in `ABI/JSON.md`. /// /// - Returns: The result of invoking the testing library. The type of this /// value is subject to change. @@ -31,7 +30,7 @@ @_spi(Experimental) @_spi(ForToolsIntegrationOnly) public typealias ABIEntryPoint_v0 = @Sendable ( _ argumentsJSON: UnsafeRawBufferPointer?, - _ eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void + _ recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void ) async -> CInt /// Get the entry point to the testing library used by tools that want to remain @@ -49,7 +48,8 @@ public typealias ABIEntryPoint_v0 = @Sendable ( /// /// The returned function can be thought of as equivalent to /// `swift test --experimental-event-stream-output` except that, instead of -/// streaming events to a named pipe or file, it streams them to a callback. +/// streaming JSON records to a named pipe or file, it streams them to an +/// in-process callback. /// /// - Warning: This function's signature and the structure of its JSON inputs /// and outputs have not been finalized yet. @@ -57,67 +57,14 @@ public typealias ABIEntryPoint_v0 = @Sendable ( @_spi(Experimental) @_spi(ForToolsIntegrationOnly) public func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) - result.initialize { argumentsJSON, eventHandler in + result.initialize { argumentsJSON, recordHandler in let args = try! argumentsJSON.map { argumentsJSON in try JSON.decode(__CommandLineArguments_v0.self, from: argumentsJSON) } - let eventHandler = eventHandlerForStreamingEvents_v0(to: eventHandler) + let eventHandler = try! eventHandlerForStreamingEvents(version: args?.experimentalEventStreamVersion, forwardingTo: recordHandler) return await entryPoint(passing: args, eventHandler: eventHandler) } return .init(result) } #endif - -// MARK: - Experimental event streaming - -#if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT) -/// A type containing an event snapshot and snapshots of the contents of an -/// event context suitable for streaming over JSON. -/// -/// This function is not part of the public interface of the testing library. -/// External adopters are not necessarily written in Swift and are expected to -/// decode the JSON produced for this type in implementation-specific ways. -struct EventAndContextSnapshot { - /// A snapshot of the event. - var event: Event.Snapshot - - /// A snapshot of the event context. - var eventContext: Event.Context.Snapshot -} - -extension EventAndContextSnapshot: Codable {} - -/// Create an event handler that encodes events as JSON and forwards them to an -/// ABI-friendly event handler. -/// -/// - Parameters: -/// - eventHandler: The event handler to forward events to. See -/// ``ABIEntryPoint_v0`` for more information. -/// -/// - Returns: An event handler. -/// -/// The resulting event handler outputs data as JSON. For each event handled by -/// the resulting event handler, a JSON object representing it and its -/// associated context is created and is passed to `eventHandler`. These JSON -/// objects are guaranteed not to contain any ASCII newline characters (`"\r"` -/// or `"\n"`). -/// -/// Note that `_eventHandlerForStreamingEvents_v0(toFileAtPath:)` calls this -/// function and performs additional postprocessing before writing JSON data. -func eventHandlerForStreamingEvents_v0( - to eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void -) -> Event.Handler { - return { event, context in - let snapshot = EventAndContextSnapshot( - event: Event.Snapshot(snapshotting: event), - eventContext: Event.Context.Snapshot(snapshotting: context) - ) - try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in - eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in - eventHandler(eventAndContextJSON) - } - } - } -} -#endif diff --git a/Sources/Testing/EntryPoints/ABIv0/ABIv0.Record+Streaming.swift b/Sources/Testing/EntryPoints/ABIv0/ABIv0.Record+Streaming.swift new file mode 100644 index 000000000..f6aaca920 --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/ABIv0.Record+Streaming.swift @@ -0,0 +1,106 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if canImport(Foundation) && (!SWT_NO_FILE_IO || !SWT_NO_ABI_ENTRY_POINT) +extension ABIv0.Record { + /// Create an event handler that encodes events as JSON and forwards them to + /// an ABI-friendly event handler. + /// + /// - Parameters: + /// - eventHandler: The event handler to forward events to. See + /// ``ABIv0/EntryPoint`` for more information. + /// + /// - Returns: An event handler. + /// + /// The resulting event handler outputs data as JSON. For each event handled + /// by the resulting event handler, a JSON object representing it and its + /// associated context is created and is passed to `eventHandler`. + /// + /// Note that `_eventHandlerForStreamingEvents(toFileAtPath:)` calls this + /// function and performs additional postprocessing before writing JSON data. + static func eventHandler( + forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void + ) -> Event.Handler { + let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder() + return { event, context in + if case let .runStarted(plan) = event.kind { + // Gather all tests in the run and forward records for them. + for test in plan.steps.lazy.map(\.test) { + try? JSON.withEncoding(of: Self(encoding: test)) { testJSON in + eventHandler(testJSON) + } + } + } + + let messages = humanReadableOutputRecorder.record(event, in: context) + if let eventRecord = Self(encoding: event, in: context, messages: messages) { + try? JSON.withEncoding(of: eventRecord, eventHandler) + } + } + } +} + +// MARK: - Experimental event streaming + +/// A type containing an event snapshot and snapshots of the contents of an +/// event context suitable for streaming over JSON. +/// +/// This type is not part of the public interface of the testing library. +/// External adopters are not necessarily written in Swift and are expected to +/// decode the JSON produced for this type in implementation-specific ways. +/// +/// - Warning: This type will be removed when the ABI version 0 JSON schema is +/// finalized. +struct EventAndContextSnapshot { + /// A snapshot of the event. + var event: Event.Snapshot + + /// A snapshot of the event context. + var eventContext: Event.Context.Snapshot +} + +extension EventAndContextSnapshot: Codable {} + +/// Create an event handler that encodes events as JSON and forwards them to an +/// ABI-friendly event handler. +/// +/// - Parameters: +/// - eventHandler: The event handler to forward events to. See +/// ``ABIEntryPoint_v0`` for more information. +/// +/// - Returns: An event handler. +/// +/// The resulting event handler outputs data as JSON. For each event handled by +/// the resulting event handler, a JSON object representing it and its +/// associated context is created and is passed to `eventHandler`. These JSON +/// objects are guaranteed not to contain any ASCII newline characters (`"\r"` +/// or `"\n"`). +/// +/// Note that `_eventHandlerForStreamingEvents_v0(toFileAtPath:)` calls this +/// function and performs additional postprocessing before writing JSON data. +/// +/// - Warning: This function will be removed when the ABI version 0 JSON schema +/// is finalized. +func eventHandlerForStreamingEventSnapshots( + to eventHandler: @escaping @Sendable (_ eventAndContextJSON: UnsafeRawBufferPointer) -> Void +) -> Event.Handler { + return { event, context in + let snapshot = EventAndContextSnapshot( + event: Event.Snapshot(snapshotting: event), + eventContext: Event.Context.Snapshot(snapshotting: context) + ) + try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in + eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in + eventHandler(eventAndContextJSON) + } + } + } +} +#endif diff --git a/Sources/Testing/EntryPoints/ABIv0/ABIv0.Record.swift b/Sources/Testing/EntryPoints/ABIv0/ABIv0.Record.swift new file mode 100644 index 000000000..520728b76 --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/ABIv0.Record.swift @@ -0,0 +1,85 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of records for the ABI entry point + /// and event stream output. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct Record: Sendable { + /// The version of this record. + /// + /// The value of this property corresponds to the ABI version i.e. `0`. + var version = 0 + + /// An enumeration describing the various kinds of record. + enum Kind: Sendable { + /// A test record. + case test(EncodedTest) + + /// An event record. + case event(EncodedEvent) + } + + /// The kind of record. + var kind: Kind + + init(encoding test: borrowing Test) { + kind = .test(EncodedTest(encoding: test)) + } + + init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { + guard let event = EncodedEvent(encoding: event, in: eventContext, messages: messages) else { + return nil + } + kind = .event(event) + } + } +} + +// MARK: - Codable + +extension ABIv0.Record: Codable { + private enum CodingKeys: String, CodingKey { + case version + case kind + case payload + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(version, forKey: .version) + switch kind { + case let .test(test): + try container.encode("test", forKey: .kind) + try container.encode(test, forKey: .payload) + case let .event(event): + try container.encode("event", forKey: .kind) + try container.encode(event, forKey: .payload) + } + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + version = try container.decode(Int.self, forKey: .version) + switch try container.decode(String.self, forKey: .kind) { + case "test": + let test = try container.decode(ABIv0.EncodedTest.self, forKey: .payload) + kind = .test(test) + case "event": + let event = try container.decode(ABIv0.EncodedEvent.self, forKey: .payload) + kind = .event(event) + case let kind: + throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "Unrecognized record kind '\(kind)'")) + } + } +} diff --git a/Sources/Testing/EntryPoints/ABIv0/ABIv0.swift b/Sources/Testing/EntryPoints/ABIv0/ABIv0.swift new file mode 100644 index 000000000..b872ed13a --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/ABIv0.swift @@ -0,0 +1,12 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A namespace for ABI version 0 symbols. +enum ABIv0: Sendable {} diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift new file mode 100644 index 000000000..bf9573962 --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift @@ -0,0 +1,103 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of ``Event`` for the ABI entry point + /// and event stream output. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct EncodedEvent: Sendable { + /// An enumeration describing the various kinds of event. + /// + /// Note that the set of encodable events is a subset of all events + /// generated at runtime by the testing library. + /// + /// For descriptions of individual cases, see ``Event/Kind``. + enum Kind: String, Sendable { + case runStarted + case testStarted + case testCaseStarted + case issueRecorded + case knownIssueRecorded + case testCaseEnded + case testEnded + case testSkipped + case runEnded + } + + /// The kind of event. + var kind: Kind + + /// The source location of the event, if applicable. + var sourceLocation: SourceLocation? + + /// The instant at which the event occurred on the current system's + /// suspending clock. + var timestamp: Double + + /// The instant at which the event occurred on the wall clock. + var timestampSince1970: Double + + /// Human-readable messages associated with this event that can be presented + /// to the user. + var messages: [EncodedMessage] + + /// The ID of the test associated with this event, if any. + var testID: EncodedTest.ID? + + /// The ID of the test case associated with this event, if any. + /// + /// - Warning: Test cases are not yet part of the JSON schema. + var _testCase: EncodedTestCase? + + init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { + if let test = eventContext.test { + sourceLocation = test.sourceLocation + } + switch event.kind { + case .runStarted: + kind = .runStarted + case .testStarted: + kind = .testStarted + case .testCaseStarted: + kind = .testCaseStarted + case let .issueRecorded(issue): + if issue.isKnown { + kind = .knownIssueRecorded + } else { + kind = .issueRecorded + } + sourceLocation = issue.sourceLocation + case .testCaseEnded: + kind = .testCaseEnded + case .testEnded: + kind = .testEnded + case .testSkipped: + kind = .testSkipped + case .runEnded: + kind = .runEnded + default: + return nil + } + timestamp = Double(event.instant.suspending) + timestampSince1970 = Double(event.instant.wall) + self.messages = messages.map(EncodedMessage.init) + testID = event.testID.map(EncodedTest.ID.init) + _testCase = eventContext.testCase.map(EncodedTestCase.init) + } + } +} + +// MARK: - Codable + +extension ABIv0.EncodedEvent: Codable {} +extension ABIv0.EncodedEvent.Kind: Codable {} diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift new file mode 100644 index 000000000..cd212d548 --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift @@ -0,0 +1,78 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of + /// ``Event/HumanReadableOutputRecorder/Message`` for the ABI entry point and + /// event stream output. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct EncodedMessage: Sendable { + /// A type implementing the JSON encoding of ``Event/Symbol`` for the ABI + /// entry point and event stream output. + /// + /// For descriptions of individual cases, see ``Event/Symbol``. + enum Symbol: String, Sendable { + case `default` + case skip + case pass + case passWithKnownIssue + case fail + case difference + case warning + case details + + init(encoding symbol: Event.Symbol) { + self = switch symbol { + case .default: + .default + case .skip: + .skip + case let .pass(knownIssueCount): + if knownIssueCount > 0 { + .passWithKnownIssue + } else { + .pass + } + case .fail: + .fail + case .difference: + .difference + case .warning: + .warning + case .details: + .details + } + } + } + + /// The symbol associated with this message. + var symbol: Symbol + + /// The human-readable, unformatted text associated with this message. + var text: String + + /// The human-readable, Markdown-formatted text associated with this + /// message, if any. + var markdown: String? + + init(encoding message: borrowing Event.HumanReadableOutputRecorder.Message) { + symbol = Symbol(encoding: message.symbol ?? .default) + text = message.conciseStringValue ?? message.stringValue + } + } +} + +// MARK: - Codable + +extension ABIv0.EncodedMessage: Codable {} +extension ABIv0.EncodedMessage.Symbol: Codable {} diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedTest.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedTest.swift new file mode 100644 index 000000000..556587b34 --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedTest.swift @@ -0,0 +1,122 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of ``Test`` for the ABI entry point + /// and event stream output. + /// + /// The properties and members of this type are documented in ABI/JSON.md. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct EncodedTest: Sendable { + /// An enumeration describing the various kinds of test. + enum Kind: String, Sendable { + /// A test suite. + case suite + + /// A test function. + case function + + /// A parameterized test function. + case parameterizedFunction + } + + /// The kind of test. + var kind: Kind + + /// The programmatic name of the test, such as its corresponding Swift + /// function or type name. + var name: String + + /// The developer-supplied human-readable name of the test. + var displayName: String? + + /// The source location of this test. + var sourceLocation: SourceLocation + + /// A type implementing the JSON encoding of ``Test/ID`` for the ABI entry + /// point and event stream output. + struct ID: Codable { + /// The string value representing the corresponding test ID. + var stringValue: String + + init(encoding testID: borrowing Test.ID) { + stringValue = String(describing: copy testID) + } + + func encode(to encoder: any Encoder) throws { + try stringValue.encode(to: encoder) + } + + init(from decoder: any Decoder) throws { + stringValue = try String(from: decoder) + } + } + + /// The unique identifier of this test. + var id: ID + + /// The test cases in this test, if it is a parameterized test function. + /// + /// - Warning: Test cases are not yet part of the JSON schema. + var _testCases: [EncodedTestCase]? + + init(encoding test: borrowing Test) { + if test.isSuite { + kind = .suite + } else if test.isParameterized { + kind = .parameterizedFunction + } else { + kind = .function + } + name = test.name + displayName = test.displayName + sourceLocation = test.sourceLocation + id = ID(encoding: test.id) + if test.isParameterized { + _testCases = test.testCases?.map(EncodedTestCase.init(encoding:)) + } + } + } +} + +extension ABIv0 { + /// A type implementing the JSON encoding of ``Test/Case`` for the ABI entry + /// point and event stream output. + /// + /// The properties and members of this type are documented in ABI/JSON.md. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + /// + /// - Warning: Test cases are not yet part of the JSON schema. + struct EncodedTestCase: Sendable { + var id: String + var displayName: String + + init(encoding testCase: borrowing Test.Case) { + // TODO: define an encodable form of Test.Case.ID + id = String(describing: testCase.id) + displayName = testCase.arguments.lazy + .map(\.value) + .map(String.init(describingForTest:)) + .joined(separator: ", ") + } + } +} + +// MARK: - Codable + +extension ABIv0.EncodedTest: Codable {} +extension ABIv0.EncodedTest.Kind: Codable {} +extension ABIv0.EncodedTestCase: Codable {} diff --git a/Sources/Testing/EntryPoints/EntryPoint.swift b/Sources/Testing/EntryPoints/EntryPoint.swift index 003214734..a0e99735a 100644 --- a/Sources/Testing/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/EntryPoints/EntryPoint.swift @@ -155,8 +155,33 @@ public struct __CommandLineArguments_v0: Sendable { public var xunitOutput: String? /// The value of the `--experimental-event-stream-output` argument. + /// + /// Data is written to this file in the [JSON Lines](https://jsonlines.org) + /// text format. For each event handled by the resulting event handler, a JSON + /// object representing it and its associated context is created and is + /// written, followed by a single line feed (`"\n"`) character. These JSON + /// objects are guaranteed not to contain any ASCII newline characters (`"\r"` + /// or `"\n"`) themselves. + /// + /// The file can be a regular file, however to allow for streaming a named + /// pipe is recommended. `mkfifo()` can be used on Darwin and Linux to create + /// a named pipe; `CreateNamedPipeA()` can be used on Windows. + /// + /// The file is closed when this process terminates or the test run completes, + /// whichever occurs first. public var experimentalEventStreamOutput: String? + /// The version of the event stream schema to use when writing events to + /// ``experimentalEventStreamOutput``. + /// + /// If the value of this property is `nil`, events are encoded verbatim (using + /// ``Event/Snapshot``.) Otherwise, the corresponding stable schema is used + /// (e.g. ``ABIv0/Record`` for `0`.) + /// + /// - Warning: The behavior of this property will change when the ABI version + /// 0 JSON schema is finalized. + public var experimentalEventStreamVersion: Int? = nil + /// The value(s) of the `--filter` argument. public var filter: [String]? @@ -222,6 +247,10 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum if let eventOutputIndex = args.firstIndex(of: "--experimental-event-stream-output"), !isLastArgument(at: eventOutputIndex) { result.experimentalEventStreamOutput = args[args.index(after: eventOutputIndex)] } + // Event stream output (experimental) + if let eventOutputVersionIndex = args.firstIndex(of: "--experimental-event-stream-version"), !isLastArgument(at: eventOutputVersionIndex) { + result.experimentalEventStreamVersion = Int(args[args.index(after: eventOutputVersionIndex)]) + } #endif // XML output @@ -301,7 +330,10 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr #if canImport(Foundation) // Event stream output (experimental) if let eventStreamOutputPath = args.experimentalEventStreamOutput { - let eventHandler = try _eventHandlerForStreamingEvents_v0(toFileAtPath: eventStreamOutputPath) + let file = try FileHandle(forWritingAtPath: eventStreamOutputPath) + let eventHandler = try eventHandlerForStreamingEvents(version: args.experimentalEventStreamVersion) { json in + try? _writeJSONLine(json, to: file) + } configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in eventHandler(event, context) oldEventHandler(event, context) @@ -361,69 +393,66 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr return configuration } -// MARK: - Experimental event streaming - #if canImport(Foundation) && !SWT_NO_FILE_IO -/// Create an event handler that streams events to the file at a given path. +/// Create an event handler that streams events to the given file using the +/// specified ABI version. /// /// - Parameters: -/// - path: The path to which events should be streamed. This file will be -/// opened for writing. -/// -/// - Throws: Any error that occurs opening `path`. Once `path` is opened, -/// errors that may occur writing to it are handled by the resulting event -/// handler. +/// - version: The ABI version to use. +/// - eventHandler: The event handler to forward encoded events to. The +/// encoding of events depends on `version`. /// /// - Returns: An event handler. /// -/// The resulting event handler outputs data in the [JSON Lines](https://jsonlines.org) -/// text format. For each event handled by the resulting event handler, a JSON -/// object representing it and its associated context is created and is written -/// to `path`, followed by a single line feed (`"\n"`) character. These JSON -/// objects are guaranteed not to contain any ASCII newline characters (`"\r"` -/// or `"\n"`) themselves. +/// - Throws: If `version` is not a supported ABI version. +func eventHandlerForStreamingEvents(version: Int?, forwardingTo eventHandler: @escaping @Sendable (UnsafeRawBufferPointer) -> Void) throws -> Event.Handler { + switch version { + case nil: + eventHandlerForStreamingEventSnapshots(to: eventHandler) + case 0: + ABIv0.Record.eventHandler(forwardingTo: eventHandler) + case let .some(unsupportedVersion): + throw _EntryPointError.invalidArgument("--experimental-event-stream-version", value: "\(unsupportedVersion)") + } +} + +/// Post-process encoded JSON and write it to a file. /// -/// The file at `path` can be a regular file, however to allow for streaming a -/// named pipe is recommended. `mkfifo()` can be used on Darwin and Linux to -/// create a named pipe; `CreateNamedPipeA()` can be used on Windows. +/// - Parameters: +/// - json: The JSON to write. +/// - file: The file to write to. /// -/// The file at `path` is closed when this process terminates or the -/// corresponding call to ``Runner/run()`` returns, whichever occurs first. -private func _eventHandlerForStreamingEvents_v0(toFileAtPath path: String) throws -> Event.Handler { - // Open the event stream file for writing. - let file = try FileHandle(forWritingAtPath: path) - - return eventHandlerForStreamingEvents_v0 { eventAndContextJSON in - func isASCIINewline(_ byte: UInt8) -> Bool { - byte == 10 || byte == 13 - } +/// - Throws: Whatever is thrown when writing to `file`. +private func _writeJSONLine(_ json: UnsafeRawBufferPointer, to file: borrowing FileHandle) throws { + func isASCIINewline(_ byte: UInt8) -> Bool { + byte == UInt8(ascii: "\r") || byte == UInt8(ascii: "\n") + } #if DEBUG && !SWT_NO_FILE_IO - // We don't actually expect the JSON encoder to produce output containing - // newline characters, so in debug builds we'll log a diagnostic message. - if eventAndContextJSON.contains(where: isASCIINewline) { - let message = Event.ConsoleOutputRecorder.warning( - "JSON encoder produced one or more newline characters while encoding an event snapshot. Please file a bug report at https://github.com/apple/swift-testing/issues/new", - options: .for(.stderr) - ) + // We don't actually expect the JSON encoder to produce output containing + // newline characters, so in debug builds we'll log a diagnostic message. + if json.contains(where: isASCIINewline) { + let message = Event.ConsoleOutputRecorder.warning( + "JSON encoder produced one or more newline characters while encoding an event snapshot. Please file a bug report at https://github.com/apple/swift-testing/issues/new", + options: .for(.stderr) + ) #if SWT_TARGET_OS_APPLE - try? FileHandle.stderr.write(message) + try? FileHandle.stderr.write(message) #else - print(message) + print(message) #endif - } + } #endif - // Remove newline characters to conform to JSON lines specification. - var eventAndContextJSON = Array(eventAndContextJSON) - eventAndContextJSON.removeAll(where: isASCIINewline) + // Remove newline characters to conform to JSON lines specification. + var json = Array(json) + json.removeAll(where: isASCIINewline) - try? file.withLock { - try eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in - try file.write(eventAndContextJSON) - } - try file.write("\n") + try file.withLock { + try json.withUnsafeBytes { json in + try file.write(json) } + try file.write("\n") } } #endif diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 6b98aef27..0c671247b 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -15,8 +15,11 @@ public struct Event: Sendable { public enum Kind: Sendable { /// A test run started. /// + /// - Parameters: + /// - plan: The test plan of the run that started. + /// /// This event is the first event posted after ``Runner/run()`` is called. - case runStarted + indirect case runStarted(_ plan: Runner.Plan) /// An iteration of the test run started. /// diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index f11555b00..aa693a5b6 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -27,6 +27,11 @@ extension Event { /// The human-readable message. var stringValue: String + + /// A concise version of ``stringValue``, if available. + /// + /// Not all messages include a concise string. + var conciseStringValue: String? } /// A type that contains mutable context for @@ -388,12 +393,14 @@ extension Event.HumanReadableOutputRecorder { let primaryMessage: Message = if parameterCount == 0 { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)" + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)", + conciseStringValue: String(describing: issue.kind) ) } else { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)" + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", + conciseStringValue: String(describing: issue.kind) ) } return CollectionOfOne(primaryMessage) + additionalMessages diff --git a/Sources/Testing/Events/TimeValue.swift b/Sources/Testing/Events/TimeValue.swift index 322246e68..c5cb52ce3 100644 --- a/Sources/Testing/Events/TimeValue.swift +++ b/Sources/Testing/Events/TimeValue.swift @@ -117,3 +117,19 @@ extension timespec { self.init(tv_sec: .init(timeValue.seconds), tv_nsec: .init(timeValue.attoseconds / 1_000_000_000)) } } + +extension FloatingPoint { + /// Initialize this floating-point value with the total number of seconds + /// (including the subsecond part) represented by an instance of + /// ``TimeValue``. + /// + /// - Parameters: + /// - timeValue: The instance of ``TimeValue`` to convert. + /// + /// The resulting value may have less precision than `timeValue` as most + /// floating-point types are unable to represent a time value's + /// ``TimeValue/attoseconds`` property exactly. + init(_ timeValue: TimeValue) { + self = Self(timeValue.seconds) + (Self(timeValue.attoseconds) / (1_000_000_000_000_000_000 as Self)) + } +} diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 9256eb7b5..f20f9a609 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -321,7 +321,7 @@ extension Runner { } await Configuration.withCurrent(runner.configuration) { - Event.post(.runStarted, for: nil, testCase: nil, configuration: runner.configuration) + Event.post(.runStarted(runner.plan), for: nil, testCase: nil, configuration: runner.configuration) defer { Event.post(.runEnded, for: nil, testCase: nil, configuration: runner.configuration) } diff --git a/Sources/Testing/Support/JSON.swift b/Sources/Testing/Support/JSON.swift index cda80b890..bec21af19 100644 --- a/Sources/Testing/Support/JSON.swift +++ b/Sources/Testing/Support/JSON.swift @@ -29,6 +29,10 @@ enum JSON { // Keys must be sorted to ensure deterministic matching of encoded data. encoder.outputFormatting.insert(.sortedKeys) + if Environment.flag(named: "SWT_PRETTY_PRINT_JSON") == true { + encoder.outputFormatting.insert(.prettyPrinted) + encoder.outputFormatting.insert(.withoutEscapingSlashes) + } // Set user info keys that clients want to use during encoding. encoder.userInfo.merge(userInfo, uniquingKeysWith: { _, rhs in rhs}) diff --git a/Tests/TestingTests/ABIEntryPointTests.swift b/Tests/TestingTests/ABIEntryPointTests.swift index 7fe914002..2188af040 100644 --- a/Tests/TestingTests/ABIEntryPointTests.swift +++ b/Tests/TestingTests/ABIEntryPointTests.swift @@ -36,6 +36,7 @@ struct ABIEntryPointTests { // Construct arguments and convert them to JSON. var arguments = __CommandLineArguments_v0() arguments.filter = ["NonExistentTestThatMatchesNothingHopefully"] + arguments.experimentalEventStreamVersion = 0 let argumentsJSON = try JSON.withEncoding(of: arguments) { argumentsJSON in let result = UnsafeMutableRawBufferPointer.allocate(byteCount: argumentsJSON.count, alignment: 1) _ = memcpy(result.baseAddress!, argumentsJSON.baseAddress!, argumentsJSON.count) @@ -46,9 +47,9 @@ struct ABIEntryPointTests { } // Call the entry point function. - let result = await abiEntryPoint.pointee(.init(argumentsJSON)) { eventAndContextJSON in - let eventAndContext = try! JSON.decode(EventAndContextSnapshot.self, from: eventAndContextJSON) - _ = (eventAndContext.event, eventAndContext.eventContext) + let result = await abiEntryPoint.pointee(.init(argumentsJSON)) { recordJSON in + let record = try! JSON.decode(ABIv0.Record.self, from: recordJSON) + _ = record.version } // Validate expectations. diff --git a/Tests/TestingTests/EventTests.swift b/Tests/TestingTests/EventTests.swift index a2d303034..0ee9b6bd7 100644 --- a/Tests/TestingTests/EventTests.swift +++ b/Tests/TestingTests/EventTests.swift @@ -44,7 +44,7 @@ struct EventTests { sourceLocation: nil) ) ), - Event.Kind.runStarted, + Event.Kind.runStarted(Runner.Plan(steps: [])), Event.Kind.runEnded, Event.Kind.testCaseStarted, Event.Kind.testCaseEnded, diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 42ff19284..eb620f122 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -151,7 +151,7 @@ struct SwiftPMTests { do { let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--xunit-output", temporaryFilePath]) let eventContext = Event.Context() - configuration.eventHandler(Event(.runStarted, testID: nil, testCaseID: nil), eventContext) + configuration.eventHandler(Event(.runStarted(Runner.Plan(steps: [])), testID: nil, testCaseID: nil), eventContext) configuration.eventHandler(Event(.runEnded, testID: nil, testCaseID: nil), eventContext) } @@ -163,31 +163,37 @@ struct SwiftPMTests { } #if canImport(Foundation) - func decodeEventStream(fromFileAtPath path: String) throws -> [EventAndContextSnapshot] { + func decodeABIv0RecordStream(fromFileAtPath path: String) throws -> [ABIv0.Record] { try FileHandle(forReadingAtPath: path).readToEnd() .split(separator: 10) // "\n" .map { line in try line.withUnsafeBytes { line in - try JSON.decode(EventAndContextSnapshot.self, from: line) + try JSON.decode(ABIv0.Record.self, from: line) } } } @Test("--experimental-event-stream-output argument (writes to a stream and can be read back)") func eventStreamOutput() async throws { - // Test that events are successfully streamed to a file and can be read - // back as snapshots. + // Test that JSON records are successfully streamed to a file and can be + // read back as snapshots. let tempDirPath = try temporaryDirectory() let temporaryFilePath = appendPathComponent("\(UInt64.random(in: 0 ..< .max))", to: tempDirPath) defer { _ = remove(temporaryFilePath) } do { - let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-event-stream-output", temporaryFilePath]) + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-event-stream-output", temporaryFilePath, "--experimental-event-stream-version", "0"]) let eventContext = Event.Context() - configuration.handleEvent(Event(.runStarted, testID: nil, testCaseID: nil), in: eventContext) + + let test = Test {} + let plan = Runner.Plan( + steps: [ + Runner.Plan.Step(test: test, action: .run(options: .init(isParallelizationEnabled: true))) + ] + ) + configuration.handleEvent(Event(.runStarted(plan), testID: nil, testCaseID: nil), in: eventContext) do { - let test = Test {} let eventContext = Event.Context(test: test) configuration.handleEvent(Event(.testStarted, testID: test.id, testCaseID: nil), in: eventContext) configuration.handleEvent(Event(.testEnded, testID: test.id, testCaseID: nil), in: eventContext) @@ -195,8 +201,21 @@ struct SwiftPMTests { configuration.handleEvent(Event(.runEnded, testID: nil, testCaseID: nil), in: eventContext) } - let decodedEvents = try decodeEventStream(fromFileAtPath: temporaryFilePath) - #expect(decodedEvents.count == 4) + let decodedRecords = try decodeABIv0RecordStream(fromFileAtPath: temporaryFilePath) + let testRecords = decodedRecords.compactMap { record in + if case let .test(test) = record.kind { + return test + } + return nil + } + #expect(testRecords.count == 1) + let eventRecords = decodedRecords.compactMap { record in + if case let .event(event) = record.kind { + return event + } + return nil + } + #expect(eventRecords.count == 4) } #endif #endif From a2aa5a018bf339b6c6b1755753232651589da96b Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 9 May 2024 17:19:27 -0400 Subject: [PATCH 2/5] Incorporate feedback --- Documentation/ABI/JSON.md | 2 -- .../ABIv0/Encoded/ABIv0.EncodedMessage.swift | 16 ++++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index 2ba78dd63..75e451ee6 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -208,8 +208,6 @@ sufficient information to display the event in a human-readable format. ::= { "symbol": , "text": , ; the human-readable text of this message - ["markdown": ] ; if available/desired/whatever, Markdown encoding - ; the same string as "text" } ::= "default" | "skip" | "pass" | "passWithKnownIssue" | "fail" diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift index cd212d548..e67b15309 100644 --- a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedMessage.swift @@ -34,9 +34,9 @@ extension ABIv0 { init(encoding symbol: Event.Symbol) { self = switch symbol { case .default: - .default + .default case .skip: - .skip + .skip case let .pass(knownIssueCount): if knownIssueCount > 0 { .passWithKnownIssue @@ -44,13 +44,13 @@ extension ABIv0 { .pass } case .fail: - .fail + .fail case .difference: - .difference + .difference case .warning: - .warning + .warning case .details: - .details + .details } } } @@ -61,10 +61,6 @@ extension ABIv0 { /// The human-readable, unformatted text associated with this message. var text: String - /// The human-readable, Markdown-formatted text associated with this - /// message, if any. - var markdown: String? - init(encoding message: borrowing Event.HumanReadableOutputRecorder.Message) { symbol = Symbol(encoding: message.symbol ?? .default) text = message.conciseStringValue ?? message.stringValue From 9cbcae9d61bb09890703f27ad8bf69f121ccd21a Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 10 May 2024 12:16:29 -0400 Subject: [PATCH 3/5] Incorporate feedback --- Documentation/ABI/JSON.md | 23 +++++++----- .../ABIv0/Encoded/ABIv0.EncodedEvent.swift | 32 ++++++----------- .../ABIv0/Encoded/ABIv0.EncodedInstant.swift | 36 +++++++++++++++++++ .../ABIv0/Encoded/ABIv0.EncodedIssue.swift | 34 ++++++++++++++++++ 4 files changed, 96 insertions(+), 29 deletions(-) create mode 100644 Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedInstant.swift create mode 100644 Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index 75e451ee6..0f94b4d0a 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -48,9 +48,12 @@ array (also defined as in JSON) whose elements all follow rule ``. "column": , } - ::= "version": 0 ; will be incremented as the format changes + ::= { + "absolute": , ; floating-point seconds since system-defined epoch + "since1970": , ; floating-point seconds since 1970-01-01 00:00:00 UT +} - ::= true | false ; boolean value as in JSON + ::= "version": 0 ; will be incremented as the format changes ``` +--> ``` ::= { @@ -194,16 +197,20 @@ sufficient information to display the event in a human-readable format. ``` ::= { "kind": , - ["sourceLocation": ,] - "timestamp": , ; floating-point seconds since test epoch - "timestampSince1970": , ; floating-point seconds since UNIX epoch + "instant": , ; when the event occurred + ["issue": ,] ; the recorded issue (if "kind" is "issueRecorded") "messages": , ["testID": ,] } ::= "runStarted" | "testStarted" | "testCaseStarted" | - "issueRecorded" | "knownIssueRecorded" | "testCaseEnded" | "testEnded" | - "testSkipped" | "runEnded" ; additional event kinds may be added in the future + "issueRecorded" | "testCaseEnded" | "testEnded" | "testSkipped" | + "runEnded" ; additional event kinds may be added in the future + + ::= { + "isKnown": , ; is this a known issue or not? + ["sourceLocation": ,] ; where the issue occurred, if known +} ::= { "symbol": , diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift index bf9573962..947ad3d62 100644 --- a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedEvent.swift @@ -27,7 +27,6 @@ extension ABIv0 { case testStarted case testCaseStarted case issueRecorded - case knownIssueRecorded case testCaseEnded case testEnded case testSkipped @@ -37,15 +36,14 @@ extension ABIv0 { /// The kind of event. var kind: Kind - /// The source location of the event, if applicable. - var sourceLocation: SourceLocation? + /// The instant at which the event occurred. + var instant: EncodedInstant - /// The instant at which the event occurred on the current system's - /// suspending clock. - var timestamp: Double - - /// The instant at which the event occurred on the wall clock. - var timestampSince1970: Double + /// The issue that occurred, if any. + /// + /// The value of this property is `nil` unless the value of the + /// ``kind-swift.property`` property is ``Kind-swift.enum/issueRecorded``. + var issue: EncodedIssue? /// Human-readable messages associated with this event that can be presented /// to the user. @@ -60,9 +58,6 @@ extension ABIv0 { var _testCase: EncodedTestCase? init?(encoding event: borrowing Event, in eventContext: borrowing Event.Context, messages: borrowing [Event.HumanReadableOutputRecorder.Message]) { - if let test = eventContext.test { - sourceLocation = test.sourceLocation - } switch event.kind { case .runStarted: kind = .runStarted @@ -70,13 +65,9 @@ extension ABIv0 { kind = .testStarted case .testCaseStarted: kind = .testCaseStarted - case let .issueRecorded(issue): - if issue.isKnown { - kind = .knownIssueRecorded - } else { - kind = .issueRecorded - } - sourceLocation = issue.sourceLocation + case let .issueRecorded(recordedIssue): + kind = .issueRecorded + issue = EncodedIssue(encoding: recordedIssue) case .testCaseEnded: kind = .testCaseEnded case .testEnded: @@ -88,8 +79,7 @@ extension ABIv0 { default: return nil } - timestamp = Double(event.instant.suspending) - timestampSince1970 = Double(event.instant.wall) + instant = EncodedInstant(encoding: event.instant) self.messages = messages.map(EncodedMessage.init) testID = event.testID.map(EncodedTest.ID.init) _testCase = eventContext.testCase.map(EncodedTestCase.init) diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedInstant.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedInstant.swift new file mode 100644 index 000000000..c2548acc6 --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedInstant.swift @@ -0,0 +1,36 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of ``Test/Clock/Instant`` for the + /// ABI entry point and event stream output. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct EncodedInstant: Sendable { + /// The number of seconds since the system-defined suspending epoch. + /// + /// For more information, see [`SuspendingClock`](https://developer.apple.com/documentation/swift/suspendingclock). + var absolute: Double + + /// The number of seconds since the UNIX epoch (1970-01-01 00:00:00 UT). + var since1970: Double + + init(encoding instant: borrowing Test.Clock.Instant) { + absolute = Double(instant.suspending) + since1970 = Double(instant.wall) + } + } +} + +// MARK: - Codable + +extension ABIv0.EncodedInstant: Codable {} diff --git a/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift new file mode 100644 index 000000000..05478645c --- /dev/null +++ b/Sources/Testing/EntryPoints/ABIv0/Encoded/ABIv0.EncodedIssue.swift @@ -0,0 +1,34 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABIv0 { + /// A type implementing the JSON encoding of ``Issue`` for the ABI entry point + /// and event stream output. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + struct EncodedIssue: Sendable { + /// Whether or not this issue is known to occur. + var isKnown: Bool + + /// The location in source where this issue occurred, if available. + var sourceLocation: SourceLocation? + + init(encoding issue: borrowing Issue) { + isKnown = issue.isKnown + sourceLocation = issue.sourceLocation + } + } +} + +// MARK: - Codable + +extension ABIv0.EncodedIssue: Codable {} From e1d6ad0dce0d6e76e9297d837726f57566696990 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 10 May 2024 14:23:33 -0400 Subject: [PATCH 4/5] Incorporate feedback --- Documentation/ABI/JSON.md | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index 0f94b4d0a..b10cec34b 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -48,7 +48,7 @@ array (also defined as in JSON) whose elements all follow rule ``. "column": , } - ::= { + ::= { "absolute": , ; floating-point seconds since system-defined epoch "since1970": , ; floating-point seconds since 1970-01-01 00:00:00 UT } @@ -118,13 +118,7 @@ encounters a record whose `"kind"` field is unrecognized, the decoder should ignore that line. ``` - ::= | | - - ::= { - , - "kind": "metadata", - "payload": -} + ::= | ::= { , @@ -139,16 +133,6 @@ ignore that line. } ``` -### Metadata - -Metadata records are reserved for future use. - -``` - ::= { - ; unspecified JSON object content -} -``` - ### Tests Test records represent individual test functions and test suites. Test records From 7172d9c5a73d982488d431ac7d5b734ebdd8dfbe Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 10 May 2024 14:36:55 -0400 Subject: [PATCH 5/5] Incorporate feedback about test functions --- Documentation/ABI/JSON.md | 25 +++++++++++++------ .../ABIv0/Encoded/ABIv0.EncodedTest.swift | 19 ++++++++------ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Documentation/ABI/JSON.md b/Documentation/ABI/JSON.md index b10cec34b..f01dcbbdf 100644 --- a/Documentation/ABI/JSON.md +++ b/Documentation/ABI/JSON.md @@ -145,24 +145,33 @@ additional `"testCases"` field describing the individual test cases. --> ``` - ::= { - "kind": , - "name": , ; the unformatted function or non-qualified type name + ::= | + + ::= { + "kind": "suite", + "name": , ; the unformatted, unqualified type name ["displayName": ,] ; the user-supplied custom display name - "sourceLocation": , ; where the test is defined + "sourceLocation": , ; where the test suite is defined "id": , } - ::= "suite" | "function" | "parameterizedFunction" + ::= { + "kind": "function", + "name": , ; the unformatted function name + ["displayName": ,] ; the user-supplied custom display name + "sourceLocation": , ; where the test is defined + "id": , + "isParameterized": ; is this a parameterized test function or not? +} ::= ; an opaque string representing the test case ```