From d259fcc97607ce006772a3d744818ced6ec74181 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 10 Apr 2018 13:08:32 -0400 Subject: [PATCH 01/25] Simplify AbsolutePosition offset calculation and support columns --- Sources/SwiftSyntax/Diagnostic.swift | 9 ++- Sources/SwiftSyntax/SyntaxData.swift | 85 +++++++++++----------------- Sources/SwiftSyntax/Trivia.swift.gyb | 69 +--------------------- 3 files changed, 37 insertions(+), 126 deletions(-) diff --git a/Sources/SwiftSyntax/Diagnostic.swift b/Sources/SwiftSyntax/Diagnostic.swift index 1fd286927a5..b676b77fdc5 100644 --- a/Sources/SwiftSyntax/Diagnostic.swift +++ b/Sources/SwiftSyntax/Diagnostic.swift @@ -30,7 +30,6 @@ public struct SourceLocation: Codable { public let file: String public init(file: String, position: AbsolutePosition) { - assert(position is UTF8Position, "must be utf8 position") self.init(line: position.line, column: position.column, offset: position.byteOffset, file: file) } @@ -88,7 +87,7 @@ public enum FixIt: Codable { let string = try container.decode(String.self, forKey: .string) let loc = try container.decode(SourceLocation.self, forKey: .location) self = .insert(loc, string) - case "replace": + case "replace": let string = try container.decode(String.self, forKey: .string) let range = try container.decode(SourceRange.self, forKey: .range) self = .replace(range, string) @@ -202,7 +201,7 @@ public struct Diagnostic: Codable { /// An array of possible FixIts to apply to this diagnostic. public let fixIts: [FixIt] - /// A diagnostic builder that + /// A diagnostic builder that public struct Builder { /// An in-flight array of notes. internal var notes = [Note]() @@ -225,7 +224,7 @@ public struct Diagnostic: Codable { /// - fixIts: Any FixIts that should be attached to this note. public mutating func note(_ message: Message, location: SourceLocation? = nil, - highlights: [SourceRange] = [], + highlights: [SourceRange] = [], fixIts: [FixIt] = []) { self.notes.append(Note(message: message, location: location, highlights: highlights, fixIts: fixIts)) @@ -252,7 +251,7 @@ public struct Diagnostic: Codable { /// Adds a FixIt to replace the contents of the source file corresponding /// to the provided SourceRange with the provided text. - public mutating + public mutating func fixItReplace(_ sourceRange: SourceRange, with text: String) { fixIts.append(.replace(sourceRange, text)) } diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index ed0c0c0eb5a..ebab7dd3797 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -65,10 +65,10 @@ final class SyntaxData: Equatable { } var position: AbsolutePosition { - return positionCache.value { return calculatePosition(UTF8Position()) } + return positionCache.value { return calculatePosition(AbsolutePosition()) } } - var positionAfterSkippingLeadingTrivia: AbsolutePosition { + var positionBeforeLeadingTrivia: AbsolutePosition { let result = position.copy() _ = raw.accumulateLeadingTrivia(result) return result @@ -261,78 +261,57 @@ final class SyntaxData: Equatable { /// An absolute position in a source file as text - the absolute byteOffset from /// the start, line, and column. -public class AbsolutePosition { +public final class AbsolutePosition { public fileprivate(set) var byteOffset: Int public fileprivate(set) var line: Int public fileprivate(set) var column: Int + public let encoding: Encoding - required public init(line: Int = 1, column: Int = 1, byteOffset: Int = 0) { + public enum Encoding { + case utf8 + case utf16 + } + + public init(line: Int = 1, column: Int = 1, byteOffset: Int = 0, + encoding: Encoding = .utf8) { self.line = line self.column = column self.byteOffset = byteOffset + self.encoding = encoding } - internal func add(text: String) { - preconditionFailure("this function must be overridden") - } - - internal func copy() -> Self { - return type(of: self).init(line: line, column: column, byteOffset: byteOffset) - } -} - -extension AbsolutePosition { - - /// Add some number of columns to the position. internal func add(columns: Int) { - column += columns - byteOffset += columns + self.column += columns } - /// Add some number of newlines to the position, resetting the column. - /// Size is byte size of newline char. - /// '\n' and '\r' are 1, '\r\n' is 2. internal func add(lines: Int, size: Int) { - line += lines - column = 1 - byteOffset += lines * size + self.line += lines * size + self.column = 1 } /// Use some text as a reference for adding to the absolute position, /// taking note of newlines, etc. - fileprivate func add(text chars: C) - where C.Element: UnsignedInteger { - let cr: C.Element = 13 - let nl: C.Element = 10 - var idx = chars.startIndex - while idx != chars.endIndex { - let c = chars[idx] - idx = chars.index(after: idx) - switch c { - case cr: - if chars[idx] == nl { - add(lines: 1, size: 2) - idx = chars.index(after: idx) - } else { - add(lines: 1, size: 1) - } - case nl: - add(lines: 1, size: 1) + internal func add(text: String) { + for char in text { + switch char { + case "\n", "\r\n": + line += 1 + column = 1 default: - add(columns: 1) + column += 1 } - } - } -} -class UTF8Position: AbsolutePosition { - internal override func add(text: String) { - add(text: text.utf8) + switch encoding { + case .utf8: + byteOffset += String(char).utf8.count + case .utf16: + byteOffset += String(char).utf16.count * 2 + } + } } -} -class UTF16Position: AbsolutePosition { - internal override func add(text: String) { - add(text: text.utf16) + internal func copy() -> Self { + return type(of: self).init(line: line, column: column, + byteOffset: byteOffset) } } diff --git a/Sources/SwiftSyntax/Trivia.swift.gyb b/Sources/SwiftSyntax/Trivia.swift.gyb index 4d8f363d757..1b99341a526 100644 --- a/Sources/SwiftSyntax/Trivia.swift.gyb +++ b/Sources/SwiftSyntax/Trivia.swift.gyb @@ -99,59 +99,6 @@ extension TriviaPiece: TextOutputStreamable { case let .${trivia.lower_name}(text): target.write(text) % end -% end - } - } - - /// Computes the information from this trivia to inform the source locations - /// of the associated tokens. - /// Specifically, walks through the trivia and keeps track of every newline - /// to give a number of how many newlines and UTF8 characters appear in the - /// trivia, along with the UTF8 offset of the last column. - func characterSizes() -> (lines: Int, lastColumn: Int, utf8Length: Int) { - func calculateTextSizes(_ text: String) -> - (lines: Int, lastColumn: Int, utf8Length: Int) { - var lines = 0 - var col = 0 - var total = 0 - var prevChar: UInt8? = nil - // TODO: CR + LF should be regarded as one newline - for char in text.utf8 { - total += 1 - switch char { - case 0x0a: - if prevChar == 0x0d { - /* ASCII CR LF */ - assert(col == 0) - } else { - /* ASCII newline */ - col = 0 - lines += 1 - } - /* ASCII carriage-return */ - case 0x0d: - col = 0 - lines += 1 - - default: - col += 1 - } - prevChar = char - } - return (lines: lines, lastColumn: col, utf8Length: total) - } - switch self { -% for trivia in TRIVIAS: -% if trivia.is_new_line: - case let .${trivia.lower_name}s(n): - return (lines: n, lastColumn: 0, utf8Length: n * ${trivia.characters_len()}) -% elif trivia.is_collection(): - case let .${trivia.lower_name}s(n): - return (lines: 0, lastColumn: n, utf8Length: n * ${trivia.characters_len()}) -% else: - case let .${trivia.lower_name}(text): - return calculateTextSizes(text) -% end % end } } @@ -209,20 +156,6 @@ public struct Trivia: Codable { } % end % end - - /// Computes the total sizes and offsets of all pieces in this Trivia. - func characterSizes() -> (lines: Int, lastColumn: Int, utf8Length: Int) { - var lines = 0 - var lastColumn = 0 - var length = 0 - for piece in pieces { - let (ln, col, len) = piece.characterSizes() - lines += ln - lastColumn = col - length += len - } - return (lines: lines, lastColumn: lastColumn, utf8Length: length) - } } /// Conformance for Trivia to the Collection protocol. @@ -245,7 +178,7 @@ extension Trivia: Collection { /// Get the byteSize of this trivia public var byteSize: Int { - let pos = UTF8Position() + let pos = AbsolutePosition() for piece in pieces { piece.accumulateAbsolutePosition(pos) } From 501846683d47b53c58a89bf9fd8e5fb790e881ce Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 24 Apr 2018 11:49:24 -0400 Subject: [PATCH 02/25] Clarify comment --- Sources/SwiftSyntax/Diagnostic.swift | 6 +++++- Sources/SwiftSyntax/SyntaxData.swift | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSyntax/Diagnostic.swift b/Sources/SwiftSyntax/Diagnostic.swift index b676b77fdc5..795c20be7cc 100644 --- a/Sources/SwiftSyntax/Diagnostic.swift +++ b/Sources/SwiftSyntax/Diagnostic.swift @@ -201,7 +201,11 @@ public struct Diagnostic: Codable { /// An array of possible FixIts to apply to this diagnostic. public let fixIts: [FixIt] - /// A diagnostic builder that + /// A diagnostic builder that exposes mutating operations for notes, + /// highlights, and FixIts. When a Diagnostic is created, a builder + /// will be provided in a closure where the user can conditionally + /// add notes, highlights, and FixIts, that will then be wrapped + /// into the immutable Diagnostic object. public struct Builder { /// An in-flight array of notes. internal var notes = [Note]() diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index ebab7dd3797..63c6cfaa320 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -301,10 +301,13 @@ public final class AbsolutePosition { column += 1 } + // FIXME: This is currently very wasteful, but should be fast once the small-string + // optimization lands. switch encoding { case .utf8: byteOffset += String(char).utf8.count case .utf16: + // FIXME: Is this correct? byteOffset += String(char).utf16.count * 2 } } From 8044478eefe00e5e58d880260ca99a68f803eb06 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 24 Apr 2018 11:50:31 -0400 Subject: [PATCH 03/25] Un-rename property --- Sources/SwiftSyntax/SyntaxData.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 63c6cfaa320..27f4572719f 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -68,7 +68,7 @@ final class SyntaxData: Equatable { return positionCache.value { return calculatePosition(AbsolutePosition()) } } - var positionBeforeLeadingTrivia: AbsolutePosition { + var positionAfterSkippingLeadingTrivia: AbsolutePosition { let result = position.copy() _ = raw.accumulateLeadingTrivia(result) return result From b664867f6f24a7bf03a935a9db7cb339bd066082 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 24 Apr 2018 14:10:57 -0400 Subject: [PATCH 04/25] Monomorphize AbsolutePosition.copy() --- Sources/SwiftSyntax/SyntaxData.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 27f4572719f..72775b9ff64 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -313,8 +313,8 @@ public final class AbsolutePosition { } } - internal func copy() -> Self { - return type(of: self).init(line: line, column: column, + internal func copy() -> AbsolutePosition { + return AbsolutePosition(line: line, column: column, byteOffset: byteOffset) } } From 23502d3adecc2ef6392310569f06608b78b08691 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 24 Apr 2018 16:14:37 -0400 Subject: [PATCH 05/25] Rename byteOffset to utf8Offset and remove utf16 --- Sources/SwiftSyntax/Diagnostic.swift | 2 +- Sources/SwiftSyntax/SyntaxData.swift | 62 +--------------------------- Sources/SwiftSyntax/Trivia.swift.gyb | 4 +- 3 files changed, 4 insertions(+), 64 deletions(-) diff --git a/Sources/SwiftSyntax/Diagnostic.swift b/Sources/SwiftSyntax/Diagnostic.swift index 795c20be7cc..bf5b776efa2 100644 --- a/Sources/SwiftSyntax/Diagnostic.swift +++ b/Sources/SwiftSyntax/Diagnostic.swift @@ -31,7 +31,7 @@ public struct SourceLocation: Codable { public init(file: String, position: AbsolutePosition) { self.init(line: position.line, column: position.column, - offset: position.byteOffset, file: file) + offset: position.utf8Offset, file: file) } public init(line: Int, column: Int, offset: Int, file: String) { diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 72775b9ff64..554503888b7 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -94,7 +94,7 @@ final class SyntaxData: Equatable { } var byteSize: Int { - return getNextSiblingPos().byteOffset - self.position.byteOffset + return getNextSiblingPos().utf8Offset - self.position.utf8Offset } /// Creates a SyntaxData with the provided raw syntax, pointing to the @@ -258,63 +258,3 @@ final class SyntaxData: Equatable { return lhs === rhs } } - -/// An absolute position in a source file as text - the absolute byteOffset from -/// the start, line, and column. -public final class AbsolutePosition { - public fileprivate(set) var byteOffset: Int - public fileprivate(set) var line: Int - public fileprivate(set) var column: Int - public let encoding: Encoding - - public enum Encoding { - case utf8 - case utf16 - } - - public init(line: Int = 1, column: Int = 1, byteOffset: Int = 0, - encoding: Encoding = .utf8) { - self.line = line - self.column = column - self.byteOffset = byteOffset - self.encoding = encoding - } - - internal func add(columns: Int) { - self.column += columns - } - - internal func add(lines: Int, size: Int) { - self.line += lines * size - self.column = 1 - } - - /// Use some text as a reference for adding to the absolute position, - /// taking note of newlines, etc. - internal func add(text: String) { - for char in text { - switch char { - case "\n", "\r\n": - line += 1 - column = 1 - default: - column += 1 - } - - // FIXME: This is currently very wasteful, but should be fast once the small-string - // optimization lands. - switch encoding { - case .utf8: - byteOffset += String(char).utf8.count - case .utf16: - // FIXME: Is this correct? - byteOffset += String(char).utf16.count * 2 - } - } - } - - internal func copy() -> AbsolutePosition { - return AbsolutePosition(line: line, column: column, - byteOffset: byteOffset) - } -} diff --git a/Sources/SwiftSyntax/Trivia.swift.gyb b/Sources/SwiftSyntax/Trivia.swift.gyb index 1b99341a526..c4d34055e34 100644 --- a/Sources/SwiftSyntax/Trivia.swift.gyb +++ b/Sources/SwiftSyntax/Trivia.swift.gyb @@ -182,7 +182,7 @@ extension Trivia: Collection { for piece in pieces { piece.accumulateAbsolutePosition(pos) } - return pos.byteOffset + return pos.utf8Offset } } @@ -214,6 +214,6 @@ extension TriviaPiece { pos.add(text: text) % end % end - } + } } } From b970c42781b9cdedbaf2122de4a52b340578da51 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 24 Apr 2018 16:15:55 -0400 Subject: [PATCH 06/25] Re-add AbsolutePosition.swift --- Sources/SwiftSyntax/AbsolutePosition.swift | 60 ++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 Sources/SwiftSyntax/AbsolutePosition.swift diff --git a/Sources/SwiftSyntax/AbsolutePosition.swift b/Sources/SwiftSyntax/AbsolutePosition.swift new file mode 100644 index 00000000000..55950d65029 --- /dev/null +++ b/Sources/SwiftSyntax/AbsolutePosition.swift @@ -0,0 +1,60 @@ +//===--------------- AbsolutePosition.swift - Source Positions ------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// An absolute position in a source file as text - the absolute utf8Offset from +/// the start, line, and column. +public final class AbsolutePosition { + public fileprivate(set) var utf8Offset: Int + public fileprivate(set) var line: Int + public fileprivate(set) var column: Int + + public init(line: Int = 1, column: Int = 1, utf8Offset: Int = 0) { + self.line = line + self.column = column + self.utf8Offset = utf8Offset + } + + internal func add(columns: Int) { + print("adding \(columns) column bytes") + self.column += columns + } + + internal func add(lines: Int, size: Int) { + print("adding \(lines * size) line bytes") + self.line += lines * size + self.column = 1 + } + + /// Use some text as a reference for adding to the absolute position, + /// taking note of newlines, etc. + internal func add(text: String) { + print("adding text: \(text)") + for char in text { + switch char { + case "\n", "\r\n": + line += 1 + column = 1 + default: + column += 1 + } + + // FIXME: This is currently very wasteful, but should be fast once the + // small-string optimization lands. + utf8Offset += String(char).utf8.count + } + } + + internal func copy() -> AbsolutePosition { + return AbsolutePosition(line: line, column: column, + utf8Offset: utf8Offset) + } +} From cc54246c4fe87b49a61fa65add03b0f6b49824c3 Mon Sep 17 00:00:00 2001 From: Harlan Haskins Date: Tue, 24 Apr 2018 16:41:19 -0400 Subject: [PATCH 07/25] Actually add offsets in add(columns:) and add(lines:size:) --- Sources/SwiftSyntax/AbsolutePosition.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Sources/SwiftSyntax/AbsolutePosition.swift b/Sources/SwiftSyntax/AbsolutePosition.swift index 55950d65029..65be3b54d3d 100644 --- a/Sources/SwiftSyntax/AbsolutePosition.swift +++ b/Sources/SwiftSyntax/AbsolutePosition.swift @@ -24,20 +24,19 @@ public final class AbsolutePosition { } internal func add(columns: Int) { - print("adding \(columns) column bytes") self.column += columns + self.utf8Offset += columns } internal func add(lines: Int, size: Int) { - print("adding \(lines * size) line bytes") self.line += lines * size self.column = 1 + self.utf8Offset += lines * size } /// Use some text as a reference for adding to the absolute position, /// taking note of newlines, etc. internal func add(text: String) { - print("adding text: \(text)") for char in text { switch char { case "\n", "\r\n": From 60e37d0cc5442aa9b8644b7cdcf484d9075b0e04 Mon Sep 17 00:00:00 2001 From: Harlan Date: Thu, 26 Apr 2018 16:17:07 +0200 Subject: [PATCH 08/25] Add accessors for source locations and test diagnostic emission (#16141) * [SwiftSyntax] Add accessors for source locations and test diagnostic emission * Add tests for endLocation * Pre-emptively copy AbsolutePosition to avoid mutating it twice --- Sources/SwiftSyntax/RawSyntax.swift | 7 ++++ Sources/SwiftSyntax/Syntax.swift | 50 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index 873d4034975..dd3c3a46ad7 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -251,6 +251,13 @@ extension RawSyntax { } } + func accumulateTrailingTrivia(_ pos: AbsolutePosition) { + guard let trivia = trailingTrivia else { return } + for piece in trivia { + piece.accumulateAbsolutePosition(pos) + } + } + var isSourceFile: Bool { switch self { case .node(let kind, _, _): diff --git a/Sources/SwiftSyntax/Syntax.swift b/Sources/SwiftSyntax/Syntax.swift index 8f4535f6f04..eae8d881d21 100644 --- a/Sources/SwiftSyntax/Syntax.swift +++ b/Sources/SwiftSyntax/Syntax.swift @@ -202,6 +202,56 @@ extension Syntax { where Target: TextOutputStream { data.raw.write(to: &target) } + + /// The starting location, in the provided file, of this Syntax node. + /// - Parameters: + /// - file: The file URL this node resides in. + /// - afterLeadingTrivia: Whether to skip leading trivia when getting + /// the node's location. Defaults to `true`. + public func startLocation( + in file: URL, + afterLeadingTrivia: Bool = true + ) -> SourceLocation { + let pos = afterLeadingTrivia ? + data.position.copy() : + data.positionAfterSkippingLeadingTrivia.copy() + return SourceLocation(file: file.path, position: pos) + } + + + /// The ending location, in the provided file, of this Syntax node. + /// - Parameters: + /// - file: The file URL this node resides in. + /// - afterTrailingTrivia: Whether to skip trailing trivia when getting + /// the node's location. Defaults to `false`. + public func endLocation( + in file: URL, + afterTrailingTrivia: Bool = false + ) -> SourceLocation { + let pos = data.position.copy() + raw.accumulateAbsolutePosition(pos) + if afterTrailingTrivia { + raw.accumulateTrailingTrivia(pos) + } + return SourceLocation(file: file.path, position: pos) + } + + /// The source range, in the provided file, of this Syntax node. + /// - Parameters: + /// - file: The file URL this node resides in. + /// - afterLeadingTrivia: Whether to skip leading trivia when getting + /// the node's start location. Defaults to `true`. + /// - afterTrailingTrivia: Whether to skip trailing trivia when getting + /// the node's end location. Defaults to `false`. + public func sourceRange( + in file: URL, + afterLeadingTrivia: Bool = true, + afterTrailingTrivia: Bool = false + ) -> SourceRange { + let start = startLocation(in: file, afterLeadingTrivia: afterLeadingTrivia) + let end = endLocation(in: file, afterTrailingTrivia: afterTrailingTrivia) + return SourceRange(start: start, end: end) + } } /// Determines if two nodes are equal to each other. From 3dc6e4241eac73df7c0ff662178d859c932762fb Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Fri, 31 Aug 2018 07:53:40 -0700 Subject: [PATCH 09/25] Manually cherry-pick tests from apple/swift: 572a144d Rename byteOffset to utf8Offset and remote utf16 9323d08b [SwiftSyntax] Add accessors for source locations and test diagnostic emission (#16141) --- Tests/SwiftSyntaxTest/AbsolutePosition.swift | 30 +++++++------ Tests/SwiftSyntaxTest/DiagnosticTest.swift | 43 ++++++++++++++++++- .../SwiftSyntaxTest/Inputs/diagnostics.swift | 11 +++++ 3 files changed, 70 insertions(+), 14 deletions(-) create mode 100644 Tests/SwiftSyntaxTest/Inputs/diagnostics.swift diff --git a/Tests/SwiftSyntaxTest/AbsolutePosition.swift b/Tests/SwiftSyntaxTest/AbsolutePosition.swift index a7ace0cf5e3..3f451bafffd 100644 --- a/Tests/SwiftSyntaxTest/AbsolutePosition.swift +++ b/Tests/SwiftSyntaxTest/AbsolutePosition.swift @@ -13,10 +13,11 @@ public class AbsolutePositionTestCase: XCTestCase { XCTAssertNoThrow(try { let source = try String(contentsOf: getInput("visitor.swift")) let parsed = try SourceFileSyntax.parse(getInput("visitor.swift")) - XCTAssertEqual(parsed.position.byteOffset, 0) - XCTAssertEqual(parsed.eofToken.positionAfterSkippingLeadingTrivia.byteOffset, - source.count) - XCTAssertEqual(parsed.position.byteOffset, 0) + XCTAssertEqual(parsed.position.utf8Offset, 0) + XCTAssertEqual( + parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset, + source.count) + XCTAssertEqual(parsed.position.utf8Offset, 0) XCTAssertEqual(parsed.byteSize, source.count) }()) } @@ -25,9 +26,10 @@ public class AbsolutePositionTestCase: XCTestCase { XCTAssertNoThrow(try { let source = try String(contentsOf: getInput("closure.swift")) let parsed = try SourceFileSyntax.parse(getInput("closure.swift")) - XCTAssertEqual(parsed.eofToken.positionAfterSkippingLeadingTrivia.byteOffset, - source.count) - XCTAssertEqual(parsed.position.byteOffset, 0) + XCTAssertEqual( + parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset, + source.count) + XCTAssertEqual(parsed.position.utf8Offset, 0) XCTAssertEqual(parsed.byteSize, source.count) }()) } @@ -37,8 +39,9 @@ public class AbsolutePositionTestCase: XCTestCase { let parsed = try SourceFileSyntax.parse(getInput("visitor.swift")) let renamed = FuncRenamer().visit(parsed) as! SourceFileSyntax let renamedSource = renamed.description - XCTAssertEqual(renamed.eofToken.positionAfterSkippingLeadingTrivia.byteOffset, - renamedSource.count) + XCTAssertEqual( + renamed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset, + renamedSource.count) XCTAssertEqual(renamed.byteSize, renamedSource.count) }()) } @@ -53,8 +56,8 @@ public class AbsolutePositionTestCase: XCTestCase { _ = node.positionAfterSkippingLeadingTrivia } override func visit(_ node: TokenSyntax) { - XCTAssertEqual(node.position.byteOffset + node.leadingTrivia.byteSize, - node.positionAfterSkippingLeadingTrivia.byteOffset) + XCTAssertEqual(node.positionAfterSkippingLeadingTrivia.utf8Offset, + node.position.utf8Offset + node.leadingTrivia.byteSize) } } Visitor().visit(parsed) @@ -106,8 +109,9 @@ public class AbsolutePositionTestCase: XCTestCase { let state = root.statements[idx] XCTAssertEqual(state.leadingTrivia!.count, 3) XCTAssertEqual(state.trailingTrivia!.count, 1) - XCTAssertEqual(state.leadingTrivia!.byteSize + state.trailingTrivia!.byteSize - + state.byteSizeAfterTrimmingTrivia, state.byteSize) + XCTAssertEqual(state.byteSize, + state.leadingTrivia!.byteSize + state.trailingTrivia!.byteSize + + state.byteSizeAfterTrimmingTrivia) XCTAssertFalse(root.statements.isImplicit) } diff --git a/Tests/SwiftSyntaxTest/DiagnosticTest.swift b/Tests/SwiftSyntaxTest/DiagnosticTest.swift index 9e34a791fd1..ce89885ab98 100644 --- a/Tests/SwiftSyntaxTest/DiagnosticTest.swift +++ b/Tests/SwiftSyntaxTest/DiagnosticTest.swift @@ -18,6 +18,13 @@ extension Diagnostic.Message { /// Suggestion for the user to explicitly check a value does not equal zero. static let checkEqualToZero = Diagnostic.Message(.note, "check for explicit equality to '0'") + + static func badFunction(_ name: TokenSyntax) -> Diagnostic.Message { + return .init(.error, "bad function '\(name.text)'") + } + static func endOfFunction(_ name: TokenSyntax) -> Diagnostic.Message { + return .init(.warning, "end of function '\(name.text)'") + } } public class DiagnosticsTestCase: XCTestCase { @@ -50,4 +57,38 @@ public class DiagnosticsTestCase: XCTestCase { guard let fixIt = note.fixIts.first else { return } XCTAssertEqual(fixIt.text, " != 0") } -} \ No newline at end of file + + public func testSourceLocations() { + let engine = DiagnosticEngine() + engine.addConsumer(PrintingDiagnosticConsumer()) + let url = getInput("diagnostics.swift") + + class Visitor: SyntaxVisitor { + let url: URL + let engine: DiagnosticEngine + init(url: URL, engine: DiagnosticEngine) { + self.url = url + self.engine = engine + } + override func visit(_ function: FunctionDeclSyntax) { + let startLoc = function.identifier.startLocation(in: url) + let endLoc = function.endLocation(in: url) + engine.diagnose(.badFunction(function.identifier), location: startLoc) { + $0.highlight(function.identifier.sourceRange(in: self.url)) + } + engine.diagnose(.endOfFunction(function.identifier), location: endLoc) + } + } + + XCTAssertNoThrow(try { + let file = try SourceFileSyntax.parse(url) + Visitor(url: url, engine: engine).visit(file) + }()) + + XCTAssertEqual(6, engine.diagnostics.count) + let lines = Set(engine.diagnostics.compactMap { $0.location?.line }) + XCTAssertEqual([1, 3, 5, 7, 9, 11], lines) + let columns = Set(engine.diagnostics.compactMap { $0.location?.column }) + XCTAssertEqual([6, 2], columns) + } +} diff --git a/Tests/SwiftSyntaxTest/Inputs/diagnostics.swift b/Tests/SwiftSyntaxTest/Inputs/diagnostics.swift new file mode 100644 index 00000000000..c14a4811752 --- /dev/null +++ b/Tests/SwiftSyntaxTest/Inputs/diagnostics.swift @@ -0,0 +1,11 @@ +func foo() { + +} + +func bar() { + +} + +func baz() { + +} From a2ed7fbcbc1d0ae14462e103315495c4c5cabdc8 Mon Sep 17 00:00:00 2001 From: Harlan Date: Thu, 3 May 2018 14:40:14 -0400 Subject: [PATCH 10/25] Add descriptions for SwiftSyntax errors (#16339) --- Sources/SwiftSyntax/SwiftSyntax.swift | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSyntax/SwiftSyntax.swift b/Sources/SwiftSyntax/SwiftSyntax.swift index a1c9f0fca0d..b64b6e89a71 100644 --- a/Sources/SwiftSyntax/SwiftSyntax.swift +++ b/Sources/SwiftSyntax/SwiftSyntax.swift @@ -22,9 +22,23 @@ import Glibc /// A list of possible errors that could be encountered while parsing a /// Syntax tree. -public enum ParserError: Error { +public enum ParserError: Error, CustomStringConvertible { case swiftcFailed(Int, String) case invalidFile + + public var description: String { + switch self{ + case let .swiftcFailed(exitCode, stderr): + let stderrLines = stderr.split(separator: "\n") + return """ + swiftc returned with non-zero exit code \(exitCode) + stderr: + \(stderrLines.joined(separator: "\n ")) + """ + case .invalidFile: + return "swiftc created an invalid syntax file" + } + } } extension Syntax { From 54ea83901612f13b3f656eff11b3759300216352 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Thu, 3 May 2018 12:12:02 -0700 Subject: [PATCH 11/25] SwiftSyntax: Allow absolute position access for dangling nodes. Since absolute position is defined by accumulating proceeding nodes, we should allow its access for nodes without SourceFileSyntax as root. --- Sources/SwiftSyntax/RawSyntax.swift | 9 --------- Sources/SwiftSyntax/SyntaxData.swift | 2 -- 2 files changed, 11 deletions(-) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index dd3c3a46ad7..412ecba6364 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -257,13 +257,4 @@ extension RawSyntax { piece.accumulateAbsolutePosition(pos) } } - - var isSourceFile: Bool { - switch self { - case .node(let kind, _, _): - return kind == SyntaxKind.sourceFile - default: - return false - } - } } diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 554503888b7..5d2e335698b 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -43,7 +43,6 @@ final class SyntaxData: Equatable { fileprivate func calculatePosition(_ initPos: AbsolutePosition) -> AbsolutePosition { guard let parent = parent else { - assert(raw.isSourceFile, "cannot find SourceFileSyntax as root") // If this node is SourceFileSyntax, its location is the start of the file. return initPos } @@ -78,7 +77,6 @@ final class SyntaxData: Equatable { // If this node is root, the position of the next sibling is the end of // this node. guard let parent = parent else { - assert(raw.isSourceFile, "cannot find SourceFileSyntax as root") let result = self.position.copy() raw.accumulateAbsolutePosition(result) return result From 08fb8a0e966f70509c91e980f4dc498f24f56b1a Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 24 May 2018 14:22:49 -0700 Subject: [PATCH 12/25] Add incremental syntax tree deserialization to SwiftSyntax --- Sources/SwiftSyntax/RawSyntax.swift | 136 +++++++++++++++---- Sources/SwiftSyntax/SwiftSyntax.swift | 79 ++++++----- Sources/SwiftSyntax/Syntax.swift | 21 +-- Sources/SwiftSyntax/SyntaxBuilders.swift.gyb | 4 +- Sources/SwiftSyntax/SyntaxFactory.swift.gyb | 11 +- Sources/SwiftSyntax/SyntaxNodes.swift.gyb | 2 +- 6 files changed, 178 insertions(+), 75 deletions(-) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index 412ecba6364..453c5082330 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -12,45 +12,84 @@ import Foundation +extension CodingUserInfoKey { + /// Callback that will be called whenever a `RawSyntax` node is decoded + /// Value must have signature `(RawSyntax) -> Void` + static let rawSyntaxDecodedCallback = + CodingUserInfoKey(rawValue: "SwiftSyntax.RawSyntax.DecodedCallback")! + /// Function that shall be used to look up nodes that were omitted in the + /// syntax tree transfer. + /// Value must have signature `(SyntaxNodeId) -> RawSyntax` + static let omittedNodeLookupFunction = + CodingUserInfoKey(rawValue: "SwiftSyntax.RawSyntax.OmittedNodeLookup")! +} + +/// A ID that uniquely identifies a syntax node and stays stable across multiple +/// incremental parses +struct SyntaxNodeId: Hashable, Codable { + private let rawValue: UInt + + fileprivate init(rawValue: UInt) { + self.rawValue = rawValue + } + + init(from decoder: Decoder) throws { + self.rawValue = try decoder.singleValueContainer().decode(UInt.self) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(rawValue) + } +} + /// Represents the raw tree structure underlying the syntax tree. These nodes /// have no notion of identity and only provide structure to the tree. They /// are immutable and can be freely shared between syntax nodes. indirect enum RawSyntax: Codable { /// A tree node with a kind, an array of children, and a source presence. - case node(SyntaxKind, [RawSyntax?], SourcePresence) + case node(SyntaxKind, [RawSyntax?], SourcePresence, SyntaxNodeId?) /// A token with a token kind, leading trivia, trailing trivia, and a source /// presence. - case token(TokenKind, Trivia, Trivia, SourcePresence) + case token(TokenKind, Trivia, Trivia, SourcePresence, SyntaxNodeId?) /// The syntax kind of this raw syntax. var kind: SyntaxKind { switch self { - case .node(let kind, _, _): return kind - case .token(_, _, _, _): return .token + case .node(let kind, _, _, _): return kind + case .token(_, _, _, _, _): return .token } } var tokenKind: TokenKind? { switch self { - case .node(_, _, _): return nil - case .token(let kind, _, _, _): return kind + case .node(_, _, _, _): return nil + case .token(let kind, _, _, _, _): return kind } } /// The layout of the children of this Raw syntax node. var layout: [RawSyntax?] { switch self { - case .node(_, let layout, _): return layout - case .token(_, _, _, _): return [] + case .node(_, let layout, _, _): return layout + case .token(_, _, _, _, _): return [] } } /// The source presence of this node. var presence: SourcePresence { switch self { - case .node(_, _, let presence): return presence - case .token(_, _, _, let presence): return presence + case .node(_, _, let presence, _): return presence + case .token(_, _, _, let presence, _): return presence + } + } + + /// The ID of this node + var id: SyntaxNodeId? { + switch self { + case .node(_, _, _, let id): return id + case .token(_, _, _, _, let id): return id } } @@ -66,6 +105,9 @@ indirect enum RawSyntax: Codable { /// Keys for serializing RawSyntax nodes. enum CodingKeys: String, CodingKey { + // Shared keys + case id, omitted + // Keys for the `node` case case kind, layout, presence @@ -73,18 +115,53 @@ indirect enum RawSyntax: Codable { case tokenKind, leadingTrivia, trailingTrivia } + public enum IncrementalDecodingError: Error { + /// The JSON to decode is invalid because a node had the `omitted` flag set + /// but included no id + case omittedNodeHasNoId + /// An omitted node was encountered but no node lookup function was + /// in the decoder's `userInfo` for the `.omittedNodeLookupFunction` key + /// or the lookup function had the wrong type + case noLookupFunctionPassed + /// A node with the given ID was omitted from the tranferred syntax tree + /// but the lookup function was unable to look it up + case nodeLookupFailed(SyntaxNodeId) + } + /// Creates a RawSyntax from the provided Foundation Decoder. init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) + let id = try container.decodeIfPresent(SyntaxNodeId.self, forKey: .id) + let omitted = try container.decodeIfPresent(Bool.self, forKey: .omitted) ?? false + + if omitted { + guard let id = id else { + throw IncrementalDecodingError.omittedNodeHasNoId + } + guard let lookupFunc = decoder.userInfo[.omittedNodeLookupFunction] as? + (SyntaxNodeId) -> RawSyntax? else { + throw IncrementalDecodingError.noLookupFunctionPassed + } + guard let lookupNode = lookupFunc(id) else { + throw IncrementalDecodingError.nodeLookupFailed(id) + } + self = lookupNode + return + } + let presence = try container.decode(SourcePresence.self, forKey: .presence) if let kind = try container.decodeIfPresent(SyntaxKind.self, forKey: .kind) { let layout = try container.decode([RawSyntax?].self, forKey: .layout) - self = .node(kind, layout, presence) + self = .node(kind, layout, presence, id) } else { let kind = try container.decode(TokenKind.self, forKey: .tokenKind) let leadingTrivia = try container.decode(Trivia.self, forKey: .leadingTrivia) let trailingTrivia = try container.decode(Trivia.self, forKey: .trailingTrivia) - self = .token(kind, leadingTrivia, trailingTrivia, presence) + self = .token(kind, leadingTrivia, trailingTrivia, presence, id) + } + if let callback = decoder.userInfo[.rawSyntaxDecodedCallback] as? + (RawSyntax) -> Void { + callback(self) } } @@ -92,11 +169,17 @@ indirect enum RawSyntax: Codable { func encode(to encoder: Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) switch self { - case let .node(kind, layout, presence): + case let .node(kind, layout, presence, id): + if let id = id { + try container.encode(id, forKey: .id) + } try container.encode(kind, forKey: .kind) try container.encode(layout, forKey: .layout) try container.encode(presence, forKey: .presence) - case let .token(kind, leadingTrivia, trailingTrivia, presence): + case let .token(kind, leadingTrivia, trailingTrivia, presence, id): + if let id = id { + try container.encode(id, forKey: .id) + } try container.encode(kind, forKey: .tokenKind) try container.encode(leadingTrivia, forKey: .leadingTrivia) try container.encode(trailingTrivia, forKey: .trailingTrivia) @@ -112,7 +195,7 @@ indirect enum RawSyntax: Codable { /// - Returns: A new RawSyntax `.node` with the provided kind and layout, with /// `.missing` source presence. static func missing(_ kind: SyntaxKind) -> RawSyntax { - return .node(kind, [], .missing) + return .node(kind, [], .missing, nil) } /// Creates a RawSyntax token that's marked missing in the source with the @@ -121,7 +204,7 @@ indirect enum RawSyntax: Codable { /// - Returns: A new RawSyntax `.token` with the provided kind, no /// leading/trailing trivia, and `.missing` source presence. static func missingToken(_ kind: TokenKind) -> RawSyntax { - return .token(kind, [], [], .missing) + return .token(kind, [], [], .missing, nil) } /// Returns a new RawSyntax node with the provided layout instead of the @@ -131,8 +214,9 @@ indirect enum RawSyntax: Codable { /// - Parameter newLayout: The children of the new node you're creating. func replacingLayout(_ newLayout: [RawSyntax?]) -> RawSyntax { switch self { - case let .node(kind, _, presence): return .node(kind, newLayout, presence) - case .token(_, _, _, _): return self + case let .node(kind, _, presence, _): + return .node(kind, newLayout, presence, nil) + case .token(_, _, _, _, _): return self } } @@ -176,12 +260,12 @@ extension RawSyntax: TextOutputStreamable { func write(to target: inout Target) where Target: TextOutputStream { switch self { - case .node(_, let layout, _): + case .node(_, let layout, _, _): for child in layout { guard let child = child else { continue } child.write(to: &target) } - case let .token(kind, leadingTrivia, trailingTrivia, presence): + case let .token(kind, leadingTrivia, trailingTrivia, presence, _): guard case .present = presence else { return } for piece in leadingTrivia { piece.write(to: &target) @@ -197,12 +281,12 @@ extension RawSyntax: TextOutputStreamable { extension RawSyntax { func accumulateAbsolutePosition(_ pos: AbsolutePosition) { switch self { - case .node(_, let layout, _): + case .node(_, let layout, _, _): for child in layout { guard let child = child else { continue } child.accumulateAbsolutePosition(pos) } - case let .token(kind, leadingTrivia, trailingTrivia, presence): + case let .token(kind, leadingTrivia, trailingTrivia, presence, _): guard case .present = presence else { return } for piece in leadingTrivia { piece.accumulateAbsolutePosition(pos) @@ -216,14 +300,14 @@ extension RawSyntax { var leadingTrivia: Trivia? { switch self { - case .node(_, let layout, _): + case .node(_, let layout, _, _): for child in layout { guard let child = child else { continue } guard let result = child.leadingTrivia else { continue } return result } return nil - case let .token(_, leadingTrivia, _, presence): + case let .token(_, leadingTrivia, _, presence, _): guard case .present = presence else { return nil } return leadingTrivia } @@ -231,14 +315,14 @@ extension RawSyntax { var trailingTrivia: Trivia? { switch self { - case .node(_, let layout, _): + case .node(_, let layout, _, _): for child in layout.reversed() { guard let child = child else { continue } guard let result = child.trailingTrivia else { continue } return result } return nil - case let .token(_, _, trailingTrivia, presence): + case let .token(_, _, trailingTrivia, presence, _): guard case .present = presence else { return nil } return trailingTrivia } diff --git a/Sources/SwiftSyntax/SwiftSyntax.swift b/Sources/SwiftSyntax/SwiftSyntax.swift index b64b6e89a71..803790bb86c 100644 --- a/Sources/SwiftSyntax/SwiftSyntax.swift +++ b/Sources/SwiftSyntax/SwiftSyntax.swift @@ -41,51 +41,66 @@ public enum ParserError: Error, CustomStringConvertible { } } -extension Syntax { - fileprivate static func encodeSourceFileSyntaxInternal(_ url: URL) throws -> Data { - let swiftcRunner = try SwiftcRunner(sourceFile: url) - let result = try swiftcRunner.invoke() - return result.stdoutData - } +/// Deserializes the syntax tree from its serialized form to an object tree in +/// Swift. To deserialize incrementally transferred syntax trees, the same +/// instance of the deserializer must be used for all subsequent +/// deserializations. +public final class SyntaxTreeDeserializer { + // FIXME: This lookup table just accumulates nodes, we should invalidate nodes + // that are no longer used at some point and remove them from the table + /// Syntax nodes that have already been parsed and are able to be reused if + /// they were omitted in an incremental syntax tree transfer + private var nodeLookupTable: [SyntaxNodeId: RawSyntax] = [:] - /// Parses the Swift file at the provided URL into a `Syntax` tree in Json - /// serialization format. - /// - Parameter url: The URL you wish to parse. - /// - Returns: The syntax tree in Json format string. - public static func encodeSourceFileSyntax(_ url: URL) throws -> String { - return String(data: try encodeSourceFileSyntaxInternal(url), encoding: .utf8)! - } - - /// Parses the Swift file at the provided URL into a full-fidelity `Syntax` - /// tree. - /// - Parameter url: The URL you wish to parse. - /// - Returns: A top-level Syntax node representing the contents of the tree, - /// if the parse was successful. - /// - Throws: `ParseError.couldNotFindSwiftc` if `swiftc` could not be - /// located, `ParseError.invalidFile` if the file is invalid. - /// FIXME: Fill this out with all error cases. - public static func parse(_ url: URL) throws -> SourceFileSyntax { - return try decodeSourceFileSyntax(encodeSourceFileSyntaxInternal(url)) + public init() { } - /// Decode a serialized form of SourceFileSyntax to a syntax node. - /// - Parameter content: The data of the serialized SourceFileSyntax. + /// Decode a serialized form of SourceFileSyntax to a syntax tree. + /// - Parameter data: The UTF-8 represenation of the serialized syntax tree /// - Returns: A top-level Syntax node representing the contents of the tree, /// if the parse was successful. - fileprivate static func decodeSourceFileSyntax(_ content: Data) throws -> SourceFileSyntax { + public func deserialize(_ data: Data) throws -> SourceFileSyntax { let decoder = JSONDecoder() - let raw = try decoder.decode(RawSyntax.self, from: content) + decoder.userInfo[.rawSyntaxDecodedCallback] = self.addToLookupTable + decoder.userInfo[.omittedNodeLookupFunction] = self.lookupNode + let raw = try decoder.decode(RawSyntax.self, from: data) guard let file = makeSyntax(raw) as? SourceFileSyntax else { throw ParserError.invalidFile } return file } - /// Decode a serialized form of SourceFileSyntax to a syntax node. - /// - Parameter content: The string content of the serialized SourceFileSyntax. + // MARK: Incremental deserialization helper functions + private func lookupNode(id: SyntaxNodeId) -> RawSyntax? { + return nodeLookupTable[id] + } + + private func addToLookupTable(_ node: RawSyntax) { + guard let id = node.id else { + return + } + nodeLookupTable[id] = node + } +} + +/// Namespace for functions to retrieve a syntax tree from the swift compiler +/// and deserializing it. +public enum SyntaxTreeParser { + /// Parses the Swift file at the provided URL into a full-fidelity Syntax tree + /// - Parameter url: The URL you wish to parse. /// - Returns: A top-level Syntax node representing the contents of the tree, /// if the parse was successful. - public static func decodeSourceFileSyntax(_ content: String) throws -> SourceFileSyntax { - return try decodeSourceFileSyntax(content.data(using: .utf8)!) + /// - Throws: `ParseError.couldNotFindSwiftc` if `swiftc` could not be + /// located, `ParseError.invalidFile` if the file is invalid. + /// FIXME: Fill this out with all error cases. + public static func parse(_ url: URL) throws -> SourceFileSyntax { + let swiftcRunner = try SwiftcRunner(sourceFile: url) + let result = try swiftcRunner.invoke() + guard result.wasSuccessful else { + throw ParserError.swiftcFailed(result.exitCode, result.stderr) + } + let syntaxTreeData = result.stdoutData + let deserializer = SyntaxTreeDeserializer() + return try deserializer.deserialize(syntaxTreeData) } } diff --git a/Sources/SwiftSyntax/Syntax.swift b/Sources/SwiftSyntax/Syntax.swift index eae8d881d21..29b2fbf6539 100644 --- a/Sources/SwiftSyntax/Syntax.swift +++ b/Sources/SwiftSyntax/Syntax.swift @@ -283,33 +283,36 @@ public struct TokenSyntax: _SyntaxBase, Hashable { /// Returns a new TokenSyntax with its kind replaced /// by the provided token kind. public func withKind(_ tokenKind: TokenKind) -> TokenSyntax { - guard case let .token(_, leadingTrivia, trailingTrivia, presence) = raw else { + guard case let .token(_, leadingTrivia, trailingTrivia, presence, _) = raw else { fatalError("TokenSyntax must have token as its raw") } let (root, newData) = data.replacingSelf(.token(tokenKind, leadingTrivia, - trailingTrivia, presence)) + trailingTrivia, presence, + nil)) return TokenSyntax(root: root, data: newData) } /// Returns a new TokenSyntax with its leading trivia replaced /// by the provided trivia. public func withLeadingTrivia(_ leadingTrivia: Trivia) -> TokenSyntax { - guard case let .token(kind, _, trailingTrivia, presence) = raw else { + guard case let .token(kind, _, trailingTrivia, presence, _) = raw else { fatalError("TokenSyntax must have token as its raw") } let (root, newData) = data.replacingSelf(.token(kind, leadingTrivia, - trailingTrivia, presence)) + trailingTrivia, presence, + nil)) return TokenSyntax(root: root, data: newData) } /// Returns a new TokenSyntax with its trailing trivia replaced /// by the provided trivia. public func withTrailingTrivia(_ trailingTrivia: Trivia) -> TokenSyntax { - guard case let .token(kind, leadingTrivia, _, presence) = raw else { + guard case let .token(kind, leadingTrivia, _, presence, _) = raw else { fatalError("TokenSyntax must have token as its raw") } let (root, newData) = data.replacingSelf(.token(kind, leadingTrivia, - trailingTrivia, presence)) + trailingTrivia, presence, + nil)) return TokenSyntax(root: root, data: newData) } @@ -330,7 +333,7 @@ public struct TokenSyntax: _SyntaxBase, Hashable { /// The leading trivia (spaces, newlines, etc.) associated with this token. public var leadingTrivia: Trivia { - guard case .token(_, let leadingTrivia, _, _) = raw else { + guard case .token(_, let leadingTrivia, _, _, _) = raw else { fatalError("TokenSyntax must have token as its raw") } return leadingTrivia @@ -338,7 +341,7 @@ public struct TokenSyntax: _SyntaxBase, Hashable { /// The trailing trivia (spaces, newlines, etc.) associated with this token. public var trailingTrivia: Trivia { - guard case .token(_, _, let trailingTrivia, _) = raw else { + guard case .token(_, _, let trailingTrivia, _, _) = raw else { fatalError("TokenSyntax must have token as its raw") } return trailingTrivia @@ -346,7 +349,7 @@ public struct TokenSyntax: _SyntaxBase, Hashable { /// The kind of token this node represents. public var tokenKind: TokenKind { - guard case .token(let kind, _, _, _) = raw else { + guard case .token(let kind, _, _, _, _) = raw else { fatalError("TokenSyntax must have token as its raw") } return kind diff --git a/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb b/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb index b5da75abae7..513e652dd79 100644 --- a/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb @@ -47,7 +47,7 @@ public struct ${Builder} { layout[idx] = list.appending(elt.raw) } else { layout[idx] = RawSyntax.node( - .${child.swift_syntax_kind}, [elt.raw], .present) + .${child.swift_syntax_kind}, [elt.raw], .present, nil) } } % else: @@ -69,7 +69,7 @@ public struct ${Builder} { % end return SyntaxData(raw: .node(.${node.swift_syntax_kind}, - layout, .present)) + layout, .present, nil)) } } diff --git a/Sources/SwiftSyntax/SyntaxFactory.swift.gyb b/Sources/SwiftSyntax/SyntaxFactory.swift.gyb index b4a82ef67be..ace02f2eba3 100644 --- a/Sources/SwiftSyntax/SyntaxFactory.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxFactory.swift.gyb @@ -32,14 +32,14 @@ public enum SyntaxFactory { leadingTrivia: Trivia = [], trailingTrivia: Trivia = []) -> TokenSyntax { let data = SyntaxData(raw: .token(kind, leadingTrivia, - trailingTrivia, presence)) + trailingTrivia, presence, nil)) return TokenSyntax(root: data, data: data) } public static func makeUnknownSyntax(tokens: [TokenSyntax]) -> Syntax { let data = SyntaxData(raw: .node(.unknown, tokens.map { $0.data.raw }, - .present)) + .present, nil)) return UnknownSyntax(root: data, data: data) } @@ -65,14 +65,15 @@ public enum SyntaxFactory { ${child.swift_name}.data.raw, % end % end - ], .present)) + ], .present, nil)) return ${node.name}(root: data, data: data) } % elif node.is_syntax_collection(): public static func make${node.syntax_kind}( _ elements: [${node.collection_element_type}]) -> ${node.name} { let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, - elements.map { $0.data.raw }, .present)) + elements.map { $0.data.raw }, .present, + nil)) return ${node.name}(root: data, data: data) } % end @@ -83,7 +84,7 @@ public enum SyntaxFactory { % for child in node.children: ${make_missing_swift_child(child)}, % end - ], .present)) + ], .present, nil)) return ${node.name}(root: data, data: data) } % end diff --git a/Sources/SwiftSyntax/SyntaxNodes.swift.gyb b/Sources/SwiftSyntax/SyntaxNodes.swift.gyb index 2d577189c52..e22afa8545f 100644 --- a/Sources/SwiftSyntax/SyntaxNodes.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxNodes.swift.gyb @@ -161,7 +161,7 @@ public struct ${node.name}: ${base_type}, _SyntaxBase, Hashable { collection = col.appending(element.raw) } else { collection = RawSyntax.node(SyntaxKind.${child_node.swift_syntax_kind}, - [element.raw], .present) + [element.raw], .present, nil) } let (root, newData) = data.replacingChild(collection, at: Cursor.${child.swift_name}) From 7deb6b43e3b10bebd0cb92a194b1e07ce8eb4652 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 24 May 2018 14:22:49 -0700 Subject: [PATCH 13/25] Make RawSyntax a struct SourcePresence and ID are already shared between the two node kinds and the node's length will soon be cached in RawSyntax as well. By making it a struct, we will be able to compute the node's length when it is being constructed in the initialiser. --- Sources/SwiftSyntax/RawSyntax.swift | 140 ++++++++++--------- Sources/SwiftSyntax/SwiftSyntax.swift | 5 +- Sources/SwiftSyntax/Syntax.swift | 40 +++--- Sources/SwiftSyntax/SyntaxBuilders.swift.gyb | 8 +- Sources/SwiftSyntax/SyntaxFactory.swift.gyb | 28 ++-- Sources/SwiftSyntax/SyntaxNodes.swift.gyb | 4 +- 6 files changed, 119 insertions(+), 106 deletions(-) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index 453c5082330..748de9900a9 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -29,6 +29,16 @@ extension CodingUserInfoKey { struct SyntaxNodeId: Hashable, Codable { private let rawValue: UInt + // Start creating fresh node IDs for user generated nodes on in the upper + // half of the UInt value range so that they don't easily collide with node + // ids generated by the C++ side of libSyntax + private static var highestUsedId: UInt = UInt.max / 2 + + /// Generates a syntax node ID that has not been used yet + fileprivate static func generateFreshId() -> SyntaxNodeId { + return SyntaxNodeId(rawValue: highestUsedId + 1) + } + fileprivate init(rawValue: UInt) { self.rawValue = rawValue } @@ -43,53 +53,57 @@ struct SyntaxNodeId: Hashable, Codable { } } +/// The data that is specific to a tree or token node +fileprivate indirect enum RawSyntaxData { + /// A tree node with a kind and an array of children + case node(kind: SyntaxKind, layout: [RawSyntax?]) + /// A token with a token kind, leading trivia, and trailing trivia + case token(kind: TokenKind, leadingTrivia: Trivia, trailingTrivia: Trivia) +} + /// Represents the raw tree structure underlying the syntax tree. These nodes /// have no notion of identity and only provide structure to the tree. They /// are immutable and can be freely shared between syntax nodes. -indirect enum RawSyntax: Codable { - /// A tree node with a kind, an array of children, and a source presence. - case node(SyntaxKind, [RawSyntax?], SourcePresence, SyntaxNodeId?) +struct RawSyntax: Codable { + fileprivate let data: RawSyntaxData + let presence: SourcePresence + let id: SyntaxNodeId + + init(kind: SyntaxKind, layout: [RawSyntax?], presence: SourcePresence, + id: SyntaxNodeId? = nil) { + self.data = .node(kind: kind, layout: layout) + self.presence = presence + self.id = id ?? SyntaxNodeId.generateFreshId() + } - /// A token with a token kind, leading trivia, trailing trivia, and a source - /// presence. - case token(TokenKind, Trivia, Trivia, SourcePresence, SyntaxNodeId?) + init(kind: TokenKind, leadingTrivia: Trivia, trailingTrivia: Trivia, + presence: SourcePresence, id: SyntaxNodeId? = nil) { + self.data = .token(kind: kind, leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia) + self.presence = presence + self.id = id ?? SyntaxNodeId.generateFreshId() + } /// The syntax kind of this raw syntax. var kind: SyntaxKind { - switch self { - case .node(let kind, _, _, _): return kind - case .token(_, _, _, _, _): return .token + switch data { + case .node(let kind, _): return kind + case .token(_, _, _): return .token } } var tokenKind: TokenKind? { - switch self { - case .node(_, _, _, _): return nil - case .token(let kind, _, _, _, _): return kind + switch data { + case .node(_, _): return nil + case .token(let kind, _, _): return kind } } /// The layout of the children of this Raw syntax node. var layout: [RawSyntax?] { - switch self { - case .node(_, let layout, _, _): return layout - case .token(_, _, _, _, _): return [] - } - } - - /// The source presence of this node. - var presence: SourcePresence { - switch self { - case .node(_, _, let presence, _): return presence - case .token(_, _, _, let presence, _): return presence - } - } - - /// The ID of this node - var id: SyntaxNodeId? { - switch self { - case .node(_, _, _, let id): return id - case .token(_, _, _, _, let id): return id + switch data { + case .node(_, let layout): return layout + case .token(_, _, _): return [] } } @@ -152,12 +166,13 @@ indirect enum RawSyntax: Codable { let presence = try container.decode(SourcePresence.self, forKey: .presence) if let kind = try container.decodeIfPresent(SyntaxKind.self, forKey: .kind) { let layout = try container.decode([RawSyntax?].self, forKey: .layout) - self = .node(kind, layout, presence, id) + self.init(kind: kind, layout: layout, presence: presence, id: id) } else { let kind = try container.decode(TokenKind.self, forKey: .tokenKind) let leadingTrivia = try container.decode(Trivia.self, forKey: .leadingTrivia) let trailingTrivia = try container.decode(Trivia.self, forKey: .trailingTrivia) - self = .token(kind, leadingTrivia, trailingTrivia, presence, id) + self.init(kind: kind, leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia, presence: presence, id: id) } if let callback = decoder.userInfo[.rawSyntaxDecodedCallback] as? (RawSyntax) -> Void { @@ -168,23 +183,18 @@ indirect enum RawSyntax: Codable { /// Encodes the RawSyntax to the provided Foundation Encoder. func encode(to encoder: Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) - switch self { - case let .node(kind, layout, presence, id): - if let id = id { - try container.encode(id, forKey: .id) - } + switch self.data { + case let .node(kind, layout): + try container.encode(id, forKey: .id) try container.encode(kind, forKey: .kind) try container.encode(layout, forKey: .layout) - try container.encode(presence, forKey: .presence) - case let .token(kind, leadingTrivia, trailingTrivia, presence, id): - if let id = id { - try container.encode(id, forKey: .id) - } + case let .token(kind, leadingTrivia, trailingTrivia): + try container.encode(id, forKey: .id) try container.encode(kind, forKey: .tokenKind) try container.encode(leadingTrivia, forKey: .leadingTrivia) try container.encode(trailingTrivia, forKey: .trailingTrivia) - try container.encode(presence, forKey: .presence) } + try container.encode(presence, forKey: .presence) } /// Creates a RawSyntax node that's marked missing in the source with the @@ -195,7 +205,7 @@ indirect enum RawSyntax: Codable { /// - Returns: A new RawSyntax `.node` with the provided kind and layout, with /// `.missing` source presence. static func missing(_ kind: SyntaxKind) -> RawSyntax { - return .node(kind, [], .missing, nil) + return RawSyntax(kind: kind, layout: [], presence: .missing) } /// Creates a RawSyntax token that's marked missing in the source with the @@ -204,7 +214,7 @@ indirect enum RawSyntax: Codable { /// - Returns: A new RawSyntax `.token` with the provided kind, no /// leading/trailing trivia, and `.missing` source presence. static func missingToken(_ kind: TokenKind) -> RawSyntax { - return .token(kind, [], [], .missing, nil) + return RawSyntax(kind: kind, leadingTrivia: [], trailingTrivia: [], presence: .missing) } /// Returns a new RawSyntax node with the provided layout instead of the @@ -213,10 +223,10 @@ indirect enum RawSyntax: Codable { /// is returned. /// - Parameter newLayout: The children of the new node you're creating. func replacingLayout(_ newLayout: [RawSyntax?]) -> RawSyntax { - switch self { - case let .node(kind, _, presence, _): - return .node(kind, newLayout, presence, nil) - case .token(_, _, _, _, _): return self + switch data { + case let .node(kind, _): + return RawSyntax(kind: kind, layout: newLayout, presence: presence) + case .token(_, _, _): return self } } @@ -259,14 +269,14 @@ extension RawSyntax: TextOutputStreamable { /// - Parameter stream: The stream on which to output this node. func write(to target: inout Target) where Target: TextOutputStream { - switch self { - case .node(_, let layout, _, _): + switch data { + case .node(_, let layout): for child in layout { guard let child = child else { continue } child.write(to: &target) } - case let .token(kind, leadingTrivia, trailingTrivia, presence, _): - guard case .present = presence else { return } + case let .token(kind, leadingTrivia, trailingTrivia): + guard isPresent else { return } for piece in leadingTrivia { piece.write(to: &target) } @@ -280,14 +290,14 @@ extension RawSyntax: TextOutputStreamable { extension RawSyntax { func accumulateAbsolutePosition(_ pos: AbsolutePosition) { - switch self { - case .node(_, let layout, _, _): + switch data { + case .node(_, let layout): for child in layout { guard let child = child else { continue } child.accumulateAbsolutePosition(pos) } - case let .token(kind, leadingTrivia, trailingTrivia, presence, _): - guard case .present = presence else { return } + case let .token(kind, leadingTrivia, trailingTrivia): + guard isPresent else { return } for piece in leadingTrivia { piece.accumulateAbsolutePosition(pos) } @@ -299,31 +309,29 @@ extension RawSyntax { } var leadingTrivia: Trivia? { - switch self { - case .node(_, let layout, _, _): + switch data { + case .node(_, let layout): for child in layout { guard let child = child else { continue } guard let result = child.leadingTrivia else { continue } return result } return nil - case let .token(_, leadingTrivia, _, presence, _): - guard case .present = presence else { return nil } + case let .token(_, leadingTrivia, _): return leadingTrivia } } var trailingTrivia: Trivia? { - switch self { - case .node(_, let layout, _, _): + switch data { + case .node(_, let layout): for child in layout.reversed() { guard let child = child else { continue } guard let result = child.trailingTrivia else { continue } return result } return nil - case let .token(_, _, trailingTrivia, presence, _): - guard case .present = presence else { return nil } + case let .token(_, _, trailingTrivia): return trailingTrivia } } diff --git a/Sources/SwiftSyntax/SwiftSyntax.swift b/Sources/SwiftSyntax/SwiftSyntax.swift index 803790bb86c..22659582e3d 100644 --- a/Sources/SwiftSyntax/SwiftSyntax.swift +++ b/Sources/SwiftSyntax/SwiftSyntax.swift @@ -76,10 +76,7 @@ public final class SyntaxTreeDeserializer { } private func addToLookupTable(_ node: RawSyntax) { - guard let id = node.id else { - return - } - nodeLookupTable[id] = node + nodeLookupTable[node.id] = node } } diff --git a/Sources/SwiftSyntax/Syntax.swift b/Sources/SwiftSyntax/Syntax.swift index 29b2fbf6539..ffdca1d2ac1 100644 --- a/Sources/SwiftSyntax/Syntax.swift +++ b/Sources/SwiftSyntax/Syntax.swift @@ -283,36 +283,40 @@ public struct TokenSyntax: _SyntaxBase, Hashable { /// Returns a new TokenSyntax with its kind replaced /// by the provided token kind. public func withKind(_ tokenKind: TokenKind) -> TokenSyntax { - guard case let .token(_, leadingTrivia, trailingTrivia, presence, _) = raw else { + guard raw.kind == .token else { fatalError("TokenSyntax must have token as its raw") } - let (root, newData) = data.replacingSelf(.token(tokenKind, leadingTrivia, - trailingTrivia, presence, - nil)) + let newRaw = RawSyntax(kind: tokenKind, leadingTrivia: raw.leadingTrivia!, + trailingTrivia: raw.trailingTrivia!, + presence: raw.presence) + let (root, newData) = data.replacingSelf(newRaw) return TokenSyntax(root: root, data: newData) } /// Returns a new TokenSyntax with its leading trivia replaced /// by the provided trivia. public func withLeadingTrivia(_ leadingTrivia: Trivia) -> TokenSyntax { - guard case let .token(kind, _, trailingTrivia, presence, _) = raw else { + guard raw.kind == .token else { fatalError("TokenSyntax must have token as its raw") } - let (root, newData) = data.replacingSelf(.token(kind, leadingTrivia, - trailingTrivia, presence, - nil)) + let newRaw = RawSyntax(kind: raw.tokenKind!, leadingTrivia: leadingTrivia, + trailingTrivia: raw.trailingTrivia!, + presence: raw.presence) + let (root, newData) = data.replacingSelf(newRaw) return TokenSyntax(root: root, data: newData) } /// Returns a new TokenSyntax with its trailing trivia replaced /// by the provided trivia. public func withTrailingTrivia(_ trailingTrivia: Trivia) -> TokenSyntax { - guard case let .token(kind, leadingTrivia, _, presence, _) = raw else { + guard raw.kind == .token else { fatalError("TokenSyntax must have token as its raw") } - let (root, newData) = data.replacingSelf(.token(kind, leadingTrivia, - trailingTrivia, presence, - nil)) + let newRaw = RawSyntax(kind: raw.tokenKind!, + leadingTrivia: raw.leadingTrivia!, + trailingTrivia: trailingTrivia, + presence: raw.presence) + let (root, newData) = data.replacingSelf(newRaw) return TokenSyntax(root: root, data: newData) } @@ -333,26 +337,26 @@ public struct TokenSyntax: _SyntaxBase, Hashable { /// The leading trivia (spaces, newlines, etc.) associated with this token. public var leadingTrivia: Trivia { - guard case .token(_, let leadingTrivia, _, _, _) = raw else { + guard raw.kind == .token else { fatalError("TokenSyntax must have token as its raw") } - return leadingTrivia + return raw.leadingTrivia! } /// The trailing trivia (spaces, newlines, etc.) associated with this token. public var trailingTrivia: Trivia { - guard case .token(_, _, let trailingTrivia, _, _) = raw else { + guard raw.kind == .token else { fatalError("TokenSyntax must have token as its raw") } - return trailingTrivia + return raw.trailingTrivia! } /// The kind of token this node represents. public var tokenKind: TokenKind { - guard case .token(let kind, _, _, _, _) = raw else { + guard raw.kind == .token else { fatalError("TokenSyntax must have token as its raw") } - return kind + return raw.tokenKind! } public static func ==(lhs: TokenSyntax, rhs: TokenSyntax) -> Bool { diff --git a/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb b/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb index 513e652dd79..ecf2baa3e83 100644 --- a/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb @@ -46,8 +46,8 @@ public struct ${Builder} { if let list = layout[idx] { layout[idx] = list.appending(elt.raw) } else { - layout[idx] = RawSyntax.node( - .${child.swift_syntax_kind}, [elt.raw], .present, nil) + layout[idx] = RawSyntax(kind: .${child.swift_syntax_kind}, + layout: [elt.raw], presence: .present) } } % else: @@ -68,8 +68,8 @@ public struct ${Builder} { % end % end - return SyntaxData(raw: .node(.${node.swift_syntax_kind}, - layout, .present, nil)) + return SyntaxData(raw: RawSyntax(kind: .${node.swift_syntax_kind}, + layout: layout, presence: .present)) } } diff --git a/Sources/SwiftSyntax/SyntaxFactory.swift.gyb b/Sources/SwiftSyntax/SyntaxFactory.swift.gyb index ace02f2eba3..ec98d7e9afd 100644 --- a/Sources/SwiftSyntax/SyntaxFactory.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxFactory.swift.gyb @@ -31,15 +31,16 @@ public enum SyntaxFactory { public static func makeToken(_ kind: TokenKind, presence: SourcePresence, leadingTrivia: Trivia = [], trailingTrivia: Trivia = []) -> TokenSyntax { - let data = SyntaxData(raw: .token(kind, leadingTrivia, - trailingTrivia, presence, nil)) + let raw = RawSyntax(kind: kind, leadingTrivia: leadingTrivia, + trailingTrivia: trailingTrivia, presence: presence) + let data = SyntaxData(raw: raw) return TokenSyntax(root: data, data: data) } public static func makeUnknownSyntax(tokens: [TokenSyntax]) -> Syntax { - let data = SyntaxData(raw: .node(.unknown, - tokens.map { $0.data.raw }, - .present, nil)) + let raw = RawSyntax(kind: .unknown, layout: tokens.map { $0.data.raw }, + presence: .present) + let data = SyntaxData(raw: raw) return UnknownSyntax(root: data, data: data) } @@ -57,7 +58,8 @@ public enum SyntaxFactory { % child_params.append("%s: %s" % (child.swift_name, param_type)) % child_params = ', '.join(child_params) public static func make${node.syntax_kind}(${child_params}) -> ${node.name} { - let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, [ + let data = SyntaxData(raw: RawSyntax(kind: .${node.swift_syntax_kind}, + layout: [ % for child in node.children: % if child.is_optional: ${child.swift_name}?.data.raw ?? ${make_missing_swift_child(child)}, @@ -65,26 +67,28 @@ public enum SyntaxFactory { ${child.swift_name}.data.raw, % end % end - ], .present, nil)) + ], presence: .present)) return ${node.name}(root: data, data: data) } % elif node.is_syntax_collection(): public static func make${node.syntax_kind}( _ elements: [${node.collection_element_type}]) -> ${node.name} { - let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, - elements.map { $0.data.raw }, .present, - nil)) + let raw = RawSyntax(kind: .${node.swift_syntax_kind}, + layout: elements.map { $0.data.raw }, + presence: .present) + let data = SyntaxData(raw: raw) return ${node.name}(root: data, data: data) } % end % if not node.is_base(): public static func makeBlank${node.syntax_kind}() -> ${node.name} { - let data = SyntaxData(raw: .node(.${node.swift_syntax_kind}, [ + let data = SyntaxData(raw: RawSyntax(kind: .${node.swift_syntax_kind}, + layout: [ % for child in node.children: ${make_missing_swift_child(child)}, % end - ], .present, nil)) + ], presence: .present)) return ${node.name}(root: data, data: data) } % end diff --git a/Sources/SwiftSyntax/SyntaxNodes.swift.gyb b/Sources/SwiftSyntax/SyntaxNodes.swift.gyb index e22afa8545f..7ea12f108b2 100644 --- a/Sources/SwiftSyntax/SyntaxNodes.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxNodes.swift.gyb @@ -160,8 +160,8 @@ public struct ${node.name}: ${base_type}, _SyntaxBase, Hashable { if let col = raw[Cursor.${child.swift_name}] { collection = col.appending(element.raw) } else { - collection = RawSyntax.node(SyntaxKind.${child_node.swift_syntax_kind}, - [element.raw], .present, nil) + collection = RawSyntax(kind: SyntaxKind.${child_node.swift_syntax_kind}, + layout: [element.raw], presence: .present) } let (root, newData) = data.replacingChild(collection, at: Cursor.${child.swift_name}) From f104c5fd9d6c8ee66f420eaa7ea43791cbddfca8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 26 Jun 2018 15:53:06 -0700 Subject: [PATCH 14/25] Add type annotations to speed up compile time --- Sources/SwiftSyntax/SyntaxBuilders.swift.gyb | 5 +++-- Sources/SwiftSyntax/SyntaxFactory.swift.gyb | 12 +++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb b/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb index ecf2baa3e83..0016fcd9ec4 100644 --- a/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxBuilders.swift.gyb @@ -46,8 +46,9 @@ public struct ${Builder} { if let list = layout[idx] { layout[idx] = list.appending(elt.raw) } else { - layout[idx] = RawSyntax(kind: .${child.swift_syntax_kind}, - layout: [elt.raw], presence: .present) + layout[idx] = RawSyntax(kind: SyntaxKind.${child.swift_syntax_kind}, + layout: [elt.raw], + presence: SourcePresence.present) } } % else: diff --git a/Sources/SwiftSyntax/SyntaxFactory.swift.gyb b/Sources/SwiftSyntax/SyntaxFactory.swift.gyb index ec98d7e9afd..9e999537add 100644 --- a/Sources/SwiftSyntax/SyntaxFactory.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxFactory.swift.gyb @@ -58,8 +58,7 @@ public enum SyntaxFactory { % child_params.append("%s: %s" % (child.swift_name, param_type)) % child_params = ', '.join(child_params) public static func make${node.syntax_kind}(${child_params}) -> ${node.name} { - let data = SyntaxData(raw: RawSyntax(kind: .${node.swift_syntax_kind}, - layout: [ + let layout: [RawSyntax?] = [ % for child in node.children: % if child.is_optional: ${child.swift_name}?.data.raw ?? ${make_missing_swift_child(child)}, @@ -67,15 +66,18 @@ public enum SyntaxFactory { ${child.swift_name}.data.raw, % end % end - ], presence: .present)) + ] + let raw = RawSyntax(kind: SyntaxKind.${node.swift_syntax_kind}, + layout: layout, presence: SourcePresence.present) + let data = SyntaxData(raw: raw) return ${node.name}(root: data, data: data) } % elif node.is_syntax_collection(): public static func make${node.syntax_kind}( _ elements: [${node.collection_element_type}]) -> ${node.name} { - let raw = RawSyntax(kind: .${node.swift_syntax_kind}, + let raw = RawSyntax(kind: SyntaxKind.${node.swift_syntax_kind}, layout: elements.map { $0.data.raw }, - presence: .present) + presence: SourcePresence.present) let data = SyntaxData(raw: raw) return ${node.name}(root: data, data: data) } From 156cc9ff2698954a99daef075e765bbe9d887aaf Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 25 Jul 2018 23:20:27 -0700 Subject: [PATCH 15/25] Don't throw just because compilation fails This was originally "[swiftSyntax] Add test cases for the SyntaxClassifier" (apple/swift-syntax e7c90be3d9fa2a93d917a3597bc4d6cf58ce5533). We don't actually pull in any SyntaxClassifier logic, but this commit does fix a bug where compiler errors cause the entire parse to fail. --- Sources/SwiftSyntax/SwiftSyntax.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Sources/SwiftSyntax/SwiftSyntax.swift b/Sources/SwiftSyntax/SwiftSyntax.swift index 22659582e3d..beec89c9e57 100644 --- a/Sources/SwiftSyntax/SwiftSyntax.swift +++ b/Sources/SwiftSyntax/SwiftSyntax.swift @@ -93,9 +93,6 @@ public enum SyntaxTreeParser { public static func parse(_ url: URL) throws -> SourceFileSyntax { let swiftcRunner = try SwiftcRunner(sourceFile: url) let result = try swiftcRunner.invoke() - guard result.wasSuccessful else { - throw ParserError.swiftcFailed(result.exitCode, result.stderr) - } let syntaxTreeData = result.stdoutData let deserializer = SyntaxTreeDeserializer() return try deserializer.deserialize(syntaxTreeData) From 9bf9210137b116dc11c0338e810aa9ce34d8dad3 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 30 May 2018 14:39:31 -0700 Subject: [PATCH 16/25] Record the nodes that have been reused during an incremental transfer This way we will be able to avoid reclassifying these nodes for syntax highlighting since we know they haven't changed. --- Sources/SwiftSyntax/RawSyntax.swift | 6 +++--- Sources/SwiftSyntax/SwiftSyntax.swift | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index 748de9900a9..c9c432befac 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -26,7 +26,7 @@ extension CodingUserInfoKey { /// A ID that uniquely identifies a syntax node and stays stable across multiple /// incremental parses -struct SyntaxNodeId: Hashable, Codable { +public struct SyntaxNodeId: Hashable, Codable { private let rawValue: UInt // Start creating fresh node IDs for user generated nodes on in the upper @@ -43,11 +43,11 @@ struct SyntaxNodeId: Hashable, Codable { self.rawValue = rawValue } - init(from decoder: Decoder) throws { + public init(from decoder: Decoder) throws { self.rawValue = try decoder.singleValueContainer().decode(UInt.self) } - func encode(to encoder: Encoder) throws { + public func encode(to encoder: Encoder) throws { var container = encoder.singleValueContainer() try container.encode(rawValue) } diff --git a/Sources/SwiftSyntax/SwiftSyntax.swift b/Sources/SwiftSyntax/SwiftSyntax.swift index beec89c9e57..568178eb484 100644 --- a/Sources/SwiftSyntax/SwiftSyntax.swift +++ b/Sources/SwiftSyntax/SwiftSyntax.swift @@ -52,6 +52,10 @@ public final class SyntaxTreeDeserializer { /// they were omitted in an incremental syntax tree transfer private var nodeLookupTable: [SyntaxNodeId: RawSyntax] = [:] + /// The IDs of the nodes that were reused as part of incremental syntax + /// parsing during the last deserialization + public var reusedNodeIds: Set = [] + public init() { } @@ -60,6 +64,7 @@ public final class SyntaxTreeDeserializer { /// - Returns: A top-level Syntax node representing the contents of the tree, /// if the parse was successful. public func deserialize(_ data: Data) throws -> SourceFileSyntax { + reusedNodeIds = [] let decoder = JSONDecoder() decoder.userInfo[.rawSyntaxDecodedCallback] = self.addToLookupTable decoder.userInfo[.omittedNodeLookupFunction] = self.lookupNode @@ -72,6 +77,7 @@ public final class SyntaxTreeDeserializer { // MARK: Incremental deserialization helper functions private func lookupNode(id: SyntaxNodeId) -> RawSyntax? { + reusedNodeIds.insert(id) return nodeLookupTable[id] } From 6294ff3a89272707813ed45460195bdca2361d35 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 27 Jun 2018 17:15:01 -0700 Subject: [PATCH 17/25] Refactor AbsolutePosition AbsolutePosition being a mutable reference type easily leads to bugs where an AbsolutePosition is modified. Making it immutable eliminates this issue. Furthermore, the introduction of SourceLength should allow easier manipulation of AbsolutePositions on the client side. We still cannot make AbsolutePosition a value type since it is used inside AtomicCache, but the immutability gives the same safety. --- Sources/SwiftSyntax/AbsolutePosition.swift | 44 ++-------- Sources/SwiftSyntax/RawSyntax.swift | 75 ++++++++++------- Sources/SwiftSyntax/SourceLength.swift | 93 ++++++++++++++++++++++ Sources/SwiftSyntax/Syntax.swift | 69 ++++++++++++++-- Sources/SwiftSyntax/SyntaxData.swift | 41 +++------- Sources/SwiftSyntax/Trivia.swift.gyb | 28 ++++--- 6 files changed, 234 insertions(+), 116 deletions(-) create mode 100644 Sources/SwiftSyntax/SourceLength.swift diff --git a/Sources/SwiftSyntax/AbsolutePosition.swift b/Sources/SwiftSyntax/AbsolutePosition.swift index 65be3b54d3d..7b0f265bfe7 100644 --- a/Sources/SwiftSyntax/AbsolutePosition.swift +++ b/Sources/SwiftSyntax/AbsolutePosition.swift @@ -13,47 +13,15 @@ /// An absolute position in a source file as text - the absolute utf8Offset from /// the start, line, and column. public final class AbsolutePosition { - public fileprivate(set) var utf8Offset: Int - public fileprivate(set) var line: Int - public fileprivate(set) var column: Int + public let utf8Offset: Int + public let line: Int + public let column: Int - public init(line: Int = 1, column: Int = 1, utf8Offset: Int = 0) { + static let startOfFile = AbsolutePosition(line: 1, column: 1, utf8Offset: 0) + + public init(line: Int, column: Int, utf8Offset: Int) { self.line = line self.column = column self.utf8Offset = utf8Offset } - - internal func add(columns: Int) { - self.column += columns - self.utf8Offset += columns - } - - internal func add(lines: Int, size: Int) { - self.line += lines * size - self.column = 1 - self.utf8Offset += lines * size - } - - /// Use some text as a reference for adding to the absolute position, - /// taking note of newlines, etc. - internal func add(text: String) { - for char in text { - switch char { - case "\n", "\r\n": - line += 1 - column = 1 - default: - column += 1 - } - - // FIXME: This is currently very wasteful, but should be fast once the - // small-string optimization lands. - utf8Offset += String(char).utf8.count - } - } - - internal func copy() -> AbsolutePosition { - return AbsolutePosition(line: line, column: column, - utf8Offset: utf8Offset) - } } diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index c9c432befac..176d5803e0f 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -67,8 +67,43 @@ fileprivate indirect enum RawSyntaxData { struct RawSyntax: Codable { fileprivate let data: RawSyntaxData let presence: SourcePresence + + /// A ID that uniquely identifies this node and is persistent across + /// incremental parses let id: SyntaxNodeId + var _contentLength = AtomicCache() + + /// The length of this node excluding its leading and trailing trivia + var contentLength: SourceLength { + return _contentLength.value() { + switch data { + case .node(kind: _, layout: let layout): + let firstElementIndex = layout.firstIndex(where: { $0 != nil }) + let lastElementIndex = layout.lastIndex(where: { $0 != nil }) + + var contentLength = SourceLength.zero + for (offset, element) in layout.enumerated() { + guard let element = element else { + continue + } + // Skip the node's leading trivia + if offset != firstElementIndex { + contentLength += element.leadingTriviaLength + } + contentLength += element.contentLength + // Skip the node's trailing trivia + if offset != lastElementIndex { + contentLength += element.trailingTriviaLength + } + } + return contentLength + case .token(kind: let kind, leadingTrivia: _, trailingTrivia: _): + return SourceLength(of: kind.text) + } + } + } + init(kind: SyntaxKind, layout: [RawSyntax?], presence: SourcePresence, id: SyntaxNodeId? = nil) { self.data = .node(kind: kind, layout: layout) @@ -289,25 +324,6 @@ extension RawSyntax: TextOutputStreamable { } extension RawSyntax { - func accumulateAbsolutePosition(_ pos: AbsolutePosition) { - switch data { - case .node(_, let layout): - for child in layout { - guard let child = child else { continue } - child.accumulateAbsolutePosition(pos) - } - case let .token(kind, leadingTrivia, trailingTrivia): - guard isPresent else { return } - for piece in leadingTrivia { - piece.accumulateAbsolutePosition(pos) - } - pos.add(text: kind.text) - for piece in trailingTrivia { - piece.accumulateAbsolutePosition(pos) - } - } - } - var leadingTrivia: Trivia? { switch data { case .node(_, let layout): @@ -335,18 +351,19 @@ extension RawSyntax { return trailingTrivia } } +} - func accumulateLeadingTrivia(_ pos: AbsolutePosition) { - guard let trivia = leadingTrivia else { return } - for piece in trivia { - piece.accumulateAbsolutePosition(pos) - } +extension RawSyntax { + var leadingTriviaLength: SourceLength { + return leadingTrivia?.sourceLength ?? .zero } - func accumulateTrailingTrivia(_ pos: AbsolutePosition) { - guard let trivia = trailingTrivia else { return } - for piece in trivia { - piece.accumulateAbsolutePosition(pos) - } + var trailingTriviaLength: SourceLength { + return trailingTrivia?.sourceLength ?? .zero + } + + /// The length of this node including all of its trivia + var totalLength: SourceLength { + return leadingTriviaLength + contentLength + trailingTriviaLength } } diff --git a/Sources/SwiftSyntax/SourceLength.swift b/Sources/SwiftSyntax/SourceLength.swift new file mode 100644 index 00000000000..009788c5c85 --- /dev/null +++ b/Sources/SwiftSyntax/SourceLength.swift @@ -0,0 +1,93 @@ +//===------------------ SourceLength.swift - Source Length ----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// The length a syntax node spans in the source code. From any AbsolutePosition +/// you reach a node's end location by either adding its UTF-8 length or by +/// inserting `lines` newlines and then moving `columns` columns to the right. +public final class SourceLength { + public let newlines: Int + public let columnsAtLastLine: Int + public let utf8Length: Int + + /// Construct the source length of a given text + public init(of text: String) { + var newlines = 0 + var columnsAtLastLine = 0 + var utf8Length = 0 + for char in text { + let charLength = String(char).utf8.count + utf8Length += charLength + switch char { + case "\n", "\r\n", "\r": + newlines += 1 + columnsAtLastLine = 0 + default: + columnsAtLastLine += charLength + } + } + self.newlines = newlines + self.columnsAtLastLine = columnsAtLastLine + self.utf8Length = utf8Length + } + + public init(newlines: Int, columnsAtLastLine: Int, utf8Length: Int) { + self.newlines = newlines + self.columnsAtLastLine = columnsAtLastLine + self.utf8Length = utf8Length + } + + /// A zero-length source length + public static let zero: SourceLength = + SourceLength(newlines: 0, columnsAtLastLine: 0, utf8Length: 0) + + /// Combine the length of two source length. Note that the addition is *not* + /// commutative (3 columns + 1 line = 1 line but 1 line + 3 columns = 1 line + /// and 3 columns) + public static func +(lhs: SourceLength, rhs: SourceLength) -> SourceLength { + let utf8Length = lhs.utf8Length + rhs.utf8Length + let newlines = lhs.newlines + rhs.newlines + let columnsAtLastLine: Int + if rhs.newlines == 0 { + columnsAtLastLine = lhs.columnsAtLastLine + rhs.columnsAtLastLine + } else { + columnsAtLastLine = rhs.columnsAtLastLine + } + return SourceLength(newlines: newlines, + columnsAtLastLine: columnsAtLastLine, + utf8Length: utf8Length) + } + + public static func +=(lhs: inout SourceLength, rhs: SourceLength) { + lhs = lhs + rhs + } +} + +extension AbsolutePosition { + /// Determine the AbsolutePosition by advancing the `lhs` by the given source + /// length. + public static func +(lhs: AbsolutePosition, rhs: SourceLength) + -> AbsolutePosition { + let utf8Offset = lhs.utf8Offset + rhs.utf8Length + let line = lhs.line + rhs.newlines + let column: Int + if rhs.newlines == 0 { + column = lhs.column + rhs.columnsAtLastLine + } else { + column = rhs.columnsAtLastLine + 1 // AbsolutePosition has 1-based columns + } + return AbsolutePosition(line: line, column: column, utf8Offset: utf8Offset) + } + + public static func +=(lhs: inout AbsolutePosition, rhs: SourceLength) { + lhs = lhs + rhs + } +} \ No newline at end of file diff --git a/Sources/SwiftSyntax/Syntax.swift b/Sources/SwiftSyntax/Syntax.swift index ffdca1d2ac1..f295fdcc64a 100644 --- a/Sources/SwiftSyntax/Syntax.swift +++ b/Sources/SwiftSyntax/Syntax.swift @@ -133,9 +133,26 @@ extension Syntax { return data.positionAfterSkippingLeadingTrivia } + /// The absolute position where this node (excluding its trailing trivia) + /// ends. + public var endPosition: AbsolutePosition { + return data.endPosition + } + + /// The absolute position where this node's trailing trivia ends + public var endPositionAfterTrailingTrivia: AbsolutePosition { + return data.endPositionAfterTrailingTrivia + } + /// The textual byte length of this node including leading and trailing trivia. public var byteSize: Int { - return data.byteSize + return totalLength.utf8Length + } + + /// The length this node takes up spelled out in the source, excluding its + /// leading or trailing trivia. + public var contentLength: SourceLength { + return raw.contentLength } /// The leading trivia of this syntax node. Leading trivia is attached to @@ -152,6 +169,21 @@ extension Syntax { return raw.trailingTrivia } + /// The length this node's leading trivia takes up spelled out in source. + public var leadingTriviaLength: SourceLength { + return raw.leadingTriviaLength + } + + /// The length this node's trailing trivia takes up spelled out in source. + public var trailingTriviaLength: SourceLength { + return raw.trailingTriviaLength + } + + /// The length of this node including all of its trivia. + public var totalLength: SourceLength { + return raw.totalLength + } + /// When isImplicit is true, the syntax node doesn't include any /// underlying tokens, e.g. an empty CodeBlockItemList. public var isImplicit: Bool { @@ -160,8 +192,7 @@ extension Syntax { /// The textual byte length of this node exluding leading and trailing trivia. public var byteSizeAfterTrimmingTrivia: Int { - return data.byteSize - (leadingTrivia?.byteSize ?? 0) - - (trailingTrivia?.byteSize ?? 0) + return contentLength.utf8Length } /// The root of the tree in which this node resides. @@ -213,8 +244,8 @@ extension Syntax { afterLeadingTrivia: Bool = true ) -> SourceLocation { let pos = afterLeadingTrivia ? - data.position.copy() : - data.positionAfterSkippingLeadingTrivia.copy() + data.position : + data.positionAfterSkippingLeadingTrivia return SourceLocation(file: file.path, position: pos) } @@ -228,10 +259,11 @@ extension Syntax { in file: URL, afterTrailingTrivia: Bool = false ) -> SourceLocation { - let pos = data.position.copy() - raw.accumulateAbsolutePosition(pos) + var pos = data.position + pos += raw.leadingTriviaLength + pos += raw.contentLength if afterTrailingTrivia { - raw.accumulateTrailingTrivia(pos) + pos += raw.trailingTriviaLength } return SourceLocation(file: file.path, position: pos) } @@ -359,6 +391,27 @@ public struct TokenSyntax: _SyntaxBase, Hashable { return raw.tokenKind! } + /// The length this node takes up spelled out in the source, excluding its + /// leading or trailing trivia. + public var contentLength: SourceLength { + return raw.contentLength + } + + /// The length this node's leading trivia takes up spelled out in source. + public var leadingTriviaLength: SourceLength { + return raw.leadingTriviaLength + } + + /// The length this node's trailing trivia takes up spelled out in source. + public var trailingTriviaLength: SourceLength { + return raw.trailingTriviaLength + } + + /// The length of this node including all of its trivia. + public var totalLength: SourceLength { + return raw.totalLength + } + public static func ==(lhs: TokenSyntax, rhs: TokenSyntax) -> Bool { return lhs._data === rhs._data } diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 5d2e335698b..916d5f0f702 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -40,11 +40,10 @@ final class SyntaxData: Equatable { let positionCache: AtomicCache - fileprivate func calculatePosition(_ initPos: AbsolutePosition) -> - AbsolutePosition { + fileprivate func calculatePosition() -> AbsolutePosition { guard let parent = parent else { // If this node is SourceFileSyntax, its location is the start of the file. - return initPos + return AbsolutePosition.startOfFile } // If the node is the first child of its parent, the location is same with @@ -55,44 +54,30 @@ final class SyntaxData: Equatable { // adding the stride of the sibling. for idx in (0.. AbsolutePosition { - // If this node is root, the position of the next sibling is the end of - // this node. - guard let parent = parent else { - let result = self.position.copy() - raw.accumulateAbsolutePosition(result) - return result - } - - // Find the first valid sibling and return its position. - for i in indexInParent+1.. TriviaPiece { return pieces[index] } - - /// Get the byteSize of this trivia - public var byteSize: Int { - let pos = AbsolutePosition() - for piece in pieces { - piece.accumulateAbsolutePosition(pos) - } - return pos.utf8Offset - } } @@ -200,18 +200,20 @@ public func +(lhs: Trivia, rhs: Trivia) -> Trivia { } extension TriviaPiece { - func accumulateAbsolutePosition(_ pos: AbsolutePosition) { + public var sourceLength: SourceLength { switch self { % for trivia in TRIVIAS: % if trivia.is_new_line: case let .${trivia.lower_name}s(count): - pos.add(lines: count, size: ${trivia.characters_len()}) + return SourceLength(newlines: count, columnsAtLastLine: 0, + utf8Length: count * ${trivia.characters_len()}) % elif trivia.is_collection(): case let .${trivia.lower_name}s(count): - pos.add(columns: count) + return SourceLength(newlines: 0, columnsAtLastLine: count, + utf8Length: count) % else: case let .${trivia.lower_name}(text): - pos.add(text: text) + return SourceLength(of: text) % end % end } From 84c6d0d8dff97da4f6955e2b8fdcebe1cdc2a3c2 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 14 Aug 2018 11:51:05 -0700 Subject: [PATCH 18/25] Make AbsolutePosition a value type AbsolutePosition had value semantics anyway, the only reason it was a reference type was so that we can use it in AtomicCache. But that can be worked around by boxing it into a reference type. --- Sources/SwiftSyntax/AbsolutePosition.swift | 2 +- Sources/SwiftSyntax/SyntaxData.swift | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Sources/SwiftSyntax/AbsolutePosition.swift b/Sources/SwiftSyntax/AbsolutePosition.swift index 7b0f265bfe7..518a4b0c7aa 100644 --- a/Sources/SwiftSyntax/AbsolutePosition.swift +++ b/Sources/SwiftSyntax/AbsolutePosition.swift @@ -12,7 +12,7 @@ /// An absolute position in a source file as text - the absolute utf8Offset from /// the start, line, and column. -public final class AbsolutePosition { +public struct AbsolutePosition { public let utf8Offset: Int public let line: Int public let column: Int diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 916d5f0f702..920b1c96bec 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -18,6 +18,15 @@ import Foundation /// exposed to clients. typealias NodeIdentifier = [Int] +/// Box a value type into a reference type +fileprivate class Box { + let value: T + + init(_ value: T) { + self.value = value + } +} + /// SyntaxData is the underlying storage for each Syntax node. /// It's modelled as an array that stores and caches a SyntaxData for each raw /// syntax node in its layout. It is up to the specific Syntax nodes to maintain @@ -38,7 +47,7 @@ final class SyntaxData: Equatable { let childCaches: [AtomicCache] - let positionCache: AtomicCache + private let positionCache: AtomicCache> fileprivate func calculatePosition() -> AbsolutePosition { guard let parent = parent else { @@ -62,7 +71,7 @@ final class SyntaxData: Equatable { /// The position of the start of this node's leading trivia var position: AbsolutePosition { - return positionCache.value { return calculatePosition() } + return positionCache.value({ return Box(calculatePosition()) }).value } /// The position of the start of this node's content, skipping its trivia @@ -93,7 +102,7 @@ final class SyntaxData: Equatable { self.indexInParent = indexInParent self.parent = parent self.childCaches = raw.layout.map { _ in AtomicCache() } - self.positionCache = AtomicCache() + self.positionCache = AtomicCache>() } /// The index path from this node to the root. This can be used to uniquely From b7deaa6c42b6c835a443bd0a92f7d872a5e28d4e Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 23 Aug 2018 09:12:14 -0700 Subject: [PATCH 19/25] Remove validate methods The methods were never executed because DEBUG was never defined in normal builds and the only way to create nodes is through generated factory methods which provide the same safety `validate` was supposed to ensure at the interface level. --- Sources/SwiftSyntax/Syntax.swift | 7 ---- .../SwiftSyntax/SyntaxCollections.swift.gyb | 3 -- Sources/SwiftSyntax/SyntaxNodes.swift.gyb | 41 ------------------- 3 files changed, 51 deletions(-) diff --git a/Sources/SwiftSyntax/Syntax.swift b/Sources/SwiftSyntax/Syntax.swift index f295fdcc64a..804ca846e0c 100644 --- a/Sources/SwiftSyntax/Syntax.swift +++ b/Sources/SwiftSyntax/Syntax.swift @@ -32,10 +32,6 @@ internal protocol _SyntaxBase: Syntax { /// property must be a descendent of the root. This relationship must /// be preserved in all circumstances where Syntax nodes are created. var _data: SyntaxData { get } - -#if DEBUG - func validate() -#endif } extension _SyntaxBase { public func validate() { @@ -302,9 +298,6 @@ public struct TokenSyntax: _SyntaxBase, Hashable { internal init(root: SyntaxData, data: SyntaxData) { self._root = root self._data = data -#if DEBUG - validate() -#endif } /// The text of the token as written in the source code. diff --git a/Sources/SwiftSyntax/SyntaxCollections.swift.gyb b/Sources/SwiftSyntax/SyntaxCollections.swift.gyb index 5f9e37c012b..c5c1b78588d 100644 --- a/Sources/SwiftSyntax/SyntaxCollections.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxCollections.swift.gyb @@ -34,9 +34,6 @@ public struct ${node.name}: _SyntaxBase { internal init(root: SyntaxData, data: SyntaxData) { self._root = root self._data = data -#if DEBUG - validate() -#endif } /// Creates a new `${node.name}` by replacing the underlying layout with diff --git a/Sources/SwiftSyntax/SyntaxNodes.swift.gyb b/Sources/SwiftSyntax/SyntaxNodes.swift.gyb index 7ea12f108b2..71fc779cf66 100644 --- a/Sources/SwiftSyntax/SyntaxNodes.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxNodes.swift.gyb @@ -48,9 +48,6 @@ public struct UnknownSyntax: _SyntaxBase { internal init(root: SyntaxData, data: SyntaxData) { self._root = root self._data = data -#if DEBUG - validate() -#endif } } @@ -85,46 +82,8 @@ public struct ${node.name}: ${base_type}, _SyntaxBase, Hashable { internal init(root: SyntaxData, data: SyntaxData) { self._root = root self._data = data -#if DEBUG - validate() -#endif } -% if node.requires_validation(): -#if DEBUG - func validate() { - if isMissing { return } - precondition(raw.layout.count == ${len(node.children)}) -% for child in node.children: -% child_var = '_' + child.swift_name - let ${child_var} = raw[Cursor.${child.swift_syntax_kind}] -% if child.token_choices: -% choices = ["." + choice.swift_kind() for choice in child.token_choices] -% choice_array = "[%s]" % ', '.join(choices) - guard let ${child_var}TokenKind = ${child_var}.tokenKind else { - fatalError("expected token child, got \(${child_var}.kind)") - } - precondition(${choice_array}.contains(${child_var}TokenKind), - "expected one of ${choice_array} for '${child.swift_name}' " + - "in node of kind ${node.swift_syntax_kind}") -% elif child.text_choices: -% choices = ", ".join("\"%s\"" % choice -% for choice in child.text_choices) - guard let ${child_var}TokenKind = ${child_var}.tokenKind else { - fatalError("expected token child, got \(${child_var}.kind)") - } - precondition([${choices}].contains(${child_var}TokenKind.text), - "expected one of '[${', '.join(child.text_choices)}]', got " + - "'\(${child_var}TokenKind.text)'") -% else: - precondition(${child_var}.kind == .${child.swift_syntax_kind}, - "expected child of kind .${child.swift_syntax_kind}, " + - "got \(${child_var}.kind)") -% end -% end - } -#endif -% end % for child in node.children: % ret_type = child.type_name % cast_symbol = 'as!' From 9cef71e7b79246238cf002985e58609c8032d161 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 Aug 2018 09:28:11 -0700 Subject: [PATCH 20/25] Make SourceLength a struct --- Sources/SwiftSyntax/RawSyntax.swift | 19 ++++++++++++++----- Sources/SwiftSyntax/SourceLength.swift | 4 ++-- Sources/SwiftSyntax/SyntaxData.swift | 9 --------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index 176d5803e0f..16b85e86118 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -24,6 +24,15 @@ extension CodingUserInfoKey { CodingUserInfoKey(rawValue: "SwiftSyntax.RawSyntax.OmittedNodeLookup")! } +/// Box a value type into a reference type +class Box { + let value: T + + init(_ value: T) { + self.value = value + } +} + /// A ID that uniquely identifies a syntax node and stays stable across multiple /// incremental parses public struct SyntaxNodeId: Hashable, Codable { @@ -72,11 +81,11 @@ struct RawSyntax: Codable { /// incremental parses let id: SyntaxNodeId - var _contentLength = AtomicCache() + var _contentLength = AtomicCache>() /// The length of this node excluding its leading and trailing trivia var contentLength: SourceLength { - return _contentLength.value() { + return _contentLength.value({ switch data { case .node(kind: _, layout: let layout): let firstElementIndex = layout.firstIndex(where: { $0 != nil }) @@ -97,11 +106,11 @@ struct RawSyntax: Codable { contentLength += element.trailingTriviaLength } } - return contentLength + return Box(contentLength) case .token(kind: let kind, leadingTrivia: _, trailingTrivia: _): - return SourceLength(of: kind.text) + return Box(SourceLength(of: kind.text)) } - } + }).value } init(kind: SyntaxKind, layout: [RawSyntax?], presence: SourcePresence, diff --git a/Sources/SwiftSyntax/SourceLength.swift b/Sources/SwiftSyntax/SourceLength.swift index 009788c5c85..224a7f65e64 100644 --- a/Sources/SwiftSyntax/SourceLength.swift +++ b/Sources/SwiftSyntax/SourceLength.swift @@ -13,7 +13,7 @@ /// The length a syntax node spans in the source code. From any AbsolutePosition /// you reach a node's end location by either adding its UTF-8 length or by /// inserting `lines` newlines and then moving `columns` columns to the right. -public final class SourceLength { +public struct SourceLength { public let newlines: Int public let columnsAtLastLine: Int public let utf8Length: Int @@ -90,4 +90,4 @@ extension AbsolutePosition { public static func +=(lhs: inout AbsolutePosition, rhs: SourceLength) { lhs = lhs + rhs } -} \ No newline at end of file +} diff --git a/Sources/SwiftSyntax/SyntaxData.swift b/Sources/SwiftSyntax/SyntaxData.swift index 920b1c96bec..2c9f04615ae 100644 --- a/Sources/SwiftSyntax/SyntaxData.swift +++ b/Sources/SwiftSyntax/SyntaxData.swift @@ -18,15 +18,6 @@ import Foundation /// exposed to clients. typealias NodeIdentifier = [Int] -/// Box a value type into a reference type -fileprivate class Box { - let value: T - - init(_ value: T) { - self.value = value - } -} - /// SyntaxData is the underlying storage for each Syntax node. /// It's modelled as an array that stores and caches a SyntaxData for each raw /// syntax node in its layout. It is up to the specific Syntax nodes to maintain From 2703c0044a364e36e0a9d8e38c63842b9f734ce5 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 Aug 2018 09:31:40 -0700 Subject: [PATCH 21/25] Make RawSyntaxData a direct enum --- Sources/SwiftSyntax/RawSyntax.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index 16b85e86118..bee43d1e9c0 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -63,7 +63,7 @@ public struct SyntaxNodeId: Hashable, Codable { } /// The data that is specific to a tree or token node -fileprivate indirect enum RawSyntaxData { +fileprivate enum RawSyntaxData { /// A tree node with a kind and an array of children case node(kind: SyntaxKind, layout: [RawSyntax?]) /// A token with a token kind, leading trivia, and trailing trivia From f511cccc27c829bec7012432b633f200eb5562f8 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 29 Aug 2018 14:55:35 -0700 Subject: [PATCH 22/25] Don't set the process terminationHandler in SwiftcInvocation --- Sources/SwiftSyntax/SwiftcInvocation.swift | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Sources/SwiftSyntax/SwiftcInvocation.swift b/Sources/SwiftSyntax/SwiftcInvocation.swift index ac7f254a632..50e2ecf3192 100644 --- a/Sources/SwiftSyntax/SwiftcInvocation.swift +++ b/Sources/SwiftSyntax/SwiftcInvocation.swift @@ -63,10 +63,6 @@ private func runCore(_ executable: URL, _ arguments: [String] = []) } let process = Process() - process.terminationHandler = { process in - stdoutPipe.fileHandleForReading.readabilityHandler = nil - stderrPipe.fileHandleForReading.readabilityHandler = nil - } process.launchPath = executable.path process.arguments = arguments process.standardOutput = stdoutPipe From e58d288b2621b5a5035c5b5d15ecc2a48baf6e6e Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Thu, 6 Sep 2018 23:29:49 -0700 Subject: [PATCH 23/25] Update tests after cherrypicks. --- Tests/SwiftSyntaxTest/AbsolutePosition.swift | 22 ++++++++++++----- Tests/SwiftSyntaxTest/DeserializeFile.swift | 4 ++-- Tests/SwiftSyntaxTest/DiagnosticTest.swift | 22 ++++++++--------- .../Inputs/nested-blocks.swift | 8 +++++++ Tests/SwiftSyntaxTest/ParseFile.swift | 10 ++++---- Tests/SwiftSyntaxTest/VisitorTest.swift | 24 ++++++++++++++++--- 6 files changed, 63 insertions(+), 27 deletions(-) create mode 100644 Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift diff --git a/Tests/SwiftSyntaxTest/AbsolutePosition.swift b/Tests/SwiftSyntaxTest/AbsolutePosition.swift index 3f451bafffd..c7a23a22c3e 100644 --- a/Tests/SwiftSyntaxTest/AbsolutePosition.swift +++ b/Tests/SwiftSyntaxTest/AbsolutePosition.swift @@ -12,7 +12,7 @@ public class AbsolutePositionTestCase: XCTestCase { public func testVisitor() { XCTAssertNoThrow(try { let source = try String(contentsOf: getInput("visitor.swift")) - let parsed = try SourceFileSyntax.parse(getInput("visitor.swift")) + let parsed = try SyntaxTreeParser.parse(getInput("visitor.swift")) XCTAssertEqual(parsed.position.utf8Offset, 0) XCTAssertEqual( parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset, @@ -25,7 +25,7 @@ public class AbsolutePositionTestCase: XCTestCase { public func testClosure() { XCTAssertNoThrow(try { let source = try String(contentsOf: getInput("closure.swift")) - let parsed = try SourceFileSyntax.parse(getInput("closure.swift")) + let parsed = try SyntaxTreeParser.parse(getInput("closure.swift")) XCTAssertEqual( parsed.eofToken.positionAfterSkippingLeadingTrivia.utf8Offset, source.count) @@ -36,7 +36,7 @@ public class AbsolutePositionTestCase: XCTestCase { public func testRename() { XCTAssertNoThrow(try { - let parsed = try SourceFileSyntax.parse(getInput("visitor.swift")) + let parsed = try SyntaxTreeParser.parse(getInput("visitor.swift")) let renamed = FuncRenamer().visit(parsed) as! SourceFileSyntax let renamedSource = renamed.description XCTAssertEqual( @@ -48,7 +48,7 @@ public class AbsolutePositionTestCase: XCTestCase { public func testCurrentFile() { XCTAssertNoThrow(try { - let parsed = try SourceFileSyntax.parse(URL(fileURLWithPath: #file)) + let parsed = try SyntaxTreeParser.parse(URL(fileURLWithPath: #file)) class Visitor: SyntaxVisitor { override func visitPre(_ node: Syntax) { _ = node.position @@ -103,7 +103,7 @@ public class AbsolutePositionTestCase: XCTestCase { public func testTrivias() { let idx = 5 - let root = createSourceFile(idx + 1) + let root = self.createSourceFile(idx + 1) XCTAssertEqual(root.leadingTrivia!.count, 3) XCTAssertEqual(root.trailingTrivia!.count, 0) let state = root.statements[idx] @@ -116,7 +116,17 @@ public class AbsolutePositionTestCase: XCTestCase { } public func testImplicit() { - let root = createSourceFile(0) + let root = self.createSourceFile(0) XCTAssertTrue(root.statements.isImplicit) } + + public func testWithoutSourceFileRoot() { + let item = SyntaxFactory.makeCodeBlockItem( + item: SyntaxFactory.makeReturnStmt( + returnKeyword: SyntaxFactory.makeToken(.returnKeyword, presence: .present) + .withLeadingTrivia(.newlines(1)).withTrailingTrivia(.newlines(1)), + expression: nil), semicolon: nil) + XCTAssertEqual(0, item.position.utf8Offset) + XCTAssertEqual(1, item.positionAfterSkippingLeadingTrivia.utf8Offset) + } } diff --git a/Tests/SwiftSyntaxTest/DeserializeFile.swift b/Tests/SwiftSyntaxTest/DeserializeFile.swift index 95f16615894..b0855e1ee5d 100644 --- a/Tests/SwiftSyntaxTest/DeserializeFile.swift +++ b/Tests/SwiftSyntaxTest/DeserializeFile.swift @@ -1,12 +1,12 @@ import XCTest import SwiftSyntax -public class DecodeSyntaxTestCase: XCTestCase { +public class DecodeSytnaxTestCase: XCTestCase { public func testBasic() { XCTAssertNoThrow(try { let inputFile = getInput("visitor.swift") let source = try String(contentsOf: inputFile) - let parsed = try SourceFileSyntax.parse(inputFile) + let parsed = try SyntaxTreeParser.parse(inputFile) XCTAssertEqual("\(parsed)", source) }()) } diff --git a/Tests/SwiftSyntaxTest/DiagnosticTest.swift b/Tests/SwiftSyntaxTest/DiagnosticTest.swift index ce89885ab98..329c1b468bc 100644 --- a/Tests/SwiftSyntaxTest/DiagnosticTest.swift +++ b/Tests/SwiftSyntaxTest/DiagnosticTest.swift @@ -1,13 +1,13 @@ import XCTest import SwiftSyntax -func loc(_ file: String = #file, line: Int = #line, +fileprivate func loc(_ file: String = #file, line: Int = #line, column: Int = #line) -> SourceLocation { return SourceLocation(line: line, column: column, offset: 0, file: file) } /// Adds static constants to Diagnostic.Message. -extension Diagnostic.Message { +fileprivate extension Diagnostic.Message { /// Error thrown when a conversion between two types is impossible. static func cannotConvert(fromType: String, toType: String) -> Diagnostic.Message { @@ -27,7 +27,7 @@ extension Diagnostic.Message { } } -public class DiagnosticsTestCase: XCTestCase { +public class DiagnosticTestCase: XCTestCase { public func testDiagnosticEmission() { let startLoc = loc() let fixLoc = loc() @@ -60,7 +60,7 @@ public class DiagnosticsTestCase: XCTestCase { public func testSourceLocations() { let engine = DiagnosticEngine() - engine.addConsumer(PrintingDiagnosticConsumer()) + // engine.addConsumer(PrintingDiagnosticConsumer()) let url = getInput("diagnostics.swift") class Visitor: SyntaxVisitor { @@ -81,14 +81,14 @@ public class DiagnosticsTestCase: XCTestCase { } XCTAssertNoThrow(try { - let file = try SourceFileSyntax.parse(url) - Visitor(url: url, engine: engine).visit(file) + let file = try SyntaxTreeParser.parse(url) + Visitor(url: url, engine: engine).visit(file) }()) - XCTAssertEqual(6, engine.diagnostics.count) - let lines = Set(engine.diagnostics.compactMap { $0.location?.line }) - XCTAssertEqual([1, 3, 5, 7, 9, 11], lines) - let columns = Set(engine.diagnostics.compactMap { $0.location?.column }) - XCTAssertEqual([6, 2], columns) + XCTAssertEqual(6, engine.diagnostics.count) + let lines = Set(engine.diagnostics.compactMap { $0.location?.line }) + XCTAssertEqual([1, 3, 5, 7, 9, 11], lines) + let columns = Set(engine.diagnostics.compactMap { $0.location?.column }) + XCTAssertEqual([6, 2], columns) } } diff --git a/Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift b/Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift new file mode 100644 index 00000000000..9993ae44877 --- /dev/null +++ b/Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift @@ -0,0 +1,8 @@ +struct Foo { + func foo() { + print("hello") + func bar() { + print("goodbye") + } + } +} diff --git a/Tests/SwiftSyntaxTest/ParseFile.swift b/Tests/SwiftSyntaxTest/ParseFile.swift index d20d0c9180c..fc71e8c8b1f 100644 --- a/Tests/SwiftSyntaxTest/ParseFile.swift +++ b/Tests/SwiftSyntaxTest/ParseFile.swift @@ -1,13 +1,13 @@ import XCTest import SwiftSyntax -struct Foo { +fileprivate struct Foo { public let x: Int private(set) var y: [Bool] } #if os(macOS) -class Test: NSObject { +fileprivate class Test: NSObject { @objc var bar: Int = 0 func test() { print(#selector(function)) @@ -22,9 +22,9 @@ public class ParseFileTestCase: XCTestCase { public func testParseSingleFile() { let currentFile = URL(fileURLWithPath: #file) XCTAssertNoThrow(try { - let currentFileContents = try String(contentsOf: currentFile) - let parsed = try SourceFileSyntax.parse(currentFile) - XCTAssertEqual("\(parsed)", currentFileContents) + let fileContents = try String(contentsOf: currentFile) + let parsed = try SyntaxTreeParser.parse(currentFile) + XCTAssertEqual("\(parsed)", fileContents) }()) } } diff --git a/Tests/SwiftSyntaxTest/VisitorTest.swift b/Tests/SwiftSyntaxTest/VisitorTest.swift index 6ccca03f6b8..0cf0714e601 100644 --- a/Tests/SwiftSyntaxTest/VisitorTest.swift +++ b/Tests/SwiftSyntaxTest/VisitorTest.swift @@ -11,7 +11,7 @@ public class SyntaxVisitorTestCase: XCTestCase { } } XCTAssertNoThrow(try { - let parsed = try SourceFileSyntax.parse(getInput("visitor.swift")) + let parsed = try SyntaxTreeParser.parse(getInput("visitor.swift")) let counter = FuncCounter() let hashBefore = parsed.hashValue counter.visit(parsed) @@ -28,7 +28,7 @@ public class SyntaxVisitorTestCase: XCTestCase { } } XCTAssertNoThrow(try { - let parsed = try SourceFileSyntax.parse(getInput("closure.swift")) + let parsed = try SyntaxTreeParser.parse(getInput("closure.swift")) let rewriter = ClosureRewriter() let rewritten = rewriter.visit(parsed) XCTAssertEqual(parsed.description, rewritten.description) @@ -49,7 +49,7 @@ public class SyntaxVisitorTestCase: XCTestCase { } } XCTAssertNoThrow(try { - let parsed = try SourceFileSyntax.parse(getInput("near-empty.swift")) + let parsed = try SyntaxTreeParser.parse(getInput("near-empty.swift")) let rewriter = VisitAnyRewriter(transform: { _ in return SyntaxFactory.makeIdentifier("") }) @@ -57,4 +57,22 @@ public class SyntaxVisitorTestCase: XCTestCase { XCTAssertEqual(rewritten.description, "") }()) } + + public func testSyntaxRewriterVisitCollection() { + class VisitCollections: SyntaxVisitor { + var numberOfCodeBlockItems = 0 + + override func visit(_ items: CodeBlockItemListSyntax) { + numberOfCodeBlockItems += items.count + super.visit(items) + } + } + + XCTAssertNoThrow(try { + let parsed = try SyntaxTreeParser.parse(getInput("nested-blocks.swift")) + let visitor = VisitCollections() + visitor.visit(parsed) + XCTAssertEqual(4, visitor.numberOfCodeBlockItems) + }()) + } } From 5d9650bdbc9f7ea617cfd1c843782a9972e121a2 Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Thu, 6 Sep 2018 23:53:16 -0700 Subject: [PATCH 24/25] Allow *ListSyntax nodes to be visited. This change was made post-4.2 in apple/swift's GYB support files. We copy it here because it's generally useful. --- Sources/SwiftSyntax/SyntaxRewriter.swift.gyb | 23 +++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/Sources/SwiftSyntax/SyntaxRewriter.swift.gyb b/Sources/SwiftSyntax/SyntaxRewriter.swift.gyb index 289a04a5a7a..c98b8e21eb7 100644 --- a/Sources/SwiftSyntax/SyntaxRewriter.swift.gyb +++ b/Sources/SwiftSyntax/SyntaxRewriter.swift.gyb @@ -2,6 +2,14 @@ from gyb_syntax_support import * # -*- mode: Swift -*- # Ignore the following admonition it applies to the resulting .swift file only + + # gyb_syntax_support was updated post-4.2 to allow list nodes to be + # visited. We can't cherrypick that in cleanly because those files + # live in apple/swift, so we've simply copied the updated implementation + # here and updated the calls below. + # See: https://github.com/apple/swift/commit/a66931e7 + def PATCHED_is_visitable(node): + return not node.is_base() }% //// Automatically Generated From SyntaxFactory.swift.gyb. //// Do Not Edit Directly! @@ -28,9 +36,12 @@ open class SyntaxRewriter { public init() {} % for node in SYNTAX_NODES: -% if is_visitable(node): - open func visit(_ node: ${node.name}) -> ${node.base_type} { -% cast = ('as! ' + node.base_type) if node.base_type != 'Syntax' else '' +% if PATCHED_is_visitable(node): +% # See: https://github.com/apple/swift/commit/a66931e7 +% base_type = 'Syntax' if node.base_kind == 'SyntaxCollection' \ +% else node.base_type + open func visit(_ node: ${node.name}) -> ${base_type} { +% cast = ('as! ' + base_type) if base_type != 'Syntax' else '' return visitChildren(node) ${cast} } @@ -70,7 +81,7 @@ open class SyntaxRewriter { switch node.raw.kind { case .token: return visit(node as! TokenSyntax) % for node in SYNTAX_NODES: -% if is_visitable(node): +% if PATCHED_is_visitable(node): case .${node.swift_syntax_kind}: return visit(node as! ${node.name}) % end % end @@ -97,7 +108,7 @@ open class SyntaxRewriter { open class SyntaxVisitor { public init() {} % for node in SYNTAX_NODES: -% if is_visitable(node): +% if PATCHED_is_visitable(node): open func visit(_ node: ${node.name}) { visitChildren(node) } @@ -120,7 +131,7 @@ open class SyntaxVisitor { switch node.raw.kind { case .token: visit(node as! TokenSyntax) % for node in SYNTAX_NODES: -% if is_visitable(node): +% if PATCHED_is_visitable(node): case .${node.swift_syntax_kind}: visit(node as! ${node.name}) % end % end From bd3484b80e46dd844b6fad3f1a7f900fe35b1b22 Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Fri, 7 Sep 2018 15:27:21 -0700 Subject: [PATCH 25/25] Apply review fixes --- Sources/SwiftSyntax/SwiftSyntax.swift | 2 +- Tests/SwiftSyntaxTest/VisitorTest.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftSyntax/SwiftSyntax.swift b/Sources/SwiftSyntax/SwiftSyntax.swift index 568178eb484..57b0b9ec3da 100644 --- a/Sources/SwiftSyntax/SwiftSyntax.swift +++ b/Sources/SwiftSyntax/SwiftSyntax.swift @@ -27,7 +27,7 @@ public enum ParserError: Error, CustomStringConvertible { case invalidFile public var description: String { - switch self{ + switch self { case let .swiftcFailed(exitCode, stderr): let stderrLines = stderr.split(separator: "\n") return """ diff --git a/Tests/SwiftSyntaxTest/VisitorTest.swift b/Tests/SwiftSyntaxTest/VisitorTest.swift index 0cf0714e601..f916fe2dadf 100644 --- a/Tests/SwiftSyntaxTest/VisitorTest.swift +++ b/Tests/SwiftSyntaxTest/VisitorTest.swift @@ -72,7 +72,7 @@ public class SyntaxVisitorTestCase: XCTestCase { let parsed = try SyntaxTreeParser.parse(getInput("nested-blocks.swift")) let visitor = VisitCollections() visitor.visit(parsed) - XCTAssertEqual(4, visitor.numberOfCodeBlockItems) + XCTAssertEqual(visitor.numberOfCodeBlockItems, 4) }()) } }