Merge pull request swiftlang#73 from akyrtzi/opt2-syntax-data

Make creation and visitation of SyntaxData more efficient

Author: akyrtzi

Diff for: Package.swift

@@ -5,7 +5,8 @@ import PackageDescription
 let package = Package(
   name: "SwiftSyntax",
   targets: [
-    .target(name: "SwiftSyntax"),
+    .target(name: "_CSwiftSyntax"),
+    .target(name: "SwiftSyntax", dependencies: ["_CSwiftSyntax"]),
     .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"], exclude: ["Inputs"]),
     .target(name: "lit-test-helper", dependencies: ["SwiftSyntax"])
   ]

Diff for: Sources/SwiftSyntax/AtomicCounter.swift

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+// This is using the C function from '_CSwiftSyntax/src/atomic-counter.c' by
+// dynamically loading it using `dlsym`. The reason this mechanism is used
+// instead of importing a header is so that we preserve the property that
+// SwiftSyntax is a self-contained Swift module.
+// (also see '_CSwiftSyntax/include/README.md')
+
+#if os(Linux)
+import Glibc
+#else
+import Darwin.C
+#endif
+
+#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS)
+  fileprivate let RTLD_DEFAULT = UnsafeMutableRawPointer(bitPattern: -2)
+#elseif os(Linux)
+  fileprivate let RTLD_DEFAULT = UnsafeMutableRawPointer(bitPattern: 0)
+#endif
+
+fileprivate typealias get_unique_counter_ty = (@convention(c) ()->UInt64)
+fileprivate let get_unique_counter_str = "cswiftsyntax_get_unique_counter"
+
+/// Provides API to get an atomically increasing counter value.
+struct AtomicCounter {
+  /// Get an atomically increasing counter value.
+  static func next() -> UInt64 {
+    return AtomicCounter.cswiftsyntax_get_unique_counter()
+  }
+
+  static fileprivate let cswiftsyntax_get_unique_counter: get_unique_counter_ty = { ()->get_unique_counter_ty in
+    let ptr = dlsym(RTLD_DEFAULT, get_unique_counter_str)!
+    return unsafeBitCast(ptr, to: get_unique_counter_ty.self)
+  }()
+}

Diff for: Sources/SwiftSyntax/IncrementalEditTransition.swift

@@ -102,8 +102,8 @@ public final class IncrementalEditTransition: IncrementalParseLookup {
     return node
   }
 
-  fileprivate func lookUpFrom(_ node: Syntax, nodeOffset: Int, prevOffset: Int,
-                              kind: SyntaxKind) -> (Int, Syntax?) {
+  fileprivate func lookUpFrom(_ node: _SyntaxBase, nodeOffset: Int, prevOffset: Int,
+                              kind: SyntaxKind) -> (Int, _SyntaxBase?) {
     if nodeCanBeReused(node, nodeOffset: nodeOffset, prevOffset: prevOffset,
                        kind: kind) {
       return (nodeOffset, node)
@@ -117,7 +117,7 @@ public final class IncrementalEditTransition: IncrementalParseLookup {
       }
       let childEnd = childOffset + child.byteSize
       if childOffset <= prevOffset && prevOffset < childEnd {
-        return lookUpFrom(child, nodeOffset: childOffset,
+        return lookUpFrom(child.base, nodeOffset: childOffset,
                           prevOffset: prevOffset, kind: kind)
       }
       // The next child starts where the previous child ended
@@ -126,7 +126,7 @@ public final class IncrementalEditTransition: IncrementalParseLookup {
     return (0, nil)
   }
 
-  fileprivate func nodeCanBeReused(_ node: Syntax, nodeOffset: Int,
+  fileprivate func nodeCanBeReused(_ node: _SyntaxBase, nodeOffset: Int,
                                    prevOffset: Int, kind: SyntaxKind) -> Bool {
     // Computing the value of NodeStart on the fly is faster than determining a
     // node's absolute position, but make sure the values match in an assertion

Diff for: Sources/SwiftSyntax/LazyNonThreadSafeCache.swift

@@ -10,19 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-/// Box a value type into a reference type
-class Box<T> {
-  let value: T
-
-  init(_ value: T) {
-    self.value = value
-  }
-}
-
 /// The data that is specific to a tree or token node
 fileprivate enum RawSyntaxData {
   /// A tree node with a kind and an array of children
-  case node(kind: SyntaxKind, layout: [RawSyntax?])
+  case node(kind: SyntaxKind, layout: [RawSyntax?], totalNodes: Int)
   /// A token with a token kind, leading trivia, and trailing trivia
   case token(kind: TokenKind, leadingTrivia: Trivia, trailingTrivia: Trivia)
 }
@@ -37,7 +28,11 @@ final class RawSyntax {
 
   init(kind: SyntaxKind, layout: [RawSyntax?], length: SourceLength,
        presence: SourcePresence) {
-    self.data = .node(kind: kind, layout: layout)
+    var totalSubNodes = 0
+    for subnode in layout {
+      totalSubNodes += subnode?.totalNodes ?? 0
+    }
+    self.data = .node(kind: kind, layout: layout, totalNodes: totalSubNodes)
     self.totalLength = length
     self.presence = presence
   }
@@ -53,26 +48,42 @@ final class RawSyntax {
   /// The syntax kind of this raw syntax.
   var kind: SyntaxKind {
     switch data {
-    case .node(let kind, _): return kind
+    case .node(let kind, _, _): return kind
     case .token(_, _, _): return .token
     }
   }
 
   var tokenKind: TokenKind? {
     switch data {
-    case .node(_, _): return nil
+    case .node(_, _, _): return nil
     case .token(let kind, _, _): return kind
     }
   }
 
   /// The layout of the children of this Raw syntax node.
   var layout: [RawSyntax?] {
     switch data {
-    case .node(_, let layout): return layout
+    case .node(_, let layout, _): return layout
     case .token(_, _, _): return []
     }
   }
 
+  /// The number of children, `present` or `missing`, in this node.
+  var numberOfChildren: Int {
+    switch data {
+    case .node(_, let layout, _): return layout.count
+    case .token(_, _, _): return 0
+    }
+  }
+
+  /// Total number of nodes in this sub-tree, including `self` node.
+  var totalNodes: Int {
+    switch data {
+    case .node(_, _, let totalSubNodes): return totalSubNodes+1
+    case .token(_, _, _): return 1
+    }
+  }
+
   /// Whether or not this node is a token one.
   var isToken: Bool {
     return tokenKind != nil
@@ -117,7 +128,7 @@ final class RawSyntax {
   /// - Parameter newLayout: The children of the new node you're creating.
   func replacingLayout(_ newLayout: [RawSyntax?]) -> RawSyntax {
     switch data {
-    case let .node(kind, _):
+    case let .node(kind, _, _):
       return .createAndCalcLength(kind: kind, layout: newLayout,
         presence: presence)
     case .token(_, _, _): return self
@@ -164,7 +175,7 @@ extension RawSyntax: TextOutputStreamable {
   func write<Target>(to target: inout Target)
     where Target: TextOutputStream {
     switch data {
-    case .node(_, let layout):
+    case .node(_, let layout, _):
       for child in layout {
         guard let child = child else { continue }
         child.write(to: &target)
@@ -185,7 +196,7 @@ extension RawSyntax: TextOutputStreamable {
 extension RawSyntax {
   var leadingTrivia: Trivia? {
     switch data {
-    case .node(_, let layout):
+    case .node(_, let layout, _):
       for child in layout {
         guard let child = child else { continue }
         guard let result = child.leadingTrivia else { break }
@@ -199,7 +210,7 @@ extension RawSyntax {
 
   var trailingTrivia: Trivia? {
     switch data {
-    case .node(_, let layout):
+    case .node(_, let layout, _):
       for child in layout.reversed() {
         guard let child = child else { continue }
         guard let result = child.trailingTrivia else { break }
@@ -284,7 +295,7 @@ extension RawSyntax {
     defer { visitor.moveUp() }
     guard isPresent else { return }
     switch data {
-    case .node(let kind,let layout):
+    case .node(let kind,let layout, _):
       let shouldVisit = visitor.shouldVisit(kind)
       var visitChildren = true
       if shouldVisit {

Diff for: Sources/SwiftSyntax/RawSyntax.swift

@@ -156,6 +156,41 @@ public final class SourceLocationConverter {
   }
 }
 
+extension _SyntaxBase {
+  func startLocation(
+    converter: SourceLocationConverter,
+    afterLeadingTrivia: Bool = true
+  ) -> SourceLocation {
+    let pos = afterLeadingTrivia ?
+      data.positionAfterSkippingLeadingTrivia :
+      data.position
+    return converter.location(for: pos)
+  }
+
+  func endLocation(
+    converter: SourceLocationConverter,
+    afterTrailingTrivia: Bool = false
+  ) -> SourceLocation {
+    var pos = data.position
+    pos += raw.leadingTriviaLength
+    pos += raw.contentLength
+    if afterTrailingTrivia {
+      pos += raw.trailingTriviaLength
+    }
+    return converter.location(for: pos)
+  }
+
+  func sourceRange(
+    converter: SourceLocationConverter,
+    afterLeadingTrivia: Bool = true,
+    afterTrailingTrivia: Bool = false
+  ) -> SourceRange {
+    let start = startLocation(converter: converter, afterLeadingTrivia: afterLeadingTrivia)
+    let end = endLocation(converter: converter, afterTrailingTrivia: afterTrailingTrivia)
+    return SourceRange(start: start, end: end)
+  }
+}
+
 extension Syntax {
   /// The starting location, in the provided file, of this Syntax node.
   /// - Parameters:
@@ -167,10 +202,8 @@ extension Syntax {
     converter: SourceLocationConverter,
     afterLeadingTrivia: Bool = true
   ) -> SourceLocation {
-    let pos = afterLeadingTrivia ?
-      data.positionAfterSkippingLeadingTrivia :
-      data.position
-    return converter.location(for: pos)
+    return base.startLocation(converter: converter,
+      afterLeadingTrivia: afterLeadingTrivia)
   }
 
   /// The ending location, in the provided file, of this Syntax node.
@@ -183,13 +216,8 @@ extension Syntax {
     converter: SourceLocationConverter,
     afterTrailingTrivia: Bool = false
   ) -> SourceLocation {
-    var pos = data.position
-    pos += raw.leadingTriviaLength
-    pos += raw.contentLength
-    if afterTrailingTrivia {
-      pos += raw.trailingTriviaLength
-    }
-    return converter.location(for: pos)
+    return base.endLocation(converter: converter,
+      afterTrailingTrivia: afterTrailingTrivia)
   }
 
   /// The source range, in the provided file, of this Syntax node.
@@ -205,9 +233,9 @@ extension Syntax {
     afterLeadingTrivia: Bool = true,
     afterTrailingTrivia: Bool = false
   ) -> SourceRange {
-    let start = startLocation(converter: converter, afterLeadingTrivia: afterLeadingTrivia)
-    let end = endLocation(converter: converter, afterTrailingTrivia: afterTrailingTrivia)
-    return SourceRange(start: start, end: end)
+    return base.sourceRange(converter: converter,
+      afterLeadingTrivia: afterLeadingTrivia,
+      afterTrailingTrivia: afterTrailingTrivia)
   }
 }
 
@@ -225,8 +253,8 @@ fileprivate func computeLines(
     lines.append(position)
   }
   var curPrefix: SourceLength = .zero
-  tree.forEachToken {
-    curPrefix = $0.forEachLineLength(prefix: curPrefix, body: addLine)
+  for token in tree.tokens {
+    curPrefix = token.forEachLineLength(prefix: curPrefix, body: addLine)
   }
   position += curPrefix
   return (lines, position)

Diff for: Sources/SwiftSyntax/SourceLocation.swift

@@ -10,8 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import Foundation
-
 /// An indicator of whether a Syntax node was found or written in the source.
 ///
 /// A `missing` node does not mean, necessarily, that the source item is