Merge pull request swiftlang#172 from dabelknap/debug-stream

Add a debug option to print out the pretty-printing token stream

Author: allevato

Sources/SwiftFormatPrettyPrint/PrettyPrint.swift

@@ -68,6 +68,9 @@ public class PrettyPrinter {
   /// breaks occur in the formatted output.
   private var isDebugMode: Bool
 
+  /// If true, the token stream is printed to the console for debugging purposes.
+  private var printTokenStream: Bool
+
   /// The current index (0..<6) of the color to be used for the next color group.
   private var currentDebugGroupMarkerColor = 0
 
@@ -84,12 +87,13 @@ public class PrettyPrinter {
   /// - Parameters:
   ///   - configuration: The configuration used to decide whitespace or breaking behavior.
   ///   - node: The node to be pretty printed.
-  public init(configuration: Configuration, node: Syntax, isDebugMode: Bool) {
+  public init(configuration: Configuration, node: Syntax, isDebugMode: Bool, printTokenStream: Bool) {
     self.configuration = configuration
     self.tokens = node.makeTokenStream(configuration: configuration)
     self.maxLineLength = configuration.lineLength
     self.isDebugMode = isDebugMode
     self.spaceRemaining = self.maxLineLength
+    self.printTokenStream = printTokenStream
   }
 
   /// Append the input string to the output buffer
@@ -107,6 +111,9 @@ public class PrettyPrinter {
   ///   - token: The token to be printed.
   ///   - length: The length of the token (number of columns).
   private func printToken(token: Token, length: Int) {
+    if self.printTokenStream {
+      printDebugToken(token: token, length: length)
+    }
     assert(length >= 0, "Token lengths must be positive")
     switch token {
 
@@ -415,6 +422,73 @@ public class PrettyPrinter {
     write(Ansi.reset)
   }
 
+  /// Used to track the indentation level for the debug token stream output
+  var debugIndent: Int = 0
+
+  /// Print out the token stream to the console for debugging.
+  ///
+  /// Indentation is applied to make identification of groups easier.
+  private func printDebugToken(token: Token, length: Int) {
+    func printDebugIndent() {
+      print(String(repeating: " ", count: debugIndent), terminator:"")
+    }
+    switch token {
+    case .syntax(let syntax):
+      printDebugIndent()
+      print("[SYNTAX \"\(syntax.text)\" Length: \(length)]")
+
+    case .break(let size, let offset):
+      printDebugIndent()
+      print("[BREAK Size: \(size) Offset: \(offset) Length: \(length)]")
+
+    case .open(let breakstyle, let offset):
+      printDebugIndent()
+      switch breakstyle {
+      case .consistent:
+        print("[OPEN Consistent Offset: \(offset) Length: \(length)]")
+      case .inconsistent:
+        print("[OPEN Inconsistent Offset: \(offset) Length: \(length)]")
+      }
+      debugIndent += 2
+
+    case .close:
+      debugIndent -= 2
+      printDebugIndent()
+      print("[CLOSE]")
+
+    case .newlines(let N, let offset):
+      printDebugIndent()
+      print("[NEWLINES N: \(N) Offset: \(offset) Length: \(length)]")
+
+    case .space(let size):
+      printDebugIndent()
+      print("[SPACE Size: \(size) Length: \(length)]")
+
+    case .reset:
+      print("[RESET]")
+
+    case .comment(let comment):
+      printDebugIndent()
+      switch comment.kind {
+      case .line:
+        print("[COMMENT Line Length: \(length)]")
+      case .docLine:
+        print("[COMMENT DocLine Length: \(length)]")
+      case .block:
+        print("[COMMENT Block Length: \(length)]")
+      case .docBlock:
+        print("[COMMENT DocBlock Length: \(length)]")
+      }
+      printDebugIndent()
+      print(comment.print(indent: debugIndent))
+
+    case .verbatim(let verbatim):
+      printDebugIndent()
+      print("[VERBATIM Length: \(length)]")
+      print(verbatim.print(indent: debugIndent))
+    }
+  }
+
   /// Writes the given number of spaces to the output.
   ///
   /// If debug mode is enabled, spaces are rendered as gray Unicode MIDDLE DOT characters.

Sources/swift-format/Run.swift

@@ -53,7 +53,7 @@ public func lintMain(path: String) -> Int {
 ///
 /// - Parameter path: The absolute path to the source file to be linted.
 /// - Returns: Zero if there were no lint errors, otherwise a non-zero number.
-public func formatMain(path: String, isDebugMode: Bool, prettyPrint: Bool) -> Int {
+public func formatMain(path: String, isDebugMode: Bool, prettyPrint: Bool, printTokenStream: Bool) -> Int {
   let url = URL(fileURLWithPath: path)
 
   let context = Context(
@@ -75,7 +75,8 @@ public func formatMain(path: String, isDebugMode: Bool, prettyPrint: Bool) -> In
       let printer = PrettyPrinter(
         configuration: context.configuration,
         node: formatted,
-        isDebugMode: isDebugMode
+        isDebugMode: isDebugMode,
+        printTokenStream: printTokenStream
       )
       print(printer.prettyPrint(), terminator: "")
     } else {

Sources/swift-format/main.swift

@@ -42,6 +42,7 @@ struct CommandLineOptions: Codable {
   var mode: Mode = .format
   var isDebugMode: Bool = false
   var prettyPrint: Bool = false
+  var printTokenStream: Bool = false
 }
 
 func processArguments(commandName: String, _ arguments: [String]) -> CommandLineOptions {
@@ -95,6 +96,14 @@ func processArguments(commandName: String, _ arguments: [String]) -> CommandLine
   )) {
     $0.prettyPrint = $1
   }
+  binder.bind(
+    option: parser.add(
+      option: "--token-stream",
+      kind: Bool.self,
+      usage: "Print out the pretty-printer token stream."
+  )) {
+    $0.printTokenStream = $1
+  }
 
   var opts = CommandLineOptions()
   do {
@@ -118,7 +127,8 @@ func main(_ arguments: [String]) -> Int32 {
       ret |= formatMain(
         path: path,
         isDebugMode: options.isDebugMode,
-        prettyPrint: options.prettyPrint
+        prettyPrint: options.prettyPrint,
+        printTokenStream: options.printTokenStream
       )
     }
     return Int32(ret)

Tests/SwiftFormatPrettyPrintTests/PrettyPrintTestCase.swift

@@ -22,7 +22,8 @@ public class PrettyPrintTestCase: XCTestCase {
       let printer = PrettyPrinter(
         configuration: context.configuration,
         node: syntax,
-        isDebugMode: false
+        isDebugMode: false,
+        printTokenStream: false
       )
       let output = printer.prettyPrint()
       XCTAssertEqual(expected, output)