Fix: Allow comment scanner to rescan tokens (fixes #216)

Template strings and regex patterns need to be rescaned to
not scan tokens as comments

Author: soda0289

Diff for: lib/ast-converter.js

@@ -191,20 +191,6 @@ function getLocFor(start, end, ast) {
  */
 function getLoc(nodeOrToken, ast) {
     return getLocFor(nodeOrToken.getStart(), nodeOrToken.end, ast);
-    // var start = nodeOrToken.getStart(),
-    //     startLoc = ast.getLineAndCharacterOfPosition(start),
-    //     endLoc = ast.getLineAndCharacterOfPosition(nodeOrToken.end);
-
-    // return {
-    //     start: {
-    //         line: startLoc.line + 1,
-    //         column: startLoc.character
-    //     },
-    //     end: {
-    //         line: endLoc.line + 1,
-    //         column: endLoc.character
-    //     }
-    // };
 }
 
 /**
@@ -451,6 +437,157 @@ function convertTokens(ast) {
     return result;
 }
 
+
+/**
+ * Converts a TypeScript comment to an Esprima comment.
+ * @param {boolean} block True if it's a block comment, false if not.
+ * @param {string} text The text of the comment.
+ * @param {int} start The index at which the comment starts.
+ * @param {int} end The index at which the comment ends.
+ * @param {Location} startLoc The location at which the comment starts.
+ * @param {Location} endLoc The location at which the comment ends.
+ * @returns {Object} The comment object.
+ * @private
+ */
+function convertTypeScriptCommentToEsprimaComment(block, text, start, end, startLoc, endLoc) {
+    var comment = {
+        type: block ? "Block" : "Line",
+        value: text
+    };
+
+    if (typeof start === "number") {
+        comment.range = [start, end];
+    }
+
+    if (typeof startLoc === "object") {
+        comment.loc = {
+            start: startLoc,
+            end: endLoc
+        };
+    }
+
+    return comment;
+}
+
+/**
+ * Convert comment from TypeScript Triva Scanner.
+ * @param  {Object} triviaScanner TS Scanner
+ * @param  {Object} ast the AST object
+ * @param  {string} code TypeScript code
+ * @returns {ESTreeComment}     the converted ESTreeComment
+ * @private
+ */
+function getCommentFromTriviaScanner(triviaScanner, ast, code) {
+    var kind = triviaScanner.getToken();
+    var isBlock = (kind === ts.SyntaxKind.MultiLineCommentTrivia);
+    var range = {
+        pos: triviaScanner.getTokenPos(),
+        end: triviaScanner.getTextPos(),
+        kind: triviaScanner.getToken()
+    };
+
+    var comment = code.substring(range.pos, range.end);
+    var text = comment.replace("//", "").replace("/*", "").replace("*/", "");
+    var loc = getLocFor(range.pos, range.end, ast);
+
+    var esprimaComment = convertTypeScriptCommentToEsprimaComment(isBlock, text, range.pos, range.end, loc.start, loc.end);
+
+    return esprimaComment;
+}
+
+
+/**
+ * Get container token node between range
+ * @param  {Object} ast the AST object
+ * @param {int} start The index at which the comment starts.
+ * @param {int} end The index at which the comment ends.
+ * @returns {TSToken}       typescript container token
+ * @private
+ */
+function getNodeContainer(ast, start, end) {
+    var container = null;
+
+    /**
+     * @param  {TSNode} node the TSNode
+     * @returns {undefined}
+     */
+    function walk(node) {
+        var nodeStart = node.pos;
+        var nodeEnd = node.end;
+
+        if (start >= nodeStart && end <= nodeEnd) {
+            if (isToken(node)) {
+                container = node;
+            } else {
+                node.getChildren().forEach(walk);
+            }
+        }
+    }
+    walk(ast);
+
+    return container;
+}
+
+/**
+ * Convert all comments for the given AST.
+ * @param  {Object} ast the AST object
+ * @param  {string} code the TypeScript code
+ * @returns {ESTreeComment[]}     the converted ESTreeComment
+ * @private
+ */
+function convertComments(ast, code) {
+    var comments = [];
+
+    /**
+     * Create a TypeScript Scanner, with skipTrivia set to false so that
+     * we can parse the comments
+     */
+    var triviaScanner = ts.createScanner(ast.languageVersion, false, 0, code);
+
+    var kind = triviaScanner.scan();
+    while (kind !== ts.SyntaxKind.EndOfFileToken) {
+        var start = triviaScanner.getTokenPos();
+        var end = triviaScanner.getTextPos();
+
+        var container = null;
+        switch (kind) {
+            case ts.SyntaxKind.SingleLineCommentTrivia:
+            case ts.SyntaxKind.MultiLineCommentTrivia:
+                var comment = getCommentFromTriviaScanner(triviaScanner, ast, code);
+
+                comments.push(comment);
+                break;
+            case ts.SyntaxKind.CloseBraceToken:
+                container = getNodeContainer(ast, start, end);
+
+                if (
+                    container.kind === ts.SyntaxKind.TemplateMiddle
+                    || container.kind === ts.SyntaxKind.TemplateTail
+                ) {
+                    kind = triviaScanner.reScanTemplateToken();
+                    continue;
+                }
+                break;
+            case ts.SyntaxKind.SlashToken:
+            case ts.SyntaxKind.SlashEqualsToken:
+                container = getNodeContainer(ast, start, end);
+
+                if (
+                    container.kind === ts.SyntaxKind.RegularExpressionLiteral
+                ) {
+                    kind = triviaScanner.reScanSlashToken();
+                    continue;
+                }
+                break;
+            default:
+                break;
+        }
+        kind = triviaScanner.scan();
+    }
+
+    return comments;
+}
+
 //------------------------------------------------------------------------------
 // Public
 //------------------------------------------------------------------------------
@@ -2088,13 +2225,8 @@ module.exports = function(ast, extra) {
         estree.tokens = convertTokens(ast);
     }
 
-    /**
-     * Add the comment nodes to the AST (that were parsed separately in parser.js)
-     * TODO: Track the progress of https://github.com/eslint/eslint/issues/6724
-     * regarding ESLint itself becoming responsible for attributing comment nodes
-     */
-    if (extra.comment || extra.attachComment) {
-        estree.comments = extra.comments || [];
+    if (extra.comment) {
+        estree.comments = convertComments(ast, extra.code);
     }
 
     return estree;

Diff for: parser.js

@@ -9,7 +9,8 @@
 "use strict";
 
 var astNodeTypes = require("./lib/ast-node-types"),
-    ts = require("typescript");
+    ts = require("typescript"),
+    convert = require("./lib/ast-converter");
 
 var extra;
 
@@ -31,61 +32,6 @@ function resetExtra() {
     };
 }
 
-/**
- * Converts a TypeScript comment to an Esprima comment.
- * @param {boolean} block True if it's a block comment, false if not.
- * @param {string} text The text of the comment.
- * @param {int} start The index at which the comment starts.
- * @param {int} end The index at which the comment ends.
- * @param {Location} startLoc The location at which the comment starts.
- * @param {Location} endLoc The location at which the comment ends.
- * @returns {Object} The comment object.
- * @private
- */
-function convertTypeScriptCommentToEsprimaComment(block, text, start, end, startLoc, endLoc) {
-    var comment = {
-        type: block ? "Block" : "Line",
-        value: text
-    };
-
-    if (typeof start === "number") {
-        comment.range = [start, end];
-    }
-
-    if (typeof startLoc === "object") {
-        comment.loc = {
-            start: startLoc,
-            end: endLoc
-        };
-    }
-
-    return comment;
-}
-
-/**
- * Returns line and column data for the given start and end positions,
- * for the given AST
- * @param  {Object} start start data
- * @param  {Object} end   end data
- * @param  {Object} ast   the AST object
- * @returns {Object}       the loc data
- */
-function getLocFor(start, end, ast) {
-    var startLoc = ast.getLineAndCharacterOfPosition(start),
-        endLoc = ast.getLineAndCharacterOfPosition(end);
-
-    return {
-        start: {
-            line: startLoc.line + 1,
-            column: startLoc.character
-        },
-        end: {
-            line: endLoc.line + 1,
-            column: endLoc.character
-        }
-    };
-}
-
 //------------------------------------------------------------------------------
 // Parser
 //------------------------------------------------------------------------------
@@ -110,7 +56,6 @@ function parse(code, options) {
     if (typeof options !== "undefined") {
         extra.range = (typeof options.range === "boolean") && options.range;
         extra.loc = (typeof options.loc === "boolean") && options.loc;
-        extra.attachComment = (typeof options.attachComment === "boolean") && options.attachComment;
 
         if (extra.loc && options.source !== null && options.source !== undefined) {
             extra.source = toString(options.source);
@@ -126,10 +71,6 @@ function parse(code, options) {
         if (typeof options.tolerant === "boolean" && options.tolerant) {
             extra.errors = [];
         }
-        if (extra.attachComment) {
-            extra.range = true;
-            extra.comments = [];
-        }
 
         if (options.ecmaFeatures && typeof options.ecmaFeatures === "object") {
             // pass through jsx option
@@ -181,41 +122,7 @@ function parse(code, options) {
 
     var ast = program.getSourceFile(FILENAME);
 
-    if (extra.attachComment || extra.comment) {
-        /**
-         * Create a TypeScript Scanner, with skipTrivia set to false so that
-         * we can parse the comments
-         */
-        var triviaScanner = ts.createScanner(ast.languageVersion, false, 0, code);
-
-        var kind = triviaScanner.scan();
-        while (kind !== ts.SyntaxKind.EndOfFileToken) {
-            if (kind !== ts.SyntaxKind.SingleLineCommentTrivia && kind !== ts.SyntaxKind.MultiLineCommentTrivia) {
-                kind = triviaScanner.scan();
-                continue;
-            }
-
-            var isBlock = (kind === ts.SyntaxKind.MultiLineCommentTrivia);
-            var range = {
-                pos: triviaScanner.getTokenPos(),
-                end: triviaScanner.getTextPos(),
-                kind: triviaScanner.getToken()
-            };
-
-            var comment = code.substring(range.pos, range.end);
-            var text = comment.replace("//", "").replace("/*", "").replace("*/", "");
-            var loc = getLocFor(range.pos, range.end, ast);
-
-            var esprimaComment = convertTypeScriptCommentToEsprimaComment(isBlock, text, range.pos, range.end, loc.start, loc.end);
-            extra.comments.push(esprimaComment);
-
-            kind = triviaScanner.scan();
-        }
-
-    }
-
-    var convert = require("./lib/ast-converter");
-
+    extra.code = code;
     return convert(ast, extra);
 }