Breaking: Allow comment scanner to rescan tokens (fixes #216)

Template strings and regex patterns need to be rescaned to
not scan tokens as comments.

Fix how comment tokens are removed.

Author: soda0289

Diff for: lib/ast-converter.js

@@ -12,6 +12,7 @@
 //------------------------------------------------------------------------------
 
 const convert = require("./convert"),
+    convertComments = require("./convert-comments").convertComments,
     nodeUtils = require("./node-utils");
 
 //------------------------------------------------------------------------------
@@ -67,10 +68,10 @@ module.exports = (ast, extra) => {
     }
 
     /**
-     * Add the comment nodes to the AST (that were parsed separately in parser.js)
+     * Optionally convert and include all comments in the AST
      */
-    if (extra.comment || extra.attachComment) {
-        estree.comments = extra.comments || [];
+    if (extra.comment) {
+        estree.comments = convertComments(ast, extra.code);
     }
 
     return estree;

Diff for: lib/convert-comments.js

@@ -0,0 +1,147 @@
+/**
+ * @fileoverview Convert comment using TypeScript token scanner
+ * @author James Henry
+ * @copyright jQuery Foundation and other contributors, https://jquery.org/
+ * MIT License
+ */
+
+"use strict";
+
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const ts = require("typescript"),
+    nodeUtils = require("./node-utils");
+
+//------------------------------------------------------------------------------
+// Private
+//------------------------------------------------------------------------------
+
+/**
+ * Converts a TypeScript comment to an Esprima comment.
+ * @param {boolean} block True if it's a block comment, false if not.
+ * @param {string} text The text of the comment.
+ * @param {int} start The index at which the comment starts.
+ * @param {int} end The index at which the comment ends.
+ * @param {Location} startLoc The location at which the comment starts.
+ * @param {Location} endLoc The location at which the comment ends.
+ * @returns {Object} The comment object.
+ * @private
+ */
+function convertTypeScriptCommentToEsprimaComment(block, text, start, end, startLoc, endLoc) {
+    const comment = {
+        type: block ? "Block" : "Line",
+        value: text
+    };
+
+    if (typeof start === "number") {
+        comment.range = [start, end];
+    }
+
+    if (typeof startLoc === "object") {
+        comment.loc = {
+            start: startLoc,
+            end: endLoc
+        };
+    }
+
+    return comment;
+}
+
+/**
+ * Convert comment from TypeScript Triva Scanner.
+ * @param  {Object} triviaScanner TS Scanner
+ * @param  {Object} ast the AST object
+ * @param  {string} code TypeScript code
+ * @returns {ESTreeComment}     the converted ESTreeComment
+ * @private
+ */
+function getCommentFromTriviaScanner(triviaScanner, ast, code) {
+    const kind = triviaScanner.getToken();
+    const isBlock = (kind === ts.SyntaxKind.MultiLineCommentTrivia);
+    const range = {
+        pos: triviaScanner.getTokenPos(),
+        end: triviaScanner.getTextPos(),
+        kind: triviaScanner.getToken()
+    };
+
+    const comment = code.substring(range.pos, range.end);
+    const text = (isBlock) ? comment.replace(/^\/\*/, "").replace(/\*\/$/, "") : comment.replace(/^\/\//, "");
+    const loc = nodeUtils.getLocFor(range.pos, range.end, ast);
+
+    const esprimaComment = convertTypeScriptCommentToEsprimaComment(isBlock, text, range.pos, range.end, loc.start, loc.end);
+
+    return esprimaComment;
+}
+
+//------------------------------------------------------------------------------
+// Public
+//------------------------------------------------------------------------------
+
+/* eslint-disable no-use-before-define */
+module.exports = {
+    convertComments
+};
+
+
+/**
+ * Convert all comments for the given AST.
+ * @param  {Object} ast the AST object
+ * @param  {string} code the TypeScript code
+ * @returns {ESTreeComment[]}     the converted ESTreeComment
+ * @private
+ */
+function convertComments(ast, code) {
+    const comments = [];
+
+    /**
+     * Create a TypeScript Scanner, with skipTrivia set to false so that
+     * we can parse the comments
+     */
+    const triviaScanner = ts.createScanner(ast.languageVersion, false, 0, code);
+
+    let kind = triviaScanner.scan();
+    while (kind !== ts.SyntaxKind.EndOfFileToken) {
+        const start = triviaScanner.getTokenPos();
+        const end = triviaScanner.getTextPos();
+
+        let container = null;
+        switch (kind) {
+            case ts.SyntaxKind.SingleLineCommentTrivia:
+            case ts.SyntaxKind.MultiLineCommentTrivia: {
+                const comment = getCommentFromTriviaScanner(triviaScanner, ast, code);
+
+                comments.push(comment);
+                break;
+            }
+            case ts.SyntaxKind.CloseBraceToken:
+                container = nodeUtils.getNodeContainer(ast, start, end);
+
+                if (
+                    container.kind === ts.SyntaxKind.TemplateMiddle ||
+                    container.kind === ts.SyntaxKind.TemplateTail
+                ) {
+                    kind = triviaScanner.reScanTemplateToken();
+                    continue;
+                }
+                break;
+            case ts.SyntaxKind.SlashToken:
+            case ts.SyntaxKind.SlashEqualsToken:
+                container = nodeUtils.getNodeContainer(ast, start, end);
+
+                if (
+                    container.kind === ts.SyntaxKind.RegularExpressionLiteral
+                ) {
+                    kind = triviaScanner.reScanSlashToken();
+                    continue;
+                }
+                break;
+            default:
+                break;
+        }
+        kind = triviaScanner.scan();
+    }
+
+    return comments;
+}

Diff for: lib/node-utils.js

@@ -187,7 +187,8 @@ module.exports = {
     fixExports,
     getTokenType,
     convertToken,
-    convertTokens
+    convertTokens,
+    getNodeContainer
 };
 /* eslint-enable no-use-before-define */
 
@@ -633,3 +634,35 @@ function convertTokens(ast) {
     walk(ast);
     return result;
 }
+
+/**
+ * Get container token node between range
+ * @param  {Object} ast the AST object
+ * @param {int} start The index at which the comment starts.
+ * @param {int} end The index at which the comment ends.
+ * @returns {TSToken}       typescript container token
+ * @private
+ */
+function getNodeContainer(ast, start, end) {
+    let container = null;
+
+    /**
+     * @param  {TSNode} node the TSNode
+     * @returns {undefined}
+     */
+    function walk(node) {
+        const nodeStart = node.pos;
+        const nodeEnd = node.end;
+
+        if (start >= nodeStart && end <= nodeEnd) {
+            if (isToken(node)) {
+                container = node;
+            } else {
+                node.getChildren().forEach(walk);
+            }
+        }
+    }
+    walk(ast);
+
+    return container;
+}

Diff for: parser.js

@@ -9,6 +9,7 @@
 
 const astNodeTypes = require("./lib/ast-node-types"),
     ts = require("typescript"),
+    convert = require("./lib/ast-converter"),
     semver = require("semver");
 
 const SUPPORTED_TYPESCRIPT_VERSIONS = require("./package.json").devDependencies.typescript;
@@ -51,61 +52,6 @@ function resetExtra() {
     };
 }
 
-/**
- * Converts a TypeScript comment to an Esprima comment.
- * @param {boolean} block True if it's a block comment, false if not.
- * @param {string} text The text of the comment.
- * @param {int} start The index at which the comment starts.
- * @param {int} end The index at which the comment ends.
- * @param {Location} startLoc The location at which the comment starts.
- * @param {Location} endLoc The location at which the comment ends.
- * @returns {Object} The comment object.
- * @private
- */
-function convertTypeScriptCommentToEsprimaComment(block, text, start, end, startLoc, endLoc) {
-    const comment = {
-        type: block ? "Block" : "Line",
-        value: text
-    };
-
-    if (typeof start === "number") {
-        comment.range = [start, end];
-    }
-
-    if (typeof startLoc === "object") {
-        comment.loc = {
-            start: startLoc,
-            end: endLoc
-        };
-    }
-
-    return comment;
-}
-
-/**
- * Returns line and column data for the given start and end positions,
- * for the given AST
- * @param  {Object} start start data
- * @param  {Object} end   end data
- * @param  {Object} ast   the AST object
- * @returns {Object}       the loc data
- */
-function getLocFor(start, end, ast) {
-    const startLoc = ast.getLineAndCharacterOfPosition(start),
-        endLoc = ast.getLineAndCharacterOfPosition(end);
-
-    return {
-        start: {
-            line: startLoc.line + 1,
-            column: startLoc.character
-        },
-        end: {
-            line: endLoc.line + 1,
-            column: endLoc.character
-        }
-    };
-}
-
 //------------------------------------------------------------------------------
 // Parser
 //------------------------------------------------------------------------------
@@ -129,7 +75,6 @@ function parse(code, options) {
     if (typeof options !== "undefined") {
         extra.range = (typeof options.range === "boolean") && options.range;
         extra.loc = (typeof options.loc === "boolean") && options.loc;
-        extra.attachComment = (typeof options.attachComment === "boolean") && options.attachComment;
 
         if (extra.loc && options.source !== null && options.source !== undefined) {
             extra.source = toString(options.source);
@@ -145,10 +90,6 @@ function parse(code, options) {
         if (typeof options.tolerant === "boolean" && options.tolerant) {
             extra.errors = [];
         }
-        if (extra.attachComment) {
-            extra.range = true;
-            extra.comments = [];
-        }
 
         if (options.ecmaFeatures && typeof options.ecmaFeatures === "object") {
             // pass through jsx option
@@ -209,45 +150,8 @@ function parse(code, options) {
 
     const ast = program.getSourceFile(FILENAME);
 
-    if (extra.attachComment || extra.comment) {
-        /**
-         * Create a TypeScript Scanner, with skipTrivia set to false so that
-         * we can parse the comments
-         */
-        const triviaScanner = ts.createScanner(ast.languageVersion, false, 0, code);
-
-        let kind = triviaScanner.scan();
-
-        while (kind !== ts.SyntaxKind.EndOfFileToken) {
-            if (kind !== ts.SyntaxKind.SingleLineCommentTrivia && kind !== ts.SyntaxKind.MultiLineCommentTrivia) {
-                kind = triviaScanner.scan();
-                continue;
-            }
-
-            const isBlock = (kind === ts.SyntaxKind.MultiLineCommentTrivia);
-            const range = {
-                pos: triviaScanner.getTokenPos(),
-                end: triviaScanner.getTextPos(),
-                kind: triviaScanner.getToken()
-            };
-
-            const comment = code.substring(range.pos, range.end);
-            const text = comment.replace("//", "").replace("/*", "").replace("*/", "");
-            const loc = getLocFor(range.pos, range.end, ast);
-
-            const esprimaComment = convertTypeScriptCommentToEsprimaComment(isBlock, text, range.pos, range.end, loc.start, loc.end);
-
-            extra.comments.push(esprimaComment);
-
-            kind = triviaScanner.scan();
-        }
-
-    }
-
-    const convert = require("./lib/ast-converter");
-
+    extra.code = code;
     return convert(ast, extra);
-
 }
 
 //------------------------------------------------------------------------------