From 7e131e19bde5249a2242ead68d8a4f859875f33a Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 25 Sep 2020 14:57:09 -0700 Subject: [PATCH 1/4] Add test --- tests/cases/fourslash/processInvalidSyntax1.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/cases/fourslash/processInvalidSyntax1.ts diff --git a/tests/cases/fourslash/processInvalidSyntax1.ts b/tests/cases/fourslash/processInvalidSyntax1.ts new file mode 100644 index 0000000000000..d497a76f54097 --- /dev/null +++ b/tests/cases/fourslash/processInvalidSyntax1.ts @@ -0,0 +1,13 @@ +/// + +// @allowJs: true + +// @Filename: unicode.js +//// obj.𝒜 ; + +// @Filename: forof.js +//// for (obj/**/.prop of arr) { +//// +//// } + +verify.baselineRename("", {}); From 97a2b87b137be2d56b8b3f0ab6ba264084aa172f Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 27 Jan 2021 12:13:20 -0800 Subject: [PATCH 2/4] =?UTF-8?q?Don=E2=80=99t=20create=20missing=20nodes=20?= =?UTF-8?q?for=20identifiers=20that=20would=20be=20valid=20in=20a=20newer?= =?UTF-8?q?=20script=20target?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/compiler/parser.ts | 8 +++- src/compiler/scanner.ts | 39 +++++++++++++++---- src/harness/fourslashImpl.ts | 4 +- .../reference/api/tsserverlibrary.d.ts | 1 + tests/baselines/reference/api/typescript.d.ts | 1 + .../reference/processInvalidSyntax1.baseline | 13 +++++++ .../cases/fourslash/processInvalidSyntax1.ts | 6 +++ 7 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 tests/baselines/reference/processInvalidSyntax1.baseline diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 064ea6788b849..5d1735afedf67 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -1637,8 +1637,8 @@ namespace ts { // with magic property names like '__proto__'. The 'identifiers' object is used to share a single string instance for // each identifier in order to reduce memory consumption. function createIdentifier(isIdentifier: boolean, diagnosticMessage?: DiagnosticMessage, privateIdentifierDiagnosticMessage?: DiagnosticMessage): Identifier { - identifierCount++; if (isIdentifier) { + identifierCount++; const pos = getNodePos(); // Store original token kind if it is not just an Identifier so we can report appropriate error later in type checker const originalKeywordKind = token(); @@ -1652,6 +1652,12 @@ namespace ts { return createIdentifier(/*isIdentifier*/ true); } + if (token() === SyntaxKind.Unknown && scanner.tryScan(() => scanner.reScanInvalidIdentifier() === SyntaxKind.Identifier)) { + // Scanner has already recorded an 'Invalid character' error, so no need to add another from the parser. + return createIdentifier(/*isIdentifier*/ true); + } + + identifierCount++; // Only for end of file because the error gets reported incorrectly on embedded script tags. const reportAtCurrentPosition = token() === SyntaxKind.EndOfFileToken; diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index b15e1ae6468ff..cd6d8c05dd231 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -43,6 +43,7 @@ namespace ts { reScanJsxToken(): JsxTokenSyntaxKind; reScanLessThanToken(): SyntaxKind; reScanQuestionToken(): SyntaxKind; + reScanInvalidIdentifier(): SyntaxKind; scanJsxToken(): JsxTokenSyntaxKind; scanJsDocToken(): JSDocSyntaxKind; scan(): SyntaxKind; @@ -966,6 +967,7 @@ namespace ts { reScanJsxToken, reScanLessThanToken, reScanQuestionToken, + reScanInvalidIdentifier, scanJsxToken, scanJsDocToken, scan, @@ -2041,14 +2043,9 @@ namespace ts { } return token = SyntaxKind.PrivateIdentifier; default: - if (isIdentifierStart(ch, languageVersion)) { - pos += charSize(ch); - while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch); - tokenValue = text.substring(tokenPos, pos); - if (ch === CharacterCodes.backslash) { - tokenValue += scanIdentifierParts(); - } - return token = getIdentifierToken(); + const identifierKind = scanIdentifier(ch, languageVersion); + if (identifierKind) { + return token = identifierKind; } else if (isWhiteSpaceSingleLine(ch)) { pos += charSize(ch); @@ -2066,6 +2063,32 @@ namespace ts { } } + function reScanInvalidIdentifier(): SyntaxKind { + Debug.assert(token === SyntaxKind.Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind.Unknown'."); + pos = tokenPos = startPos; + tokenFlags = 0; + const ch = codePointAt(text, pos); + const identifierKind = scanIdentifier(ch, ScriptTarget.ESNext); + if (identifierKind) { + return token = identifierKind; + } + pos += charSize(ch); + return token = SyntaxKind.Unknown; + } + + function scanIdentifier(startCharacter: number, languageVersion: ScriptTarget) { + let ch = startCharacter; + if (isIdentifierStart(ch, languageVersion)) { + pos += charSize(ch); + while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch); + tokenValue = text.substring(tokenPos, pos); + if (ch === CharacterCodes.backslash) { + tokenValue += scanIdentifierParts(); + } + return getIdentifierToken(); + } + } + function reScanGreaterToken(): SyntaxKind { if (token === SyntaxKind.GreaterThanToken) { if (text.charCodeAt(pos) === CharacterCodes.greaterThan) { diff --git a/src/harness/fourslashImpl.ts b/src/harness/fourslashImpl.ts index 15d0be1a143ba..aa708bfcf4c4f 100644 --- a/src/harness/fourslashImpl.ts +++ b/src/harness/fourslashImpl.ts @@ -1453,9 +1453,9 @@ namespace FourSlash { } public baselineRename(marker: string, options: FourSlashInterface.RenameOptions) { - const position = this.getMarkerByName(marker).position; + const { fileName, position } = this.getMarkerByName(marker); const locations = this.languageService.findRenameLocations( - this.activeFile.fileName, + fileName, position, options.findInStrings ?? false, options.findInComments ?? false, diff --git a/tests/baselines/reference/api/tsserverlibrary.d.ts b/tests/baselines/reference/api/tsserverlibrary.d.ts index 1598efef3279c..70e326c4c998a 100644 --- a/tests/baselines/reference/api/tsserverlibrary.d.ts +++ b/tests/baselines/reference/api/tsserverlibrary.d.ts @@ -3970,6 +3970,7 @@ declare namespace ts { reScanJsxToken(): JsxTokenSyntaxKind; reScanLessThanToken(): SyntaxKind; reScanQuestionToken(): SyntaxKind; + reScanInvalidIdentifier(): SyntaxKind; scanJsxToken(): JsxTokenSyntaxKind; scanJsDocToken(): JSDocSyntaxKind; scan(): SyntaxKind; diff --git a/tests/baselines/reference/api/typescript.d.ts b/tests/baselines/reference/api/typescript.d.ts index f8f86d5b88fb2..5c9fcf765b1f4 100644 --- a/tests/baselines/reference/api/typescript.d.ts +++ b/tests/baselines/reference/api/typescript.d.ts @@ -3970,6 +3970,7 @@ declare namespace ts { reScanJsxToken(): JsxTokenSyntaxKind; reScanLessThanToken(): SyntaxKind; reScanQuestionToken(): SyntaxKind; + reScanInvalidIdentifier(): SyntaxKind; scanJsxToken(): JsxTokenSyntaxKind; scanJsDocToken(): JSDocSyntaxKind; scan(): SyntaxKind; diff --git a/tests/baselines/reference/processInvalidSyntax1.baseline b/tests/baselines/reference/processInvalidSyntax1.baseline new file mode 100644 index 0000000000000..78b16a86a8c3c --- /dev/null +++ b/tests/baselines/reference/processInvalidSyntax1.baseline @@ -0,0 +1,13 @@ +/*====== /tests/cases/fourslash/decl.js ======*/ + +var RENAME = {}; + +/*====== /tests/cases/fourslash/unicode.js ======*/ + +RENAME.𝒜 ; + +/*====== /tests/cases/fourslash/forof.js ======*/ + +for (RENAME.prop of arr) { + +} diff --git a/tests/cases/fourslash/processInvalidSyntax1.ts b/tests/cases/fourslash/processInvalidSyntax1.ts index d497a76f54097..c65ebabe30348 100644 --- a/tests/cases/fourslash/processInvalidSyntax1.ts +++ b/tests/cases/fourslash/processInvalidSyntax1.ts @@ -2,6 +2,12 @@ // @allowJs: true +// Test validates that language service getChildren() doesn't +// crash due to invalid identifier in unicode.js. + +// @Filename: decl.js +//// var obj = {}; + // @Filename: unicode.js //// obj.𝒜 ; From 38cbea666f941ec98b4c6ad19d0697783d1ddfc2 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 27 Jan 2021 12:16:31 -0800 Subject: [PATCH 3/4] Add to test --- .../baselines/reference/processInvalidSyntax1.baseline | 10 +++++++++- tests/cases/fourslash/processInvalidSyntax1.ts | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/baselines/reference/processInvalidSyntax1.baseline b/tests/baselines/reference/processInvalidSyntax1.baseline index 78b16a86a8c3c..01b9d40eb90f6 100644 --- a/tests/baselines/reference/processInvalidSyntax1.baseline +++ b/tests/baselines/reference/processInvalidSyntax1.baseline @@ -2,10 +2,18 @@ var RENAME = {}; -/*====== /tests/cases/fourslash/unicode.js ======*/ +/*====== /tests/cases/fourslash/unicode1.js ======*/ RENAME.𝒜 ; +/*====== /tests/cases/fourslash/unicode2.js ======*/ + +RENAME.¬ ; + +/*====== /tests/cases/fourslash/unicode3.js ======*/ + +RENAME¬ + /*====== /tests/cases/fourslash/forof.js ======*/ for (RENAME.prop of arr) { diff --git a/tests/cases/fourslash/processInvalidSyntax1.ts b/tests/cases/fourslash/processInvalidSyntax1.ts index c65ebabe30348..6a23cfc2b6922 100644 --- a/tests/cases/fourslash/processInvalidSyntax1.ts +++ b/tests/cases/fourslash/processInvalidSyntax1.ts @@ -8,9 +8,15 @@ // @Filename: decl.js //// var obj = {}; -// @Filename: unicode.js +// @Filename: unicode1.js //// obj.𝒜 ; +// @Filename: unicode2.js +//// obj.¬ ; + +// @Filename: unicode3.js +//// obj¬ + // @Filename: forof.js //// for (obj/**/.prop of arr) { //// From 177f88baca5e8b83b26466cdb56696afcbb7e6c5 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 29 Jan 2021 12:47:05 -0800 Subject: [PATCH 4/4] Remove unnecessary assignment --- src/compiler/scanner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index cd6d8c05dd231..7b7fc7b9ce353 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -2073,7 +2073,7 @@ namespace ts { return token = identifierKind; } pos += charSize(ch); - return token = SyntaxKind.Unknown; + return token; // Still `SyntaKind.Unknown` } function scanIdentifier(startCharacter: number, languageVersion: ScriptTarget) {