From 15bc9ac55ca9e427f9c018b061d71a2be8032bff Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Sat, 4 May 2024 20:43:46 -0600 Subject: [PATCH 01/12] Add information to the CONTRIBUTING docs --- CONTRIBUTING.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56912a250f84a..900a3d553b456 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,13 @@ +# Notes on this Buildless TypeScript fork + +I hope that at some point this buildless feature can become a native part of TypeScript, but I also understand that this may not happen for a while, if ever. As such, this fork is designed with the understanding that I may be maintaining this thing for a while, and constantly merging in changes from upstream. To help out with this: +* I've been trying to clearly mark areas of code that I've added or modified with `// BUILDLESS: added` or `// BUILDLESS: modified` comments. You'll also see XML-looking variants of those comments if I want to note that a section of code was added or modified, e.g. `// BUILDLESS: ...some code... /// BUILDLESS: `. +* I don't want to modify too many places in this project so as to avoid merge conflicts. There's already a lot more changes in this project than what I was hoping to ever have. There's some feature requests that I may turn down, not because they wouldn't be nice to have, but because I'm worried about creating and maintaining too many modifications to this codebase. + +Anyways, that's the gist of it. Everything below this point came from the original TypeScript repo, and has some useful information to get up-and-running with this codebase. + +--- + # Instructions for Logging Issues ## 1. Read the FAQ From 5942e028867b806a1f6d71c4c7ae5925546e5f87 Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Thu, 9 May 2024 23:44:16 -0600 Subject: [PATCH 02/12] Improve error reporting with JavaScript syntax used in TS comments --- src/compiler/program.ts | 522 ++++++++++++++++++++++++++++++++++++++++ src/compiler/scanner.ts | 441 +++++++++++++++++++++++++++++++++ 2 files changed, 963 insertions(+) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index dfb878c6265a0..7beb4c2ac4593 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -308,6 +308,7 @@ import { supportedJSExtensionsFlat, SymlinkCache, SyntaxKind, + SyntaxRangeType, // BUILDLESS: added sys, System, toFileNameLowerCase, @@ -316,7 +317,10 @@ import { trace, tracing, tryCast, + TsCommentPosition, // BUILDLESS: added + TsCommentScanner, // BUILDLESS: added TsConfigSourceFile, + TsSyntaxTracker, // BUILDLESS: added TypeChecker, typeDirectiveIsEqualTo, TypeReferenceDirectiveResolutionCache, @@ -3092,12 +3096,34 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg function getJSSyntacticDiagnosticsForFile(sourceFile: SourceFile): DiagnosticWithLocation[] { return runWithCancellationToken(() => { const diagnostics: DiagnosticWithLocation[] = []; + const tsCommentScanner = createTSCommentScanner(sourceFile); // BUILDLESS: added + const tsSyntaxTracker = createTSSyntaxTracker(sourceFile); // BUILDLESS: added walk(sourceFile, sourceFile); forEachChildRecursively(sourceFile, walk, walkArray); + lookForJavaScriptInsideOfTSComments(tsSyntaxTracker, tsCommentScanner.getTSCommentPositions()); // BUILDLESS: added + + // BUILDLESS: + if (options.buildlessEject) { + ejectBuildlessFile(sourceFile, tsCommentScanner.getTSCommentPositions()); + } else if (options.buildlessConvert) { + convertToBuildlessFile(sourceFile, tsSyntaxTracker, tsCommentScanner.getBlockCommentDelimiters()); + } + // BUILDLESS: return diagnostics; function walk(node: Node, parent: Node) { + // BUILDLESS: + const maybeSkip: 'skip' | undefined = walkHelper(node, parent); + if (isLeafNode(node)) { + tsCommentScanner.scanThroughLeafNode(node); + } + return maybeSkip; + } + + function walkHelper(node: Node, parent: Node) { + // BUILDLESS: + // Return directly from the case if the given node doesnt want to visit each child // Otherwise break to visit each child @@ -3105,6 +3131,25 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.Parameter: case SyntaxKind.PropertyDeclaration: case SyntaxKind.MethodDeclaration: + // BUILDLESS: + if ( + (parent.kind === SyntaxKind.Parameter) && + (parent as ParameterDeclaration).name === node && + node.kind === SyntaxKind.Identifier && + (node as Identifier).escapedText === "this" + ) { + // Using a cloned scanner because the child nodes that haven't yet been processed expect the + // scanner to be at its current position. + const clonedTsCommentScanner = tsCommentScanner.clone(); + scanLeavesCheckingIfInTsComment(parent, IsTSSyntax.yes, clonedTsCommentScanner); + // If there's a comma that follows, mark that as needing to be in TS comments as well. + clonedTsCommentScanner.scanUntilAtChar([')', ','], Infinity); + const scannerIndex = clonedTsCommentScanner.getCurrentPos(); + if (sourceFile.text[scannerIndex] === ',') { + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(scannerIndex, scannerIndex + 1); + } + } + // BUILDLESS: if ((parent as ParameterDeclaration | PropertyDeclaration | MethodDeclaration).questionToken === node) { diagnostics.push(createDiagnosticForNode(node, Diagnostics.The_0_modifier_can_only_be_used_in_TypeScript_files, "?")); return "skip"; @@ -3141,10 +3186,52 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.ImportSpecifier: case SyntaxKind.ExportSpecifier: if ((node as ImportOrExportSpecifier).isTypeOnly) { + // BUILDLESS: + if (scanLeavesCheckingIfInTsComment(node)) { + return "skip"; + } + tsCommentScanner.scanUntilAtChar([',', '}'], node.end); + // Mark the comma as TypeScript syntax as well. + const scannerIndex = tsCommentScanner.getCurrentPos(); + if (sourceFile.text[scannerIndex] === ',') { + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(scannerIndex, scannerIndex + 1); + } + // BUILDLESS: diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, isImportSpecifier(node) ? "import...type" : "export...type")); return "skip"; } break; + // BUILDLESS: + case SyntaxKind.ImportDeclaration: { + const importDeclaration = node as ImportDeclaration; + const importClause = importDeclaration.importClause; + if ( + importClause !== undefined && + !importClause.isTypeOnly && + !importClause.name && + importClause.namedBindings !== undefined && + 'elements' in importClause.namedBindings && // eslint-disable-line local/no-in-operator + importClause.namedBindings.elements.every(element => element.isTypeOnly) + ) { + // Using a cloned scanner because the child nodes that haven't yet been processed expect the + // scanner to be at its current position. + const clonedTsCommentScanner = tsCommentScanner.clone(); + if (importDeclaration.modifiers) { + scanLeavesCheckingIfInTsComment(importDeclaration.modifiers, IsTSSyntax.no, clonedTsCommentScanner); + } + // Move past the "t" of "import". + clonedTsCommentScanner.scanUntilPastChar('t', importClause.pos); + // We can now scan the `{ ... }` part of the `import { ... } from ...`. + scanLeavesCheckingIfInTsComment(importClause, IsTSSyntax.yes, clonedTsCommentScanner); + const beforeFromPos = clonedTsCommentScanner.getCurrentPos(); + // Move past the "m" of "from" + clonedTsCommentScanner.scanUntilPastChar('m', importDeclaration.end); + const afterFromPos = clonedTsCommentScanner.getCurrentPos(); + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(beforeFromPos, afterFromPos); + } + break; + } + // BUILDLESS: case SyntaxKind.ImportEqualsDeclaration: diagnostics.push(createDiagnosticForNode(node, Diagnostics.import_can_only_be_used_in_TypeScript_files)); return "skip"; @@ -3187,6 +3274,21 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, enumKeyword)); return "skip"; case SyntaxKind.NonNullExpression: + // BUILDLESS: + // We want to use a cloned scanner, because we don't want our real scanner to actually traverse forwards yet. + // We still need to recursively descend into the node's children to see if they're correctly + // using TS comments where they should + const clonedScanner = tsCommentScanner.clone(); + // Scan through the non-null assertion node's children (omitting itself) + scanLeavesCheckingIfInTsComment((node as NonNullExpression).expression, IsTSSyntax.no, clonedScanner); + // Now scan to the end of this non-null node, which will cause the "!" to be scanned + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(clonedScanner.getCurrentPos(), node.end); + const exclamationMarkInTSComment = clonedScanner.scanTo(node.end); + if (exclamationMarkInTSComment) { + // Don't return "skip", we need to descend into the children to check for problems. + return; + } + // BUILDLESS: diagnostics.push(createDiagnosticForNode(node, Diagnostics.Non_null_assertions_can_only_be_used_in_TypeScript_files)); return "skip"; case SyntaxKind.AsExpression: @@ -3249,6 +3351,20 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.ArrowFunction: // Check type parameters if (nodes === (parent as DeclarationWithTypeParameterChildren).typeParameters) { + // BUILDLESS: + const startOfList = nodes[0]?.pos ?? parent.end; + tsCommentScanner.scanUntilAtChar('<', startOfList) + const startOfParameterListPos = tsCommentScanner.getCurrentPos(); + // Scan the "<" character + const lessThanInTSComment = tsCommentScanner.scanTo(startOfList); + // Scan between "<" and ">" + const contentInTSComment = scanLeavesCheckingIfInTsComment(nodes); + // Scan until just before ">" (which has the same effect as scanning until just after) + const greaterThanInTSComment = tsCommentScanner.scanUntilNextSignificantChar(parent.end); + const endOfParameterListPos = tsCommentScanner.getCurrentPos() + 1; // + 1 to move past the ">" + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(startOfParameterListPos, endOfParameterListPos); + if (lessThanInTSComment && contentInTSComment && greaterThanInTSComment) return "skip"; + // BUILDLESS: diagnostics.push(createDiagnosticForNodeArray(nodes, Diagnostics.Type_parameter_declarations_can_only_be_used_in_TypeScript_files)); return "skip"; } @@ -3291,6 +3407,20 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.TaggedTemplateExpression: // Check type arguments if (nodes === (parent as NodeWithTypeArguments).typeArguments) { + // BUILDLESS: + const startOfList = nodes[0]?.pos ?? parent.end; + tsCommentScanner.scanUntilAtChar('<', startOfList) + const startOfParameterListPos = tsCommentScanner.getCurrentPos(); + // Scan the "<" character + const lessThanInTSComment = tsCommentScanner.scanTo(startOfList); + // Scan between "<" and ">" + const contentInTSComment = scanLeavesCheckingIfInTsComment(nodes); + // Scan until just before ">" (which has the same effect as scanning until just after) + const greaterThanInTSComment = tsCommentScanner.scanUntilNextSignificantChar(parent.end); + const endOfParameterListPos = tsCommentScanner.getCurrentPos() + 1; // + 1 to move past the ">" + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(startOfParameterListPos, endOfParameterListPos); + if (lessThanInTSComment && contentInTSComment && greaterThanInTSComment) return "skip"; + // BUILDLESS: diagnostics.push(createDiagnosticForNodeArray(nodes, Diagnostics.Type_arguments_can_only_be_used_in_TypeScript_files)); return "skip"; } @@ -3338,6 +3468,398 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg function createDiagnosticForNode(node: Node, message: DiagnosticMessage, ...args: DiagnosticArguments): DiagnosticWithLocation { return createDiagnosticForNodeInSourceFile(sourceFile, node, message, ...args); } + + // BUILDLESS: + const enum IsTSSyntax { + /** The node being scanned is not TS syntax */ + no, + /** The node being scanned is TS syntax */ + yes, + /** from the point the scanner currently it as, to the point just after the node is TS syntax. */ + yesAndSoIsEarlierUnscannedContent, + } + + function scanLeavesCheckingIfInTsComment( + rootNodes: Node | NodeArray, + isTSSyntax = IsTSSyntax.yes, + // By default it uses the comment scanner from the outer scope, + // but you can supply an alternative one if, for example, you cloned it. + commentScanner: TsCommentScanner = tsCommentScanner + ): boolean { + const startPos = rootNodes.pos; + const endPos = rootNodes.end; + const rootNodesArray: Node[] = Array.isArray(rootNodes) ? [...rootNodes] : [rootNodes]; + let fullyInTsComment = true; + if (rootNodesArray.length > 0) { + if (isTSSyntax === IsTSSyntax.yes) { + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(startPos, endPos); + } + if (isTSSyntax === IsTSSyntax.yesAndSoIsEarlierUnscannedContent) { + tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(commentScanner.getCurrentPos(), endPos); + const significantTextInTypeScriptComment = commentScanner.scanTo(startPos); + fullyInTsComment &&= significantTextInTypeScriptComment; + } + } + + for (const rootNode of rootNodesArray) { + if (isLeafNode(rootNode)) { + const nodeWasInTSComment = commentScanner.scanThroughLeafNode(rootNode); + fullyInTsComment &&= nodeWasInTSComment; + continue; + } + forEachChildRecursively(rootNode, node => { + if (isLeafNode(node)) { + const nodeWasInTSComment = commentScanner.scanThroughLeafNode(node); + fullyInTsComment &&= nodeWasInTSComment; + } + }); + } + + const endInTSComment = commentScanner.scanTo(endPos); + fullyInTsComment &&= endInTSComment; + return fullyInTsComment; + } + + function lookForJavaScriptInsideOfTSComments( + tsSyntaxTracker: TsSyntaxTracker, + tsCommentPositions: TsCommentPosition[], + ) { + let tsCommentPositionIndex = 0 + for (const syntaxRange of tsSyntaxTracker.getSyntaxRanges()) { + if (syntaxRange.type !== SyntaxRangeType.javaScript) continue; + const jsSyntaxStart = syntaxRange.start; + const jsSyntaxEnd = syntaxRange.end; + while (true) { + const tsCommentPosition: TsCommentPosition | undefined = tsCommentPositions[tsCommentPositionIndex]; + // If undefined, then there are no more TS comments to check for error with, so we can do an early return. + if (tsCommentPosition === undefined) return; + if (tsCommentPosition.colonPos + tsCommentPosition.colonCount > jsSyntaxEnd) break; + tsCommentPositionIndex++; + if (tsCommentPosition.close < jsSyntaxStart) continue; + + let startErrorRange = tsCommentPosition.colonPos + tsCommentPosition.colonCount; + let endErrorRange = tsCommentPosition.close; + if (startErrorRange < jsSyntaxStart) startErrorRange = jsSyntaxStart; + if (endErrorRange > jsSyntaxEnd) endErrorRange = jsSyntaxEnd; + if (startErrorRange === endErrorRange) continue; + + // Ignoring whitespace because the whitespace tracking isn't very accurate. + // Ignoring commas, because whether or not the comma should be there is a fairly difficult question to answer. + // For example, with: + // import type { a /*::, type b, */ c } from '...'; + // it would be a syntax error, because, ignoring the comment, there's no commas between "a" and "c", + // but which of the two commas inside the comment is JavaScript syntax and which is TypeScript? + // TODO: It still would be good to decide on some way to report an error in this kind of scenario. + while (startErrorRange < endErrorRange) { + const char = sourceFile.text[startErrorRange]; + if (char === ' ' || char === '\t' || char === '\n' || char === '\r' || char === ',') { + startErrorRange++; + } else { + break; + } + } + while (endErrorRange > startErrorRange) { + const char = sourceFile.text[endErrorRange]; + if (char === ' ' || char === '\t' || char === '\n' || char === '\r' || char === ',') { + endErrorRange++; + } else { + break; + } + } + if (startErrorRange === endErrorRange) continue; + + diagnostics.push(createFileDiagnostic(sourceFile, startErrorRange, endErrorRange - startErrorRange, Diagnostics.JavaScript_syntax_can_not_be_used_inside_of_a_TS_comment)); + } + } + } + + function ejectBuildlessFile(sourceFile: SourceFile, tsCommentPositions: TsCommentPosition[]) { + const fs = require('fs'); + + interface TextSpan { + start: number + end: number + } + + const text = sourceFile.text; + const spansToRemove: TextSpan[] = [] + // Calculate what comment delimiters to remove from the text + { + for (const { open, colonPos, close, colonCount, containedInnerOpeningBlockComment } of tsCommentPositions) { + // Handle the opening comment + { + let start = open; + let end = colonPos; + // If "::", advance past it. + // Otherwise, its ":", and we don't want to touch that. + if (colonCount === 2) { + end += 2; + if (text[end] === ' ') end++; + } + if (colonCount === 1 && text[start - 1] === ' ') { + start--; + } + // In the case of `x /* :: ?: ... */` we want to ignore the whitespace between `x` and `/*`. + else if (colonCount === 2 && text[end] === '?' && text[end + 1] === ':' && text[start - 1] === ' ') { + start--; + } + // In the case of `x /* :: !: ... */` we want to ignore the whitespace between `x` and `/*`. + else if (colonCount === 2 && text[end] === '!' && text[end + 1] === ':' && text[start - 1] === ' ') { + start--; + } + // In cases such as `import { x /*::, type y */ }` we want to ignore the whitespace between `x` and the comma. + else if (colonCount === 2 && text[colonPos + 2] === ',' && text[start - 1] === ' ') { + start--; + } + // If `/*::` is standing on its own line, remove that line + else if (colonCount === 2 && text[end] === '\n') { + const maybeStartOfLineIndex = tryMoveToStartOfLine(text, start, /*moveOnIndex*/ false); + // We remove the newline after the comment instead of before, + // because there might not be a newline before the comment if this + // is at the start of the file. + if (maybeStartOfLineIndex !== undefined) { + start = maybeStartOfLineIndex; + end++; + } + } + spansToRemove.push({ start, end }); + } + // Handle the closing comment + // In the case of `/*:: ... /* ... */`, we only want to remove the opening delimiter, not the closing one. + if (!containedInnerOpeningBlockComment) { + let start = close; + const end = close + 2; + if (text[start - 1] === ' ') { + start--; + } + // If `*/` is standing on its own line, remove that line + if (colonCount === 2 && [undefined, '\n'].includes(text[end])) { + const maybeStartOfLineIndex = tryMoveToStartOfLine(text, start, /*moveOnIndex*/ true); + if (maybeStartOfLineIndex !== undefined) { + start = maybeStartOfLineIndex; + } + } + spansToRemove.push({ start, end }); + } + } + } + + // Generated updated file text + let newText = ''; + { + spansToRemove.sort((a, b) => a.start - b.start); + let curPos = 0; + for (const spanToRemove of spansToRemove) { + newText += text.slice(curPos, spanToRemove.start); + curPos = Math.max(curPos, spanToRemove.end); + } + newText += text.slice(curPos); + } + + // Can't think of any reason for this condition to not be true, + // but we'll still handled it in case it can happen. + const newPath = sourceFile.path.endsWith('.js') + ? sourceFile.path.slice(0, -'.js'.length) + '.ts' + : sourceFile.path + '.ts'; + + fs.writeFileSync(newPath, newText); + fs.unlinkSync(sourceFile.path); + } + + function convertToBuildlessFile( + sourceFile: SourceFile, + tsSyntaxTracker: TsSyntaxTracker, + blockCommentDelimiters: Map + ) { + const fs = require('fs'); + const text = sourceFile.text; + + interface TSSyntaxRange { + start: number + end: number + openCommentIsFirstOnLine: boolean + closeCommentIsLastOnLine: boolean + multiline: boolean + openCommentIndent: string | undefined + } + + interface TextInsert { + at: number + value: string + } + + // Add metadata to the TS node positions + const tsSyntaxRanges: TSSyntaxRange[] = [...tsSyntaxTracker.getSyntaxRanges()] + .filter(syntaxRange => syntaxRange.type === SyntaxRangeType.typeScript) + .map(tsSyntaxRange => { + const maybeStartOfLineIndex = tryMoveToStartOfLine(text, tsSyntaxRange.start, /*moveOnIndex*/ false); + const openCommentIsFirstOnLine = maybeStartOfLineIndex !== undefined; + const openCommentIndent = maybeStartOfLineIndex !== undefined + ? text.slice(maybeStartOfLineIndex, tsSyntaxRange.start) + : undefined; + + const closeCommentIsLastOnLine = tryMoveToEndOfLine(text, tsSyntaxRange.end) !== undefined; + + const multiline = text.slice(tsSyntaxRange.start, tsSyntaxRange.end).includes('\n'); + + return { ...tsSyntaxRange, openCommentIsFirstOnLine, closeCommentIsLastOnLine, multiline, openCommentIndent }; + }); + + // Merge node-positions that are next to each other. + const tsSyntaxRanges2: TSSyntaxRange[] = []; + if (tsSyntaxRanges.length > 0) { + tsSyntaxRanges2.push(tsSyntaxRanges.shift()!); + for (const tsNodePosition of tsSyntaxRanges) { + const lastNodePosition = tsSyntaxRanges2.at(-1)!; + const textInBetween = text.slice(lastNodePosition.end, tsNodePosition.start); + let merged = false; + + const mergeNodePositionWithPrevious = () => { + lastNodePosition.end = tsNodePosition.end; + lastNodePosition.multiline ||= tsNodePosition.multiline || textInBetween.includes('\n'); + lastNodePosition.closeCommentIsLastOnLine = tsNodePosition.closeCommentIsLastOnLine; + merged = true; + }; + + if (lastNodePosition.openCommentIsFirstOnLine && tsNodePosition.closeCommentIsLastOnLine) { + if (textInBetween.match(/^[ \t\n]*$/g)) { + mergeNodePositionWithPrevious(); + } + } + if (!merged && !lastNodePosition.multiline && !tsNodePosition.multiline) { + if (textInBetween.match(/^[ ]*$/g)) { + mergeNodePositionWithPrevious(); + } + } + + if (!merged) { + tsSyntaxRanges2.push(tsNodePosition); + } + } + } + + // Calculate text to insert + const textInserts: TextInsert[] = []; + for (const tsSyntaxRange of tsSyntaxRanges2) { + let moveDelimitersOnTheirOwnLine = tsSyntaxRange.multiline && tsSyntaxRange.openCommentIsFirstOnLine && tsSyntaxRange.closeCommentIsLastOnLine; + const maybeColonIndex = lookForChar(text, ':', [' '], tsSyntaxRange.start, tsSyntaxRange.end); + if (maybeColonIndex !== undefined) { + moveDelimitersOnTheirOwnLine = false; // `/*:-style comments won't go on a line of their own + textInserts.push({ + at: maybeColonIndex, + value: ' /*' // `/*` with the colon right after makes `/*:`. + }) + } + else if (moveDelimitersOnTheirOwnLine) { + textInserts.push({ + at: tsSyntaxRange.start, + value: `/*::\n${tsSyntaxRange.openCommentIndent}`, + }); + } + else { + const maybeQuestionMarkIndex = lookForChar(text, '?', [' '], tsSyntaxRange.start, tsSyntaxRange.end); + const maybeExclamationMarkIndex = lookForChar(text, '!', [' '], tsSyntaxRange.start, tsSyntaxRange.end); + // Is this a "?:" or "!:" annotation? If so, we'll handle white-space a little differently. + const isTypeAnnotation = ( + (maybeQuestionMarkIndex !== undefined && text[maybeQuestionMarkIndex + 1] === ':') || + (maybeExclamationMarkIndex !== undefined && text[maybeExclamationMarkIndex + 1] === ':') + ); + const spaceBefore = text[tsSyntaxRange.start - 1] === ' '; + textInserts.push({ + at: tsSyntaxRange.start, + value: isTypeAnnotation && !spaceBefore ? ' /*:: ' : '/*:: ', + }); + } + + // Special handling for inner block comments. + // If the range we want to comment out contains a `/* ... */`, we'll + // want to add a `/*::` after the `*/` to bring us back into TS-comment land. + for (let i = tsSyntaxRange.start + 1; i < tsSyntaxRange.end - 1; i++) { + const delimiter = blockCommentDelimiters.get(i); + if (delimiter === BlockCommentDelimiterType.close) { + textInserts.push({ + at: i + 2, + value: '/*::' + }); + } + } + + if (moveDelimitersOnTheirOwnLine) { + textInserts.push({ + at: tsSyntaxRange.end, + value: `\n${tsSyntaxRange.openCommentIndent}*/`, + }); + } + else { + textInserts.push({ + at: tsSyntaxRange.end, + value: ' */', + }); + } + } + + // Generated updated file text + let newText = ''; + { + let curPos = 0; + for (const textInsert of textInserts) { + newText += text.slice(curPos, textInsert.at) + textInsert.value; + curPos = textInsert.at; + } + newText += text.slice(curPos); + } + + // Can't think of any reason for this condition to not be true, + // but we'll still handled it in case it can happen. + const newPath = sourceFile.path.endsWith('.ts') + ? sourceFile.path.slice(0, -'.ts'.length) + '.js' + : sourceFile.path + '.js'; + + fs.writeFileSync(newPath, newText); + fs.unlinkSync(sourceFile.path); + } + + function lookForChar(text: string, charToFind: string, charsToNotBreakOn: string[], startAt: number, stopAt: number): number | undefined { + for (let i = startAt; i < stopAt; i++) { + const char = text[i]; + if (char === charToFind) return i; + if (charsToNotBreakOn.includes(char)) continue; + return undefined; + } + return undefined; + } + + /** + * Returns an index if you can make it to the start by passing through only whitespace. + * Returns undefined if there are other characters in the way. + * The character that "index" is currently on will be ignored, only characters before it will be considered.. + * @param moveOnIndex if false, returns the index right after the new line. + */ + function tryMoveToStartOfLine(text: string, index: number, moveOnIndex: boolean) { + while (true) { + index--; + if (text[index] === ' ' || text[index] === '\t') continue; + if (text[index] === undefined) return index + 1; + if (text[index] === '\n') return moveOnIndex ? index : index + 1; + return undefined; + } + } + + /** + * Returns an index if you can make it to the end by passing through only whitespace. + * Returns undefined if there are other characters in the way. + */ + function tryMoveToEndOfLine(text: string, index: number) { + index--; + while (true) { + index++; + if (text[index] === ' ' || text[index] === '\t') continue; + if (text[index] === undefined) return index - 1; + if (text[index] === '\n') return index; + return undefined; + } + } + // BUILDLESS: }); } diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 5d93e57ebe660..c9b0a822ceac2 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -3917,6 +3917,447 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean } } +// BUILDLESS: +const enum TsCommentScannerState { + notInComment, + inBlockComment, + inTSBlockComment, +} + +export const enum BlockCommentDelimiterType { + openNonTsComment, // A `/*` not followed by colons + openTsCommentWithoutColons, // Just the `/*` part of `/*:` or `/*::`. + singleColonForOpenTsComment, // Just the `:` part of `/*:` + doubleColonForOpenTsComment, // Just the `::` part of `/*::` + close, // Any `*/`, even if it pertains to a non-TS comment. +} + +export interface TsCommentPosition { + open: number + colonPos: number + close: number + colonCount: 1 | 2 + // e.g. `/*:: ... /* ... */` contains an inner opening block comment. The `*/` is closing both of them at the same time. + // When ejecting something that looks like this, we only want to remove the `/*::`, not the `*/`. + containedInnerOpeningBlockComment: boolean +} + +export const enum SyntaxRangeType { + typeScript, + javaScript, + whitespace, +} + +export interface SyntaxRange { + start: number + end: number + type: SyntaxRangeType +} + +// Records the position of interesting comment-related information. +// When an openTSCommentWithoutColons is recorded, it will be followed by +// either a singleColonForOpenTSComment or doubleColonForOpenTSComment. +type BlockCommentDelimiterPositions = Map + +export function createTSCommentScanner( + { text }: SourceFileLike, + _pos?: number, + _state?: TsCommentScannerState, + _delimiterPositions?: BlockCommentDelimiterPositions, +) { + let pos = _pos ?? 0; + let state = _state ?? TsCommentScannerState.notInComment; + // This is a shared value that all clones share and mutate. + // Used for converting JS files to TS. + const delimiterPositions: BlockCommentDelimiterPositions = _delimiterPositions ?? new Map(); + const tsCommentScanner = { + getCurrentPos() { + return pos; + }, + /** + * The state of the scanner is cloned, so you can move forwards with the clone to inspect something in the future + * without effecting the current scanner. + * (with some exceptions for tracking information that's needed for code transformations) + */ + clone() { + // console.log(`CLONING SCANNER - expect duplicate scans past this point.`) // BUILDLESS: DEBUG: comment-scan + return createTSCommentScanner({ text }, pos, state, _delimiterPositions); + }, + /** + * Scans from where we left off to the start of this node. + * Then continues scanning into the node until non-whitespace, non-comments are hit, + * at which point the scanner position will just jump to the end of the node. + * This is only smart enough to scan through whitespace and comments - if it + * were to receive something like `'/*'`, it would get tripped up. This means + * that scanToNode() should be called, in order, on every leaf node in a file. + * + * It is designed to be safe (and do nothing) if you re-scan the same node multiple times. + * + * @returns true if the just-scanned node was inside a TS comment, otherwise returns false. + * Also returns false when given a node that couldn't be scanned, because the scanner already passed that point. + */ + scanThroughLeafNode(node: { kind: SyntaxKind, pos: number, end: number }): boolean { + // console.log(`SCAN ${SyntaxKind[(node as any).kind]} from:${pos} to-node-start:${node.pos} hard-limit-at-node-end:${node.end}${node.end < pos ? ' FAIL' : ''}`); // BUILDLESS: DEBUG: comment-scan + if (node.end < pos) return false; + commentScan(text, pos, node.pos, /*stopOnSignificantText*/ false, onCommentDelimiter); + commentScan(text, node.pos, node.end, /*stopOnSignificantText*/ true, onCommentDelimiter); + pos = node.end; + return state === TsCommentScannerState.inTSBlockComment; + }, + /** + * Scans to the provided position. + * First is scans past any whitespace and normal comments to find where significant text starts, then it'll + * scan through the significant text until it reaches the end. + * Once it finds significant text, all text going forwards is treated as significant. + * + * @returns true if the scanned significant text was in TypeScript comments, otherwise returns false. + * Also returns false when nothing could be scanned, because the scanner already passed that point. + */ + scanTo(end: number): boolean { + // console.log(`SCAN to ${end}.${end < pos ? ' FAIL' : ''}`) // BUILDLESS: DEBUG: comment-scan + if (end < pos) return false; + const significantTextAt = commentScan(text, pos, end, /*stopOnSignificantText*/ true, onCommentDelimiter); + let allInTSComment = state === TsCommentScannerState.inTSBlockComment; + commentScan(text, significantTextAt, end, /*stopOnSignificantText*/ false, (pos, commentDelimiterType) => { + allInTSComment = false; + onCommentDelimiter(pos, commentDelimiterType); + }); + pos = end; + return allInTSComment; + }, + /** + * Stops scanning as soon as non-whitespace/not-generic-comment character is encountered. + * @param max Mostly a failsafe - if something goes wrong, we'll stop jumping at this point. + * + * @return true if in TS comment where it stopped at, which typically means the following node is in a TypeScript comment. + * Otherwise, returns false. Also returns false when nothing could be scanned, because the scanner already passed that point. + */ + scanUntilNextSignificantChar(max: number) { + // console.log(`SCAN UNTIL NEXT SIGNIFICANT CHAR (max: ${max})${max < pos ? ' FAIL' : ''}`); // BUILDLESS: DEBUG: comment-scan + if (max < pos) return false; + commentScan(text, pos, max, /*stopOnSignificantText*/ true, onCommentDelimiter); + return state === TsCommentScannerState.inTSBlockComment; + }, + /** + * @param char The character(s) to break on. whitespace and characters found as part of comments won't be matched. + * @param max Mostly a failsafe - if something goes wrong, we'll stop jumping at this point. + */ + scanUntilAtChar(char: string | string[], max: number) { + // console.log(`SCAN UNTIL AT ${JSON.stringify(char)}, (max: ${max})}`); // BUILDLESS: DEBUG: comment-scan + const charList: string[] = Array.isArray(char) ? char : [char]; + while (pos < max) { + commentScan(text, pos, max, /*stopOnSignificantText*/ true, onCommentDelimiter) + if (charList.includes(text[pos])) return; + pos++; + } + }, + /** + * @param char The character(s) to break on. whitespace and characters found as part of comments won't be matched. + * @param max Mostly a failsafe - if something goes wrong, we'll stop jumping at this point. + */ + scanUntilPastChar(char: string | string[], max: number) { + // console.log(`SCAN UNTIL PAST ${JSON.stringify(char)}, (max: ${max})}`); // BUILDLESS: DEBUG: comment-scan + const charList: string[] = Array.isArray(char) ? char : [char]; + while (pos < max) { + commentScan(text, pos, max, /*stopOnSignificantText*/ true, onCommentDelimiter) + if (charList.includes(text[pos])) { + // Move one past + pos++; + return; + } + pos++; + } + }, + getBlockCommentDelimiters(): Map { + return delimiterPositions; + }, + /** + * Returns all TS-comment pairs in the file. + * Prerequisites: + * - The whole source file has been scanned. + */ + getTSCommentPositions(): TsCommentPosition[] { + const result: TsCommentPosition[] = []; + const delimiters = [...delimiterPositions] + .map(([pos, delimiterType]) => ({ pos, type: delimiterType })) + .sort((a, b) => a.pos - b.pos); + + let tsCommentPosition: Partial = { + containedInnerOpeningBlockComment: false + }; + let inTSComment = false; + for (const delimiter of delimiters) { + if (!inTSComment && delimiter.type === BlockCommentDelimiterType.openTsCommentWithoutColons) { + tsCommentPosition.open = delimiter.pos; + inTSComment = true; + } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.singleColonForOpenTsComment) { + tsCommentPosition.colonPos = delimiter.pos; + tsCommentPosition.colonCount = 1; + } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.doubleColonForOpenTsComment) { + tsCommentPosition.colonPos = delimiter.pos; + tsCommentPosition.colonCount = 2; + } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.openNonTsComment) { + tsCommentPosition.containedInnerOpeningBlockComment = true; + } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.close) { + tsCommentPosition.close = delimiter.pos; + result.push(tsCommentPosition as TsCommentPosition); + tsCommentPosition = { + containedInnerOpeningBlockComment: false + }; + inTSComment = false; + } + } + + return result; + }, + }; + + function onCommentDelimiter(at: number, commentDelimiterType: BlockCommentDelimiterType) { + delimiterPositions.set(at, commentDelimiterType); + if (commentDelimiterType === BlockCommentDelimiterType.close) { + state = TsCommentScannerState.notInComment; + } else if (commentDelimiterType === BlockCommentDelimiterType.openTsCommentWithoutColons) { + state = TsCommentScannerState.inTSBlockComment + } else if (commentDelimiterType === BlockCommentDelimiterType.openNonTsComment) { + state = TsCommentScannerState.inBlockComment; + } + } + + return tsCommentScanner; +} + +/** + * Tracks which spans in a file pertain to TypeScript syntax. + * This is used to convert TypeScript files to JavaScript + TS comments. + */ +export function createTSSyntaxTracker({ text }: SourceFileLike) { + const whitespaceSyntaxRanges: SyntaxRange[] = []; + const tsSyntaxRanges: SyntaxRange[] = []; + return { + /** + * This range is TypeScript syntax, and if we're converting TS to buildless JS files, + * this range should be moved into TS comments. + * whitespace/comments at the start will get recorded as whitespace. + */ + skipWhitespaceThenMarkRangeAsTS(start: number, end: number) { + // Skip past whitespace/comments + // console.log('(iterating as part of mark-range-as-ts - this does not alter a scanner instance)'); // BUILDLESS: DEBUG: comment-scan + const realStart = commentScan(text, start, end, /*stopOnSignificantText*/ true); + if (start < realStart) { + whitespaceSyntaxRanges.push({ start, end: realStart, type: SyntaxRangeType.whitespace }); + } + // console.log(`Mark TypeScript range - from ${realStart} to ${end} (original start: ${start})`); // BUILDLESS: DEBUG: comment-scan + if (realStart < end) { + tsSyntaxRanges.push({ start: realStart, end, type: SyntaxRangeType.typeScript }); + } + }, + /** + * Returns where all TypeScript and JavaScript syntax is located in the file. + * The returned ranges will either be marked as a TypeScript range or a JavaScript range. + * Ranges of whitespace won't be returned. + */ + *getSyntaxRanges(): Generator { + tsSyntaxRanges.sort((a, b) => a.start - b.start); + whitespaceSyntaxRanges.sort((a, b) => a.start - b.start); + + let whitespaceRangeIndex = 0; + let tsRangeIndex = 0; + let pos = 0; + while (pos < text.length) { + const whitespaceStart = whitespaceSyntaxRanges[whitespaceRangeIndex]?.start ?? Infinity; + const tsStart = tsSyntaxRanges[tsRangeIndex]?.start ?? Infinity; + + if (pos === tsStart) { + let end = -Infinity; + // Merge overlapping TS ranges + while (true) { + end = Math.max(end, tsSyntaxRanges[tsRangeIndex].end); + tsRangeIndex++; + if (tsSyntaxRanges[tsRangeIndex] === undefined || tsSyntaxRanges[tsRangeIndex].start > end) { + break; + } + } + yield { start: tsStart, end, type: SyntaxRangeType.typeScript }; + pos = end; + } + // `pos` might be past `whitespaceStart` because we may have found a TS range in + // the middle of this whitespace range and TS ranges trump whitespace ranges. + // This would eventually place `pos` right after the TS range, + // which is in the middle of the whitespace range. + else if (pos >= whitespaceStart) { + const whitespaceEnd = whitespaceSyntaxRanges[whitespaceRangeIndex].end; + if (whitespaceEnd < tsStart) { + pos = Math.max(pos, whitespaceEnd); + whitespaceRangeIndex++; + } else { + pos = tsStart; + } + } + else { + const end = Math.min(tsStart, whitespaceStart, text.length); + yield { start: pos, end, type: SyntaxRangeType.javaScript }; + pos = end; + } + } + }, + }; +} + +/** + * All characters will be scanned until `end` is reached. + * + * @param stopOnSignificantText If a non-whitespace, non-comment character is encountered, stop scanning. + * + * @returns the index where it stopped scanning. + */ +function commentScan(text: string, start: number, end: number, stopOnSignificantText: boolean, onCommentDelimiter?: (pos: number, commentDelimiterType: BlockCommentDelimiterType) => void) { + let pos = start; + // A lot of this is copy-pasted from scan(), with some modifications. + while (true) { + const ch = codePointAt(text, pos); + if (pos >= end) { + return pos; + } + // console.log(` char-scan ${JSON.stringify(text[pos])} (at ${pos})${pos >= end ? ' (past soft limit)' : ''}`) // BUILDLESS: DEBUG: comment-scan + + if (pos === 0) { + // Special handling for shebang + if (ch === CharacterCodes.hash && isShebangTrivia(text, pos)) { + // console.log(' she-bang skip'); // BUILDLESS: DEBUG: comment-scan + pos = scanShebangTrivia(text, pos); + Debug.assertLessThan(pos, end); + continue; + } + } + + switch (ch) { + case CharacterCodes.lineFeed: + case CharacterCodes.carriageReturn: + pos++; + continue; + case CharacterCodes.tab: + case CharacterCodes.verticalTab: + case CharacterCodes.formFeed: + case CharacterCodes.space: + case CharacterCodes.nonBreakingSpace: + case CharacterCodes.ogham: + case CharacterCodes.enQuad: + case CharacterCodes.emQuad: + case CharacterCodes.enSpace: + case CharacterCodes.emSpace: + case CharacterCodes.threePerEmSpace: + case CharacterCodes.fourPerEmSpace: + case CharacterCodes.sixPerEmSpace: + case CharacterCodes.figureSpace: + case CharacterCodes.punctuationSpace: + case CharacterCodes.thinSpace: + case CharacterCodes.hairSpace: + case CharacterCodes.zeroWidthSpace: + case CharacterCodes.narrowNoBreakSpace: + case CharacterCodes.mathematicalSpace: + case CharacterCodes.ideographicSpace: + case CharacterCodes.byteOrderMark: + pos++; + continue; + case CharacterCodes.asterisk: + if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { + onCommentDelimiter?.(pos, BlockCommentDelimiterType.close) + // console.log(' closing comment skip'); // BUILDLESS: DEBUG: comment-scan + pos += 2; + continue; + } else { + pos += 1; + continue; + } + case CharacterCodes.slash: + // Single-line comment + if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { + pos += 2; + + while (pos < end) { + if (isLineBreak(text.charCodeAt(pos))) { + break; + } + pos++; + } + // console.log(' single line comment skip'); // BUILDLESS: DEBUG: comment-scan + continue; + } + // Multi-line comment + if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { + const saveStartPos = pos; + pos += 2; + + let isTsComment = false; + while (pos < end) { + if (text.charCodeAt(pos) === CharacterCodes.space) { + // continue on + } else if (text.charCodeAt(pos) === CharacterCodes.colon) { + isTsComment = true; + break; + } else { + break; + } + pos++; + } + + if (isTsComment) { + const colonCount = + text.charCodeAt(pos + 1) !== CharacterCodes.colon ? 1 : + text.charCodeAt(pos + 2) !== CharacterCodes.colon ? 2 : + 3; // Represents 3 or more + + if (colonCount === 1 || colonCount === 2) { + onCommentDelimiter?.(saveStartPos, BlockCommentDelimiterType.openTsCommentWithoutColons) + onCommentDelimiter?.( + pos, + colonCount === 1 + ? BlockCommentDelimiterType.singleColonForOpenTsComment + : BlockCommentDelimiterType.doubleColonForOpenTsComment + ); + pos += colonCount; + // console.log(' ts opening comment skip'); // BUILDLESS: DEBUG: comment-scan + continue; + } else { // `/*::: ... */` + // A triple-colon isn't allowed. + // We'll just say that this isn't a recognized TypeScript comment. + pos += 3; + } + } + + onCommentDelimiter?.(saveStartPos, BlockCommentDelimiterType.openNonTsComment) + // console.log(' non-ts block comment skip'); // BUILDLESS: DEBUG: comment-scan + while (pos < end) { + const ch = text.charCodeAt(pos); + + if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { + onCommentDelimiter?.(pos, BlockCommentDelimiterType.close) + pos += 2; + break; + } + + pos++; + } + continue; + } + + pos++; + continue; + default: + if (pos >= end || stopOnSignificantText) { + return pos; + } + pos++; + continue; + } + } +} + +export type TsCommentScanner = ReturnType; +export type TsSyntaxTracker = ReturnType; +// BUILDLESS: + +/** @internal */ function codePointAt(s: string, i: number): number { // TODO(jakebailey): this is wrong and should have ?? 0; but all users are okay with it return s.codePointAt(i)!; From ce033fafec3028e8f38527683b23949e5756a4ba Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Fri, 10 May 2024 00:05:33 -0600 Subject: [PATCH 03/12] Fix style issues --- src/compiler/program.ts | 3 +- src/compiler/scanner.ts | 262 ++++++++++++++++++++++++++++++++-------- 2 files changed, 213 insertions(+), 52 deletions(-) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 7beb4c2ac4593..12de2decc4959 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -3105,7 +3105,8 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg // BUILDLESS: if (options.buildlessEject) { ejectBuildlessFile(sourceFile, tsCommentScanner.getTSCommentPositions()); - } else if (options.buildlessConvert) { + } + else if (options.buildlessConvert) { convertToBuildlessFile(sourceFile, tsSyntaxTracker, tsCommentScanner.getBlockCommentDelimiters()); } // BUILDLESS: diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index c9b0a822ceac2..ca5ea537a522a 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -128,6 +128,60 @@ export interface Scanner { tryScan(callback: () => T): T; } +// BUILDLESS: +export const enum BlockCommentDelimiterType { + openNonTsComment, // A `/*` not followed by colons + openTsCommentWithoutColons, // Just the `/*` part of `/*:` or `/*::`. + singleColonForOpenTsComment, // Just the `:` part of `/*:` + doubleColonForOpenTsComment, // Just the `::` part of `/*::` + close, // Any `*/`, even if it pertains to a non-TS comment. +} + +export interface TsCommentPosition { + open: number; + colonPos: number; + close: number; + colonCount: 1 | 2; + // e.g. `/*:: ... /* ... */` contains an inner opening block comment. The `*/` is closing both of them at the same time. + // When ejecting something that looks like this, we only want to remove the `/*::`, not the `*/`. + containedInnerOpeningBlockComment: boolean; +} + +export interface SyntaxRange { + start: number; + end: number; + type: SyntaxRangeType; +} + +export const enum SyntaxRangeType { + typeScript, + javaScript, + whitespace, +} + +// Records the position of interesting comment-related information. +// When an openTSCommentWithoutColons is recorded, it will be followed by +// either a singleColonForOpenTSComment or doubleColonForOpenTSComment. +type BlockCommentDelimiterPositions = Map + +export interface TsCommentScanner { + getCurrentPos(): number; + clone(): TsCommentScanner; + scanThroughLeafNode(node: { kind: SyntaxKind, pos: number, end: number }): boolean; + scanTo(end: number): boolean; + scanUntilNextSignificantChar(max: number): boolean; + scanUntilAtChar(char: string | string[], max: number): void; + scanUntilPastChar(char: string | string[], max: number): void; + getBlockCommentDelimiters(): Map; + getTSCommentPositions(): TsCommentPosition[]; +} + +export interface TsSyntaxTracker { + skipWhitespaceThenMarkRangeAsTS(start: number, end: number): void; + getSyntaxRanges(): Generator +} +// BUILDLESS: + /** @internal */ export const textToKeywordObj: MapLike = { abstract: SyntaxKind.AbstractKeyword, @@ -680,6 +734,47 @@ export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boole } if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { pos += 2; + // BUILDLESS: + let isTsComment = false; + while (pos < text.length) { + if (text.charCodeAt(pos) === CharacterCodes.space) { + // continue on + } + else if (text.charCodeAt(pos) === CharacterCodes.colon) { + isTsComment = true; + break; + } + else { + break; + } + pos++; + } + + if (isTsComment) { + const colonCount = + text.charCodeAt(pos + 1) !== CharacterCodes.colon ? 1 : + text.charCodeAt(pos + 2) !== CharacterCodes.colon ? 2 : + 3; // Represents 3 or more + + if (colonCount === 1 || colonCount === 2) { + if (colonCount === 1) { // `/*: ... */` + // Not moving the pos variable. + // Instead, it will stay positioned right before the ":", + // which will let the ":" get treated as its own token. + break; + } + else { // `/*:: ... */` + pos += 2; + continue; + } + } + else { // `/*::: ... */` + // A triple-colon isn't allowed. + // We'll just say that this isn't a recognized TypeScript comment. + pos += 3; + } + } + // BUILDLESS: while (pos < text.length) { if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { pos += 2; @@ -864,6 +959,16 @@ function iterateCommentRanges(reduce: boolean, text: string, pos: number, case CharacterCodes.space: pos++; continue; + // BUILDLESS: + case CharacterCodes.asterisk: + if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { + pos += 2; + continue; + } + else { + break scan; + } + // BUILDLESS: case CharacterCodes.slash: const nextChar = text.charCodeAt(pos + 1); let hasTrailingNewLine = false; @@ -881,6 +986,46 @@ function iterateCommentRanges(reduce: boolean, text: string, pos: number, } } else { + // BUILDLESS: + let isTsComment = false; + while (pos < text.length) { + if (text.charCodeAt(pos) === CharacterCodes.space) { + // continue on + } + else if (text.charCodeAt(pos) === CharacterCodes.colon) { + isTsComment = true; + break; + } + else { + break; + } + pos++; + } + + // We will ignore TS comments + if (isTsComment) { + const colonCount = + text.charCodeAt(pos + 1) !== CharacterCodes.colon ? 1 : + text.charCodeAt(pos + 2) !== CharacterCodes.colon ? 2 : + 3; // Represents 3 or more + + if (colonCount === 1 || colonCount === 2) { + if (colonCount === 1) { // `/*: ... */` + // The single ":" counts as a significant token, so break out of the loop. + break scan; + } + else { // `/*:: ... */` + pos += 2; + continue; + } + } + else { // `/*::: ... */` + // A triple-colon isn't allowed. + // We'll just say that this isn't a recognized TypeScript comment. + pos += 3; + } + } + // BUILDLESS: while (pos < text.length) { if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { pos += 2; @@ -2040,6 +2185,49 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean pos += 2; const isJSDoc = charCodeUnchecked(pos) === CharacterCodes.asterisk && charCodeUnchecked(pos + 1) !== CharacterCodes.slash; + // BUILDLESS: + let isTsComment = false; + while (pos < end) { + if (text.charCodeAt(pos) === CharacterCodes.space) { + // continue on + } + else if (text.charCodeAt(pos) === CharacterCodes.colon) { + isTsComment = true; + break; + } + else { + break; + } + pos++; + } + + if (isTsComment) { + const colonCount = + text.charCodeAt(pos + 1) !== CharacterCodes.colon ? 1 : + text.charCodeAt(pos + 2) !== CharacterCodes.colon ? 2 : + 3; // Represents 3 or more + + if (colonCount === 1 || colonCount === 2) { + if (colonCount === 1) { // `/*: ... */` + // Not moving the pos variable. + // Instead, it will stay positioned right before the ":", + // which will let the ":" get parsed later as its own token. + } + else { // `/*:: ... */` + pos += 2; + } + + tokenFlags |= TokenFlags.enteringTSComment; + continue; + } + else { // `/*::: ... */` + // A triple-colon isn't allowed. + // We'll just say that this isn't a recognized TypeScript comment. + pos += 3; + } + } + // BUILDLESS: + let commentClosed = false; let lastLineStart = tokenStart; while (pos < end) { @@ -3924,53 +4112,18 @@ const enum TsCommentScannerState { inTSBlockComment, } -export const enum BlockCommentDelimiterType { - openNonTsComment, // A `/*` not followed by colons - openTsCommentWithoutColons, // Just the `/*` part of `/*:` or `/*::`. - singleColonForOpenTsComment, // Just the `:` part of `/*:` - doubleColonForOpenTsComment, // Just the `::` part of `/*::` - close, // Any `*/`, even if it pertains to a non-TS comment. -} - -export interface TsCommentPosition { - open: number - colonPos: number - close: number - colonCount: 1 | 2 - // e.g. `/*:: ... /* ... */` contains an inner opening block comment. The `*/` is closing both of them at the same time. - // When ejecting something that looks like this, we only want to remove the `/*::`, not the `*/`. - containedInnerOpeningBlockComment: boolean -} - -export const enum SyntaxRangeType { - typeScript, - javaScript, - whitespace, -} - -export interface SyntaxRange { - start: number - end: number - type: SyntaxRangeType -} - -// Records the position of interesting comment-related information. -// When an openTSCommentWithoutColons is recorded, it will be followed by -// either a singleColonForOpenTSComment or doubleColonForOpenTSComment. -type BlockCommentDelimiterPositions = Map - export function createTSCommentScanner( { text }: SourceFileLike, _pos?: number, _state?: TsCommentScannerState, _delimiterPositions?: BlockCommentDelimiterPositions, -) { +): TsCommentScanner { let pos = _pos ?? 0; let state = _state ?? TsCommentScannerState.notInComment; // This is a shared value that all clones share and mutate. // Used for converting JS files to TS. const delimiterPositions: BlockCommentDelimiterPositions = _delimiterPositions ?? new Map(); - const tsCommentScanner = { + const tsCommentScanner: TsCommentScanner = { getCurrentPos() { return pos; }, @@ -4090,15 +4243,19 @@ export function createTSCommentScanner( if (!inTSComment && delimiter.type === BlockCommentDelimiterType.openTsCommentWithoutColons) { tsCommentPosition.open = delimiter.pos; inTSComment = true; - } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.singleColonForOpenTsComment) { + } + else if (inTSComment && delimiter.type === BlockCommentDelimiterType.singleColonForOpenTsComment) { tsCommentPosition.colonPos = delimiter.pos; tsCommentPosition.colonCount = 1; - } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.doubleColonForOpenTsComment) { + } + else if (inTSComment && delimiter.type === BlockCommentDelimiterType.doubleColonForOpenTsComment) { tsCommentPosition.colonPos = delimiter.pos; tsCommentPosition.colonCount = 2; - } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.openNonTsComment) { + } + else if (inTSComment && delimiter.type === BlockCommentDelimiterType.openNonTsComment) { tsCommentPosition.containedInnerOpeningBlockComment = true; - } else if (inTSComment && delimiter.type === BlockCommentDelimiterType.close) { + } + else if (inTSComment && delimiter.type === BlockCommentDelimiterType.close) { tsCommentPosition.close = delimiter.pos; result.push(tsCommentPosition as TsCommentPosition); tsCommentPosition = { @@ -4116,9 +4273,11 @@ export function createTSCommentScanner( delimiterPositions.set(at, commentDelimiterType); if (commentDelimiterType === BlockCommentDelimiterType.close) { state = TsCommentScannerState.notInComment; - } else if (commentDelimiterType === BlockCommentDelimiterType.openTsCommentWithoutColons) { + } + else if (commentDelimiterType === BlockCommentDelimiterType.openTsCommentWithoutColons) { state = TsCommentScannerState.inTSBlockComment - } else if (commentDelimiterType === BlockCommentDelimiterType.openNonTsComment) { + } + else if (commentDelimiterType === BlockCommentDelimiterType.openNonTsComment) { state = TsCommentScannerState.inBlockComment; } } @@ -4130,7 +4289,7 @@ export function createTSCommentScanner( * Tracks which spans in a file pertain to TypeScript syntax. * This is used to convert TypeScript files to JavaScript + TS comments. */ -export function createTSSyntaxTracker({ text }: SourceFileLike) { +export function createTSSyntaxTracker({ text }: SourceFileLike): TsSyntaxTracker { const whitespaceSyntaxRanges: SyntaxRange[] = []; const tsSyntaxRanges: SyntaxRange[] = []; return { @@ -4265,7 +4424,8 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant // console.log(' closing comment skip'); // BUILDLESS: DEBUG: comment-scan pos += 2; continue; - } else { + } + else { pos += 1; continue; } @@ -4292,10 +4452,12 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant while (pos < end) { if (text.charCodeAt(pos) === CharacterCodes.space) { // continue on - } else if (text.charCodeAt(pos) === CharacterCodes.colon) { + } + else if (text.charCodeAt(pos) === CharacterCodes.colon) { isTsComment = true; break; - } else { + } + else { break; } pos++; @@ -4318,7 +4480,8 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant pos += colonCount; // console.log(' ts opening comment skip'); // BUILDLESS: DEBUG: comment-scan continue; - } else { // `/*::: ... */` + } + else { // `/*::: ... */` // A triple-colon isn't allowed. // We'll just say that this isn't a recognized TypeScript comment. pos += 3; @@ -4352,9 +4515,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant } } } - -export type TsCommentScanner = ReturnType; -export type TsSyntaxTracker = ReturnType; // BUILDLESS: /** @internal */ From c8f2bc7af5710474e7e9b0f1532579991c0e0018 Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Sat, 11 May 2024 10:08:51 -0600 Subject: [PATCH 04/12] Remove limitation in README --- README.md | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3314c58f49221..16463122863e4 100644 --- a/README.md +++ b/README.md @@ -47,4 +47,142 @@ with any additional questions or comments. ## Roadmap -For details on our planned features and future direction, please refer to our [roadmap](https://github.com/microsoft/TypeScript/wiki/Roadmap). +Lets test it out. Create a new file called `main.js` and add the following to it: + +```javascript +/*:: +interface Coordinate { + readonly x: number + readonly y: number +} +*/ + +function createCoordinate( + x /*: number */, + y /*: number */, +) /*: Coordinate */ { + return { x, y }; +} + +createCoordinate(3, 'not a number'); +``` + +For now, ignore any errors that your editor may be showing - those may be incorrect, and will be addressed in the [editor support](#editor-support) section. + +Run `npx tsc`. It should report an error that looks like this: + +``` +main.js:15:21 - error TS2345: Argument of type 'string' is not assignable to parameter of type 'number'. + +15 createCoordinate(3, 'not a number'); + ~~~~~~~~~~~~~~ +``` + +If you configured your project to emit declaration files, you should see `.d.ts` files appear next to your `.js` files as well. + +Now fix the error by changing `'not a number'` into a real number then re-run `npx tsc`. + +## Editor Support + +The Buildless TypeScript library installed inside of your project provides a language server that your editor can talk to. This language server provides your editor with syntax highlighting support, type error information, refactoring helpers, and more - you just have to get your editor to use the custom language server. + +In the case of VS code, the VS Code editor comes with its own internal TypeScript installation that it uses by default, but [you can ask it to switch to the workspace's version instead](https://code.visualstudio.com/docs/typescript/typescript-compiling#_using-the-workspace-version-of-typescript). Add a `.vscode` folder to the root of your project with a `settings.json` file containing the following: + +``` +{ + "typescript.tsdk": "node_modules/buildlesstypescript/lib" +} +``` + +Next, open your command palette (typically `ctrl+shift+P`) and run the command "TypeScript: Select TypeScript Version". An option should appear to let you use the workspace version of TypeScript. + +Most editors should have similar features available. + +## Converting an Existing Project + +Buildless TypeScript comes with a CLI tool that can be used to convert your existing TypeScript project into Buildless TypeScript. The only thing the tool does is rename files from `.ts` to `.js` and add TS comment delimiters to it (the `/*::`, `/*:`, and `*/` stuff) - this means it has some important limitations to be aware of: +* Most TypeScript syntax has no effect on the runtime behavior of your codebase, but there are a few pieces of older syntax that do, including enums, namespaces, and more. The auto-conversion tool is not smart enough to handle syntax like this - it'll treat it the same as all other TypeScript syntax - sticking it inside of TS comments. +- TypeScript can be used to transpile a project into an older version of JavaScript. Check your project's `package.json`'s `"target"` field to see what version of JavaScript it is currently configured to transpile to. Once you've switched to Buildless TypeScript, there won't be a transpilation step anymore, and that `"target"` field will be ignored. +* TypeScript can automatically convert ES module syntax (`import ... from ...`) into CommonJS (`require(...)`). Check your `package.json`'s `"module"` field - if it is set to `"commonjs"` that means it is automatically performing this conversion for you. Buildless TypeScript works best with ES modules - you can support CommonJS, but [it is a pain](https://github.com/theScottyJam/buildlesstypescript/blob/main/docs/using-commonjs.md), and this converter tool will not automatically convert your ES import/export syntax into CommonJS. +* No adjustments will be made to your build setup. For example, if your `package.json` expects files to be in a `build/` folder, and you've switched to Buildless TypeScript, then you won't have a `build/` folder anymore and you'll need to make the appropriate adjustments. Depending on how your unit tests are set up, their configuration may need tweaking as well. +* While it does a fairly good job at formatting the comment delimiters, it is not perfect and may make some odd choices here and there. + +To convert a project, simply open a shell at the root of your project and run the command `npx --package=buildlesstypescript tsc --buildlessConvert`. This will automatically convert all files in your project that would typically get type-checked. If you want to control which files are being converted, the easiest option is to temporarily [add/edit the include/exclude fields](https://www.typescriptlang.org/tsconfig/#include) in `tsconfig.json` before running the conversion command. + +## Ejecting + +An ejection tool is provided to automatically convert your Buildless TypeScript files into normal TypeScript files. The tool does nothing more than remove TypeScript comment delimiters (the `/*::`, `/*:`, and `*/` stuff) and rename your `.js` files to `.ts`. As such, there are a couple of things to be aware of: +* It will not configure your build step for you, you will have to configure that after you eject. +* For the most part it should do fine at formatting the final TypeScript file, especially if you loosely follow [the Buildless TypeScript style guide](https://github.com/theScottyJam/buildlesstypescript/blob/main/docs/style-guide.md) - but don't feel pressured into following that guide, its completely optional. + +To eject a project, simply open a shell at the root of your project and run the command `npx tsc --buildlessEject`. This will automatically eject all files in your project that would typically get type-checked. If you want to control which files are being converted, the easiest option is to temporarily [add/edit the include/exclude fields](https://www.typescriptlang.org/tsconfig/#include) in `tsconfig.json` before running the conversion command. + +If you want to eject to JavaScript instead of TypeScript, that can be done through a two-step process - first convert to TypeScript, then build the TypeScript project. You can then replace your source code with the build artifacts. + +## Limitations + +Known low-priority issues. There are plans to fix these issues in the future. +* You must use the `as` keyword to do type assertions. Old-style angle-bracket assertions (e.g. `myNumber`) do not work. +* If you put TypeScript syntax in a JavaScript file, it will tell you to move it to a TypeScript file. What it should say instead is to move it into a TS comment. + +Known limitations. There are currently no plans to change these - mostly in an effort to keep this fork from getting too complicated. +* The single-colon and double-colon comments only work with block comments. You can not use line comments. e.g. `//: ...` and `//:: ...` do not work. +* There are no plans to support JSX - JSX requires a build-step anyways, so might as well use tsx instead if you want TypeScript support. + +## Q&A + +### Why? + +A tool like this isn't for everyone, but there is a growing appetite for using TypeScript without a build step during development - there's been a number of projects who have moved to putting all types inside of JSDocs instead of ts files for precisely this reason. And yes, its true that many projects will still need some sort of build step for the production build (e.g. to minify the code) - this project focuses solely on removing the build step during development. + +The reasons why its nice to not have a build step: +* You don't have to deal with mapping files, which don't always work great. +* Faster execution time - if you want to run your code, you can just run it. ts-node can be fairly slow. +* Configuration can sometimes be quite difficult. + +JSDocs have their own issues: +* They're very verbose to use +* They use syntax that's different from TypeScript, which increases their learning curve. +* They don't support all of TypeScript syntax. + +Microsoft recognizes that the build step isn't ideal - they made [a whole JavaScript proposal](https://github.com/tc39/proposal-type-annotations) to try and convince the EcmaScript committee to allow us to write TypeScript without a build step - JavaScript engines would just ignore the TypeScript syntax. We'll see if that proposal makes any progress or not. + +This is actually prior art for this sort of thing as well - [Flow will do something very similar](https://flow.org/en/docs/types/comments/) - if you don't want to have a compile step, you can put your Flow syntax in `/*:: ... */` and `/*: ... */` comments in the same. + +### How do you write JSDoc comments inside of a TS comment? + +For example, say you are defining an interface and you'd like to document some of its properties. In TypeScript you would be able to write the following: + +```typescript +interface User { + readonly username: string + readonly birthday: Date + /** @deprecated */ + readonly age: number +} +``` + +With Buildless TypeScript, you can add JSDoc in the middle of your interface definition in the exact same way, you just have to be wary of the fact that the closing block comment delimiter (`*/`) is going to close both the JSDoc comment and your TS comment. To fix that, just re-open your TS comment right afterwards, like this: + +```javascript +/*:: +interface User { + readonly username: string + readonly birthday: Date + /** @deprecated *//*:: + readonly age: number +} +*/ +``` + +### What's up with the huge version numbers? + +This project's version numbers can be parsed as follows - If you omit the last two digits of each segment, you will get a TypeScript version number, so a Buildless TypeScript version number of `500.301.304` really means "TypeScript version 5.3.3". The remaining digits are to allow Buildless TypeScript to release semver compatible releases between TypeScript versions. With the ``500.301.304`` example again, that version number says that `4` patch releases and `1` minor release have come out for this tool since it provided the TypeScript `5.3.3` release. Every time this fork incorporates a new TypeScript version, it will reset the last two digits back to `00`. + +### How should I format the TS comments? + +For those who like being told how to format their code, [a style guide is available](https://github.com/theScottyJam/buildlesstypescript/blob/main/docs/style-guide.md). If you don't like being told what to do, don't click on the link. + +### This project is awesome, how can I contribute? + +The best way to contribute is by going to [this TypeScript feature request](https://github.com/microsoft/TypeScript/issues/48650) and adding a thumbs up to the proposal, asking for TypeScript-in-comments to become a native feature. From b51a8f4af5006de7ad5bac13dda345612bf80cae Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Sat, 11 May 2024 10:46:11 -0600 Subject: [PATCH 05/12] Fix broken test related to underlined region of import type --- src/compiler/program.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 12de2decc4959..89b9a044c0125 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -3129,6 +3129,37 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg // Otherwise break to visit each child switch (parent.kind) { + // BUILDLESS: + case SyntaxKind.AsExpression: + if ((parent as AsExpression).type === node) { + // Move scanner just before "as" + tsCommentScanner.scanTo((parent as AsExpression).expression.end); + // Scan "as" and the type node that follows + if (scanLeavesCheckingIfInTsComment(node, IsTSSyntax.yesAndSoIsEarlierUnscannedContent)) return "skip"; + diagnostics.push(createDiagnosticForNode((parent as AsExpression).type, Diagnostics.Type_assertion_expressions_can_only_be_used_in_TypeScript_files)); + return "skip"; + } + break; + case SyntaxKind.SatisfiesExpression: + if ((parent as SatisfiesExpression).type === node) { + // Move scanner just before "satisfies" + tsCommentScanner.scanTo((parent as SatisfiesExpression).expression.end); + // Scan "satisfies" and the type node that follows + if (scanLeavesCheckingIfInTsComment(node, IsTSSyntax.yesAndSoIsEarlierUnscannedContent)) return "skip"; + diagnostics.push(createDiagnosticForNode((parent as SatisfiesExpression).type, Diagnostics.Type_satisfaction_expressions_can_only_be_used_in_TypeScript_files)); + return "skip"; + } + break; + case SyntaxKind.ImportDeclaration: + if ((parent as ImportDeclaration).importClause === node) { + if ((node as ImportClause).isTypeOnly) { + if (scanLeavesCheckingIfInTsComment(parent)) return "skip"; // BUILDLESS: added + diagnostics.push(createDiagnosticForNode(parent, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, "import type")); + return "skip"; + } + } + break; + // BUILDLESS: case SyntaxKind.Parameter: case SyntaxKind.PropertyDeclaration: case SyntaxKind.MethodDeclaration: From 29b2ddad8f986bc74f8dfdcb0282d342981de98c Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Sat, 11 May 2024 11:40:58 -0600 Subject: [PATCH 06/12] Fix test relates to incorrect comment-finding behavior --- src/compiler/scanner.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index ca5ea537a522a..231397bd5640d 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -959,16 +959,6 @@ function iterateCommentRanges(reduce: boolean, text: string, pos: number, case CharacterCodes.space: pos++; continue; - // BUILDLESS: - case CharacterCodes.asterisk: - if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { - pos += 2; - continue; - } - else { - break scan; - } - // BUILDLESS: case CharacterCodes.slash: const nextChar = text.charCodeAt(pos + 1); let hasTrailingNewLine = false; From 78c5c2c375492c280f4cbd4a897c81356f218905 Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Sat, 11 May 2024 11:54:03 -0600 Subject: [PATCH 07/12] Move some types to types.ts to fix some tests --- src/compiler/scanner.ts | 22 +++------------------- src/compiler/types.ts | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 231397bd5640d..6a3149a39abaf 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -2,6 +2,8 @@ import { append, arraysEqual, binarySearch, + BlockCommentDelimiterPositions, // BUILDLESS: added + BlockCommentDelimiterType, // BUILDLESS: added CharacterCodes, CommentDirective, CommentDirectiveType, @@ -33,6 +35,7 @@ import { SyntaxKind, TextRange, TokenFlags, + TsCommentScannerState, // BUILDLESS: added } from "./_namespaces/ts.js"; export type ErrorCallback = (message: DiagnosticMessage, length: number, arg0?: any) => void; @@ -129,14 +132,6 @@ export interface Scanner { } // BUILDLESS: -export const enum BlockCommentDelimiterType { - openNonTsComment, // A `/*` not followed by colons - openTsCommentWithoutColons, // Just the `/*` part of `/*:` or `/*::`. - singleColonForOpenTsComment, // Just the `:` part of `/*:` - doubleColonForOpenTsComment, // Just the `::` part of `/*::` - close, // Any `*/`, even if it pertains to a non-TS comment. -} - export interface TsCommentPosition { open: number; colonPos: number; @@ -159,11 +154,6 @@ export const enum SyntaxRangeType { whitespace, } -// Records the position of interesting comment-related information. -// When an openTSCommentWithoutColons is recorded, it will be followed by -// either a singleColonForOpenTSComment or doubleColonForOpenTSComment. -type BlockCommentDelimiterPositions = Map - export interface TsCommentScanner { getCurrentPos(): number; clone(): TsCommentScanner; @@ -4096,12 +4086,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean } // BUILDLESS: -const enum TsCommentScannerState { - notInComment, - inBlockComment, - inTSBlockComment, -} - export function createTSCommentScanner( { text }: SourceFileLike, _pos?: number, diff --git a/src/compiler/types.ts b/src/compiler/types.ts index aae7c5494a17d..d376a8008eacb 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -9933,6 +9933,28 @@ export const enum ListFormat { JSDocComment = MultiLine | AsteriskDelimited, } +// BUILDLESS: + +export const enum BlockCommentDelimiterType { + openNonTsComment, // A `/*` not followed by colons + openTsCommentWithoutColons, // Just the `/*` part of `/*:` or `/*::`. + singleColonForOpenTsComment, // Just the `:` part of `/*:` + doubleColonForOpenTsComment, // Just the `::` part of `/*::` + close, // Any `*/`, even if it pertains to a non-TS comment. +} + +// Used to record the position of interesting comment-related information. +// When an openTSCommentWithoutColons is recorded, it will be followed by +// either a singleColonForOpenTSComment or doubleColonForOpenTSComment. +export type BlockCommentDelimiterPositions = Map + +export const enum TsCommentScannerState { + notInComment, + inBlockComment, + inTSBlockComment, +} +// BUILDLESS: + /** @internal */ export const enum PragmaKindFlags { None = 0, From 5c152b3d3a5f29758d5e8c6aa5a817110e85d7bc Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Mon, 20 May 2024 21:56:27 -0600 Subject: [PATCH 08/12] Prepare changes for MR * Revert changes to files such as README, CONTRIBUTING, etc * Remove `// BUILDLESS: ...` comments * etc --- CONTRIBUTING.md | 10 - README.md | 140 +------- package.json | 2 +- src/compiler/diagnosticMessages.json | 4 + src/compiler/parser.ts | 17 +- src/compiler/program.ts | 497 +++++++-------------------- src/compiler/scanner.ts | 49 +-- src/compiler/types.ts | 4 +- src/compiler/utilities.ts | 10 +- 9 files changed, 172 insertions(+), 561 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 900a3d553b456..56912a250f84a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,13 +1,3 @@ -# Notes on this Buildless TypeScript fork - -I hope that at some point this buildless feature can become a native part of TypeScript, but I also understand that this may not happen for a while, if ever. As such, this fork is designed with the understanding that I may be maintaining this thing for a while, and constantly merging in changes from upstream. To help out with this: -* I've been trying to clearly mark areas of code that I've added or modified with `// BUILDLESS: added` or `// BUILDLESS: modified` comments. You'll also see XML-looking variants of those comments if I want to note that a section of code was added or modified, e.g. `// BUILDLESS: ...some code... /// BUILDLESS: `. -* I don't want to modify too many places in this project so as to avoid merge conflicts. There's already a lot more changes in this project than what I was hoping to ever have. There's some feature requests that I may turn down, not because they wouldn't be nice to have, but because I'm worried about creating and maintaining too many modifications to this codebase. - -Anyways, that's the gist of it. Everything below this point came from the original TypeScript repo, and has some useful information to get up-and-running with this codebase. - ---- - # Instructions for Logging Issues ## 1. Read the FAQ diff --git a/README.md b/README.md index 16463122863e4..6197eb4408588 100644 --- a/README.md +++ b/README.md @@ -47,142 +47,4 @@ with any additional questions or comments. ## Roadmap -Lets test it out. Create a new file called `main.js` and add the following to it: - -```javascript -/*:: -interface Coordinate { - readonly x: number - readonly y: number -} -*/ - -function createCoordinate( - x /*: number */, - y /*: number */, -) /*: Coordinate */ { - return { x, y }; -} - -createCoordinate(3, 'not a number'); -``` - -For now, ignore any errors that your editor may be showing - those may be incorrect, and will be addressed in the [editor support](#editor-support) section. - -Run `npx tsc`. It should report an error that looks like this: - -``` -main.js:15:21 - error TS2345: Argument of type 'string' is not assignable to parameter of type 'number'. - -15 createCoordinate(3, 'not a number'); - ~~~~~~~~~~~~~~ -``` - -If you configured your project to emit declaration files, you should see `.d.ts` files appear next to your `.js` files as well. - -Now fix the error by changing `'not a number'` into a real number then re-run `npx tsc`. - -## Editor Support - -The Buildless TypeScript library installed inside of your project provides a language server that your editor can talk to. This language server provides your editor with syntax highlighting support, type error information, refactoring helpers, and more - you just have to get your editor to use the custom language server. - -In the case of VS code, the VS Code editor comes with its own internal TypeScript installation that it uses by default, but [you can ask it to switch to the workspace's version instead](https://code.visualstudio.com/docs/typescript/typescript-compiling#_using-the-workspace-version-of-typescript). Add a `.vscode` folder to the root of your project with a `settings.json` file containing the following: - -``` -{ - "typescript.tsdk": "node_modules/buildlesstypescript/lib" -} -``` - -Next, open your command palette (typically `ctrl+shift+P`) and run the command "TypeScript: Select TypeScript Version". An option should appear to let you use the workspace version of TypeScript. - -Most editors should have similar features available. - -## Converting an Existing Project - -Buildless TypeScript comes with a CLI tool that can be used to convert your existing TypeScript project into Buildless TypeScript. The only thing the tool does is rename files from `.ts` to `.js` and add TS comment delimiters to it (the `/*::`, `/*:`, and `*/` stuff) - this means it has some important limitations to be aware of: -* Most TypeScript syntax has no effect on the runtime behavior of your codebase, but there are a few pieces of older syntax that do, including enums, namespaces, and more. The auto-conversion tool is not smart enough to handle syntax like this - it'll treat it the same as all other TypeScript syntax - sticking it inside of TS comments. -- TypeScript can be used to transpile a project into an older version of JavaScript. Check your project's `package.json`'s `"target"` field to see what version of JavaScript it is currently configured to transpile to. Once you've switched to Buildless TypeScript, there won't be a transpilation step anymore, and that `"target"` field will be ignored. -* TypeScript can automatically convert ES module syntax (`import ... from ...`) into CommonJS (`require(...)`). Check your `package.json`'s `"module"` field - if it is set to `"commonjs"` that means it is automatically performing this conversion for you. Buildless TypeScript works best with ES modules - you can support CommonJS, but [it is a pain](https://github.com/theScottyJam/buildlesstypescript/blob/main/docs/using-commonjs.md), and this converter tool will not automatically convert your ES import/export syntax into CommonJS. -* No adjustments will be made to your build setup. For example, if your `package.json` expects files to be in a `build/` folder, and you've switched to Buildless TypeScript, then you won't have a `build/` folder anymore and you'll need to make the appropriate adjustments. Depending on how your unit tests are set up, their configuration may need tweaking as well. -* While it does a fairly good job at formatting the comment delimiters, it is not perfect and may make some odd choices here and there. - -To convert a project, simply open a shell at the root of your project and run the command `npx --package=buildlesstypescript tsc --buildlessConvert`. This will automatically convert all files in your project that would typically get type-checked. If you want to control which files are being converted, the easiest option is to temporarily [add/edit the include/exclude fields](https://www.typescriptlang.org/tsconfig/#include) in `tsconfig.json` before running the conversion command. - -## Ejecting - -An ejection tool is provided to automatically convert your Buildless TypeScript files into normal TypeScript files. The tool does nothing more than remove TypeScript comment delimiters (the `/*::`, `/*:`, and `*/` stuff) and rename your `.js` files to `.ts`. As such, there are a couple of things to be aware of: -* It will not configure your build step for you, you will have to configure that after you eject. -* For the most part it should do fine at formatting the final TypeScript file, especially if you loosely follow [the Buildless TypeScript style guide](https://github.com/theScottyJam/buildlesstypescript/blob/main/docs/style-guide.md) - but don't feel pressured into following that guide, its completely optional. - -To eject a project, simply open a shell at the root of your project and run the command `npx tsc --buildlessEject`. This will automatically eject all files in your project that would typically get type-checked. If you want to control which files are being converted, the easiest option is to temporarily [add/edit the include/exclude fields](https://www.typescriptlang.org/tsconfig/#include) in `tsconfig.json` before running the conversion command. - -If you want to eject to JavaScript instead of TypeScript, that can be done through a two-step process - first convert to TypeScript, then build the TypeScript project. You can then replace your source code with the build artifacts. - -## Limitations - -Known low-priority issues. There are plans to fix these issues in the future. -* You must use the `as` keyword to do type assertions. Old-style angle-bracket assertions (e.g. `myNumber`) do not work. -* If you put TypeScript syntax in a JavaScript file, it will tell you to move it to a TypeScript file. What it should say instead is to move it into a TS comment. - -Known limitations. There are currently no plans to change these - mostly in an effort to keep this fork from getting too complicated. -* The single-colon and double-colon comments only work with block comments. You can not use line comments. e.g. `//: ...` and `//:: ...` do not work. -* There are no plans to support JSX - JSX requires a build-step anyways, so might as well use tsx instead if you want TypeScript support. - -## Q&A - -### Why? - -A tool like this isn't for everyone, but there is a growing appetite for using TypeScript without a build step during development - there's been a number of projects who have moved to putting all types inside of JSDocs instead of ts files for precisely this reason. And yes, its true that many projects will still need some sort of build step for the production build (e.g. to minify the code) - this project focuses solely on removing the build step during development. - -The reasons why its nice to not have a build step: -* You don't have to deal with mapping files, which don't always work great. -* Faster execution time - if you want to run your code, you can just run it. ts-node can be fairly slow. -* Configuration can sometimes be quite difficult. - -JSDocs have their own issues: -* They're very verbose to use -* They use syntax that's different from TypeScript, which increases their learning curve. -* They don't support all of TypeScript syntax. - -Microsoft recognizes that the build step isn't ideal - they made [a whole JavaScript proposal](https://github.com/tc39/proposal-type-annotations) to try and convince the EcmaScript committee to allow us to write TypeScript without a build step - JavaScript engines would just ignore the TypeScript syntax. We'll see if that proposal makes any progress or not. - -This is actually prior art for this sort of thing as well - [Flow will do something very similar](https://flow.org/en/docs/types/comments/) - if you don't want to have a compile step, you can put your Flow syntax in `/*:: ... */` and `/*: ... */` comments in the same. - -### How do you write JSDoc comments inside of a TS comment? - -For example, say you are defining an interface and you'd like to document some of its properties. In TypeScript you would be able to write the following: - -```typescript -interface User { - readonly username: string - readonly birthday: Date - /** @deprecated */ - readonly age: number -} -``` - -With Buildless TypeScript, you can add JSDoc in the middle of your interface definition in the exact same way, you just have to be wary of the fact that the closing block comment delimiter (`*/`) is going to close both the JSDoc comment and your TS comment. To fix that, just re-open your TS comment right afterwards, like this: - -```javascript -/*:: -interface User { - readonly username: string - readonly birthday: Date - /** @deprecated *//*:: - readonly age: number -} -*/ -``` - -### What's up with the huge version numbers? - -This project's version numbers can be parsed as follows - If you omit the last two digits of each segment, you will get a TypeScript version number, so a Buildless TypeScript version number of `500.301.304` really means "TypeScript version 5.3.3". The remaining digits are to allow Buildless TypeScript to release semver compatible releases between TypeScript versions. With the ``500.301.304`` example again, that version number says that `4` patch releases and `1` minor release have come out for this tool since it provided the TypeScript `5.3.3` release. Every time this fork incorporates a new TypeScript version, it will reset the last two digits back to `00`. - -### How should I format the TS comments? - -For those who like being told how to format their code, [a style guide is available](https://github.com/theScottyJam/buildlesstypescript/blob/main/docs/style-guide.md). If you don't like being told what to do, don't click on the link. - -### This project is awesome, how can I contribute? - -The best way to contribute is by going to [this TypeScript feature request](https://github.com/microsoft/TypeScript/issues/48650) and adding a thumbs up to the proposal, asking for TypeScript-in-comments to become a native feature. +For details on our planned features and future direction, please refer to our [roadmap](https://github.com/microsoft/TypeScript/wiki/Roadmap). \ No newline at end of file diff --git a/package.json b/package.json index e479356ebb9a5..35d8d9abe52a6 100644 --- a/package.json +++ b/package.json @@ -110,4 +110,4 @@ "node": "20.1.0", "npm": "8.19.4" } -} +} \ No newline at end of file diff --git a/src/compiler/diagnosticMessages.json b/src/compiler/diagnosticMessages.json index 725a6b73aa450..86a2d91c2f48b 100644 --- a/src/compiler/diagnosticMessages.json +++ b/src/compiler/diagnosticMessages.json @@ -6885,6 +6885,10 @@ "category": "Error", "code": 8039 }, + "JavaScript syntax can not be used inside of a TS comment": { + "category": "Error", + "code": 8040 + }, "Declaration emit for this file requires using private name '{0}'. An explicit type annotation may unblock declaration emit.": { "category": "Error", diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index a63735ad6bfa2..d8982c611b27c 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -1251,6 +1251,18 @@ export function forEachChild(node: Node, cbNode: (node: Node) => T | undefine return fn === undefined ? undefined : fn(node, cbNode, cbNodes); } +/** + * The definition of a leaf node isn't super precise here. + * For example, an empty object could be thought of as a leaf node since it doesn't have any children, + * but this function would claim that it isn't a leaf node. + * For the purposes of how this function gets used, + * what's important is that it won't ever claim that a node that has children is a leaf node. + * Other types of minor errors are fine. + */ +export function isLeafNode(node: Node) { + return (forEachChildTable as Record>)[node.kind] === undefined; +} + /** * Invokes a callback for each child of the given node. The 'cbNode' callback is invoked for all child nodes * stored in properties. If a 'cbNodes' callback is specified, it is invoked for embedded arrays; additionally, @@ -5241,6 +5253,7 @@ namespace Parser { } } + const firstTokenPrecededByOpeningTSComment = (scanner.getTokenFlags() & TokenFlags.enteringTSComment) !== 0; const first = token(); const second = nextToken(); @@ -5327,7 +5340,7 @@ namespace Parser { } // JSX overrides - if (languageVariant === LanguageVariant.JSX) { + if (languageVariant === LanguageVariant.JSX && !firstTokenPrecededByOpeningTSComment) { const isArrowFunctionInJsx = lookAhead(() => { parseOptional(SyntaxKind.ConstKeyword); const third = nextToken(); @@ -6531,7 +6544,7 @@ namespace Parser { } function parseTypeArgumentsInExpression() { - if ((contextFlags & NodeFlags.JavaScriptFile) !== 0) { + if ((contextFlags & NodeFlags.JavaScriptFile) !== 0 && (scanner.getTokenFlags() & TokenFlags.enteringTSComment) === 0) { // TypeArguments must not be parsed in JavaScript files to avoid ambiguity with binary operators. return undefined; } diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 89b9a044c0125..b9ae4290c2e92 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -48,6 +48,8 @@ import { createSourceFile, CreateSourceFileOptions, createSymlinkCache, + createTSCommentScanner, + createTSSyntaxTracker, createTypeChecker, createTypeReferenceDirectiveResolutionCache, CustomTransformers, @@ -197,6 +199,7 @@ import { isIncrementalCompilation, isInJSFile, isJSDocImportTag, + isLeafNode, isLiteralImportTypeNode, isModifier, isModuleDeclaration, @@ -239,6 +242,7 @@ import { NodeFlags, nodeModulesPathPart, NodeWithTypeArguments, + NonNullExpression, noop, normalizePath, notImplementedResolver, @@ -308,7 +312,7 @@ import { supportedJSExtensionsFlat, SymlinkCache, SyntaxKind, - SyntaxRangeType, // BUILDLESS: added + SyntaxRangeType, sys, System, toFileNameLowerCase, @@ -316,11 +320,10 @@ import { toPath as ts_toPath, trace, tracing, - tryCast, - TsCommentPosition, // BUILDLESS: added - TsCommentScanner, // BUILDLESS: added + TsCommentPosition, + TsCommentScanner, TsConfigSourceFile, - TsSyntaxTracker, // BUILDLESS: added + TsSyntaxTracker, TypeChecker, typeDirectiveIsEqualTo, TypeReferenceDirectiveResolutionCache, @@ -3093,28 +3096,37 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg return -1; } + /* + The general algorithm works like this. + + We create an instance of a ts-comment-scanner. This scanner is in charge of scanning all of the spaces + between nodes, looking for opening TS comments (/*:: or /*:) and their matching, closing pair. + It'll keep some internal state, remembering if its current position is inside or outside of a TS comment. + For it to work, it must scan all leaf nodes, once each, in order. + + Inside of getJSSyntacticDiagnosticsForFile() we'll recurse into the AST tree. + If we reach a leaf node, we make sure to tell the ts-comment-scanner to traverse it (and to check if it was in a TS comment). + If we come across a piece of syntax that's TypeScript only, we'll call a scanLeavesCheckingIfInTsComment() + which goes and traverses the subtree of our current node, looking for all leaves, then it'll run the scanner + across those leaves, asking the scanner at each step if the leaf was inside or outside of a TS comment. + If all leaves were inside a TS comment, we'll skip reporting a "this can only be used in JS files" error. + + There's also a fair amount of work that goes into tracking the exact boundaries of TypeScript syntax, + so if we wish to convert a TypeScript file to JavaScript+comment, we'd be able to do so. + */ + function getJSSyntacticDiagnosticsForFile(sourceFile: SourceFile): DiagnosticWithLocation[] { return runWithCancellationToken(() => { const diagnostics: DiagnosticWithLocation[] = []; - const tsCommentScanner = createTSCommentScanner(sourceFile); // BUILDLESS: added - const tsSyntaxTracker = createTSSyntaxTracker(sourceFile); // BUILDLESS: added + const tsCommentScanner = createTSCommentScanner(sourceFile); + const tsSyntaxTracker = createTSSyntaxTracker(sourceFile); walk(sourceFile, sourceFile); forEachChildRecursively(sourceFile, walk, walkArray); - lookForJavaScriptInsideOfTSComments(tsSyntaxTracker, tsCommentScanner.getTSCommentPositions()); // BUILDLESS: added - - // BUILDLESS: - if (options.buildlessEject) { - ejectBuildlessFile(sourceFile, tsCommentScanner.getTSCommentPositions()); - } - else if (options.buildlessConvert) { - convertToBuildlessFile(sourceFile, tsSyntaxTracker, tsCommentScanner.getBlockCommentDelimiters()); - } - // BUILDLESS: + lookForJavaScriptInsideOfTSComments(tsSyntaxTracker, tsCommentScanner.getTSCommentPositions()); return diagnostics; function walk(node: Node, parent: Node) { - // BUILDLESS: const maybeSkip: 'skip' | undefined = walkHelper(node, parent); if (isLeafNode(node)) { tsCommentScanner.scanThroughLeafNode(node); @@ -3123,13 +3135,10 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg } function walkHelper(node: Node, parent: Node) { - // BUILDLESS: - - // Return directly from the case if the given node doesnt want to visit each child + // Return directly from the case if the given node doesn't want to visit each child // Otherwise break to visit each child switch (parent.kind) { - // BUILDLESS: case SyntaxKind.AsExpression: if ((parent as AsExpression).type === node) { // Move scanner just before "as" @@ -3153,17 +3162,15 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.ImportDeclaration: if ((parent as ImportDeclaration).importClause === node) { if ((node as ImportClause).isTypeOnly) { - if (scanLeavesCheckingIfInTsComment(parent)) return "skip"; // BUILDLESS: added + if (scanLeavesCheckingIfInTsComment(parent)) return "skip"; diagnostics.push(createDiagnosticForNode(parent, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, "import type")); return "skip"; } } break; - // BUILDLESS: case SyntaxKind.Parameter: case SyntaxKind.PropertyDeclaration: case SyntaxKind.MethodDeclaration: - // BUILDLESS: if ( (parent.kind === SyntaxKind.Parameter) && (parent as ParameterDeclaration).name === node && @@ -3181,8 +3188,8 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(scannerIndex, scannerIndex + 1); } } - // BUILDLESS: if ((parent as ParameterDeclaration | PropertyDeclaration | MethodDeclaration).questionToken === node) { + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.The_0_modifier_can_only_be_used_in_TypeScript_files, "?")); return "skip"; } @@ -3195,22 +3202,54 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.FunctionDeclaration: case SyntaxKind.ArrowFunction: case SyntaxKind.VariableDeclaration: + if ( + (parent.kind === SyntaxKind.PropertyDeclaration || parent.kind === SyntaxKind.VariableDeclaration) && + (parent as PropertyDeclaration).exclamationToken === node + ) { + // Make sure we mark the "!" as being TypeScript syntax. + // If you use a "!", you'll get an error indirectly, because using "!" means you're also using ": ..." type syntax. + // So they don't have to directly report an error for the use of this character. + scanLeavesCheckingIfInTsComment(node); + } // type annotation if ((parent as FunctionLikeDeclaration | VariableDeclaration | ParameterDeclaration | PropertyDeclaration).type === node) { + const isFunctionLike = (node: Node): node is FunctionLikeDeclaration => ( + node.kind === SyntaxKind.MethodDeclaration || + node.kind === SyntaxKind.MethodSignature || + node.kind === SyntaxKind.Constructor || + node.kind === SyntaxKind.GetAccessor || + node.kind === SyntaxKind.SetAccessor || + node.kind === SyntaxKind.FunctionExpression || + node.kind === SyntaxKind.FunctionDeclaration || + node.kind === SyntaxKind.ArrowFunction + ); + // Move scanner just before the ":" + if (parent.kind === SyntaxKind.Parameter) { + tsCommentScanner.scanTo((parent as ParameterDeclaration).name.end); + } + else if (parent.kind === SyntaxKind.PropertyDeclaration) { + tsCommentScanner.scanTo((parent as PropertyDeclaration).name.end); + } + else if (isFunctionLike(parent)) { + // Moves to the closing parenthesis + tsCommentScanner.scanTo(parent.parameters.end); + // Move past the closing parenthesis so we're right before the colon for the return type. + tsCommentScanner.scanUntilPastChar(')', parent.end); + } + else if (parent.kind === SyntaxKind.VariableDeclaration) { + tsCommentScanner.scanTo((parent as VariableDeclaration).name.end); + } + // Scan the ":" and the type node that follows + if (scanLeavesCheckingIfInTsComment(node, IsTSSyntax.yesAndSoIsEarlierUnscannedContent)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.Type_annotations_can_only_be_used_in_TypeScript_files)); return "skip"; } } switch (node.kind) { - case SyntaxKind.ImportClause: - if ((node as ImportClause).isTypeOnly) { - diagnostics.push(createDiagnosticForNode(parent, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, "import type")); - return "skip"; - } - break; case SyntaxKind.ExportDeclaration: if ((node as ExportDeclaration).isTypeOnly) { + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, "export type")); return "skip"; } @@ -3218,7 +3257,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.ImportSpecifier: case SyntaxKind.ExportSpecifier: if ((node as ImportOrExportSpecifier).isTypeOnly) { - // BUILDLESS: if (scanLeavesCheckingIfInTsComment(node)) { return "skip"; } @@ -3228,12 +3266,10 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg if (sourceFile.text[scannerIndex] === ',') { tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(scannerIndex, scannerIndex + 1); } - // BUILDLESS: diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, isImportSpecifier(node) ? "import...type" : "export...type")); return "skip"; } break; - // BUILDLESS: case SyntaxKind.ImportDeclaration: { const importDeclaration = node as ImportDeclaration; const importClause = importDeclaration.importClause; @@ -3263,12 +3299,13 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg } break; } - // BUILDLESS: case SyntaxKind.ImportEqualsDeclaration: + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.import_can_only_be_used_in_TypeScript_files)); return "skip"; case SyntaxKind.ExportAssignment: if ((node as ExportAssignment).isExportEquals) { + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.export_can_only_be_used_in_TypeScript_files)); return "skip"; } @@ -3276,37 +3313,42 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.HeritageClause: const heritageClause = node as HeritageClause; if (heritageClause.token === SyntaxKind.ImplementsKeyword) { + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.implements_clauses_can_only_be_used_in_TypeScript_files)); return "skip"; } break; case SyntaxKind.InterfaceDeclaration: + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; const interfaceKeyword = tokenToString(SyntaxKind.InterfaceKeyword); Debug.assertIsDefined(interfaceKeyword); diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, interfaceKeyword)); return "skip"; case SyntaxKind.ModuleDeclaration: + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; const moduleKeyword = node.flags & NodeFlags.Namespace ? tokenToString(SyntaxKind.NamespaceKeyword) : tokenToString(SyntaxKind.ModuleKeyword); Debug.assertIsDefined(moduleKeyword); diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, moduleKeyword)); return "skip"; case SyntaxKind.TypeAliasDeclaration: + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.Type_aliases_can_only_be_used_in_TypeScript_files)); return "skip"; case SyntaxKind.Constructor: case SyntaxKind.MethodDeclaration: case SyntaxKind.FunctionDeclaration: if (!(node as FunctionLikeDeclaration).body) { + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; diagnostics.push(createDiagnosticForNode(node, Diagnostics.Signature_declarations_can_only_be_used_in_TypeScript_files)); return "skip"; } return; case SyntaxKind.EnumDeclaration: + if (scanLeavesCheckingIfInTsComment(node)) return "skip"; const enumKeyword = Debug.checkDefined(tokenToString(SyntaxKind.EnumKeyword)); diagnostics.push(createDiagnosticForNode(node, Diagnostics._0_declarations_can_only_be_used_in_TypeScript_files, enumKeyword)); return "skip"; case SyntaxKind.NonNullExpression: - // BUILDLESS: // We want to use a cloned scanner, because we don't want our real scanner to actually traverse forwards yet. // We still need to recursively descend into the node's children to see if they're correctly // using TS comments where they should @@ -3320,15 +3362,8 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg // Don't return "skip", we need to descend into the children to check for problems. return; } - // BUILDLESS: diagnostics.push(createDiagnosticForNode(node, Diagnostics.Non_null_assertions_can_only_be_used_in_TypeScript_files)); return "skip"; - case SyntaxKind.AsExpression: - diagnostics.push(createDiagnosticForNode((node as AsExpression).type, Diagnostics.Type_assertion_expressions_can_only_be_used_in_TypeScript_files)); - return "skip"; - case SyntaxKind.SatisfiesExpression: - diagnostics.push(createDiagnosticForNode((node as SatisfiesExpression).type, Diagnostics.Type_satisfaction_expressions_can_only_be_used_in_TypeScript_files)); - return "skip"; case SyntaxKind.TypeAssertionExpression: Debug.fail(); // Won't parse these in a JS file anyway, as they are interpreted as JSX. } @@ -3383,7 +3418,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.ArrowFunction: // Check type parameters if (nodes === (parent as DeclarationWithTypeParameterChildren).typeParameters) { - // BUILDLESS: const startOfList = nodes[0]?.pos ?? parent.end; tsCommentScanner.scanUntilAtChar('<', startOfList) const startOfParameterListPos = tsCommentScanner.getCurrentPos(); @@ -3396,7 +3430,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg const endOfParameterListPos = tsCommentScanner.getCurrentPos() + 1; // + 1 to move past the ">" tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(startOfParameterListPos, endOfParameterListPos); if (lessThanInTSComment && contentInTSComment && greaterThanInTSComment) return "skip"; - // BUILDLESS: diagnostics.push(createDiagnosticForNodeArray(nodes, Diagnostics.Type_parameter_declarations_can_only_be_used_in_TypeScript_files)); return "skip"; } @@ -3405,7 +3438,7 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.VariableStatement: // Check modifiers if (nodes === (parent as VariableStatement).modifiers) { - checkModifiers((parent as VariableStatement).modifiers!, parent.kind === SyntaxKind.VariableStatement); + checkModifiers((parent as VariableStatement).modifiers!, parent.kind === SyntaxKind.VariableStatement, parent); return "skip"; } break; @@ -3413,12 +3446,17 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg // Check modifiers of property declaration if (nodes === (parent as PropertyDeclaration).modifiers) { for (const modifier of nodes as NodeArray) { - if ( - isModifier(modifier) - && modifier.kind !== SyntaxKind.StaticKeyword - && modifier.kind !== SyntaxKind.AccessorKeyword - ) { - diagnostics.push(createDiagnosticForNode(modifier, Diagnostics.The_0_modifier_can_only_be_used_in_TypeScript_files, tokenToString(modifier.kind))); + try { + if ( + isModifier(modifier) + && modifier.kind !== SyntaxKind.StaticKeyword + && modifier.kind !== SyntaxKind.AccessorKeyword + && !scanLeavesCheckingIfInTsComment(modifier) + ) { + diagnostics.push(createDiagnosticForNode(modifier, Diagnostics.The_0_modifier_can_only_be_used_in_TypeScript_files, tokenToString(modifier.kind))); + } + } finally { + scanLeavesCheckingIfInTsComment(modifier, IsTSSyntax.no); } } return "skip"; @@ -3427,6 +3465,7 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.Parameter: // Check modifiers of parameter declaration if (nodes === (parent as ParameterDeclaration).modifiers && some(nodes, isModifier)) { + if (scanLeavesCheckingIfInTsComment(nodes)) return "skip"; diagnostics.push(createDiagnosticForNodeArray(nodes, Diagnostics.Parameter_modifiers_can_only_be_used_in_TypeScript_files)); return "skip"; } @@ -3439,7 +3478,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg case SyntaxKind.TaggedTemplateExpression: // Check type arguments if (nodes === (parent as NodeWithTypeArguments).typeArguments) { - // BUILDLESS: const startOfList = nodes[0]?.pos ?? parent.end; tsCommentScanner.scanUntilAtChar('<', startOfList) const startOfParameterListPos = tsCommentScanner.getCurrentPos(); @@ -3452,7 +3490,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg const endOfParameterListPos = tsCommentScanner.getCurrentPos() + 1; // + 1 to move past the ">" tsSyntaxTracker.skipWhitespaceThenMarkRangeAsTS(startOfParameterListPos, endOfParameterListPos); if (lessThanInTSComment && contentInTSComment && greaterThanInTSComment) return "skip"; - // BUILDLESS: diagnostics.push(createDiagnosticForNodeArray(nodes, Diagnostics.Type_arguments_can_only_be_used_in_TypeScript_files)); return "skip"; } @@ -3460,32 +3497,43 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg } } - function checkModifiers(modifiers: NodeArray, isConstValid: boolean) { + function checkModifiers(modifiers: NodeArray, isConstValid: boolean, parent: Node) { for (const modifier of modifiers) { - switch (modifier.kind) { - case SyntaxKind.ConstKeyword: - if (isConstValid) { - continue; - } - // to report error, - // falls through - case SyntaxKind.PublicKeyword: - case SyntaxKind.PrivateKeyword: - case SyntaxKind.ProtectedKeyword: - case SyntaxKind.ReadonlyKeyword: - case SyntaxKind.DeclareKeyword: - case SyntaxKind.AbstractKeyword: - case SyntaxKind.OverrideKeyword: - case SyntaxKind.InKeyword: - case SyntaxKind.OutKeyword: - diagnostics.push(createDiagnosticForNode(modifier, Diagnostics.The_0_modifier_can_only_be_used_in_TypeScript_files, tokenToString(modifier.kind))); - break; - - // These are all legal modifiers. - case SyntaxKind.StaticKeyword: - case SyntaxKind.ExportKeyword: - case SyntaxKind.DefaultKeyword: - case SyntaxKind.AccessorKeyword: + try { + switch (modifier.kind) { + case SyntaxKind.ConstKeyword: + if (isConstValid) { + continue; + } + // to report error, + // falls through + case SyntaxKind.PublicKeyword: + case SyntaxKind.PrivateKeyword: + case SyntaxKind.ProtectedKeyword: + case SyntaxKind.ReadonlyKeyword: + case SyntaxKind.DeclareKeyword: + case SyntaxKind.AbstractKeyword: + case SyntaxKind.OverrideKeyword: + case SyntaxKind.InKeyword: + case SyntaxKind.OutKeyword: + if (modifier.kind === SyntaxKind.DeclareKeyword) { + // If its a declare keyword, the whole statement should be in a TypeScript comment. + scanLeavesCheckingIfInTsComment(parent, IsTSSyntax.yes, tsCommentScanner.clone()); + } + if (scanLeavesCheckingIfInTsComment(modifier)) { + break; + } + diagnostics.push(createDiagnosticForNode(modifier, Diagnostics.The_0_modifier_can_only_be_used_in_TypeScript_files, tokenToString(modifier.kind))); + break; + + // These are all legal modifiers. + case SyntaxKind.StaticKeyword: + case SyntaxKind.ExportKeyword: + case SyntaxKind.DefaultKeyword: + case SyntaxKind.AccessorKeyword: + } + } finally { + scanLeavesCheckingIfInTsComment(modifier, IsTSSyntax.no) } } } @@ -3501,7 +3549,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg return createDiagnosticForNodeInSourceFile(sourceFile, node, message, ...args); } - // BUILDLESS: const enum IsTSSyntax { /** The node being scanned is not TS syntax */ no, @@ -3604,294 +3651,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg } } } - - function ejectBuildlessFile(sourceFile: SourceFile, tsCommentPositions: TsCommentPosition[]) { - const fs = require('fs'); - - interface TextSpan { - start: number - end: number - } - - const text = sourceFile.text; - const spansToRemove: TextSpan[] = [] - // Calculate what comment delimiters to remove from the text - { - for (const { open, colonPos, close, colonCount, containedInnerOpeningBlockComment } of tsCommentPositions) { - // Handle the opening comment - { - let start = open; - let end = colonPos; - // If "::", advance past it. - // Otherwise, its ":", and we don't want to touch that. - if (colonCount === 2) { - end += 2; - if (text[end] === ' ') end++; - } - if (colonCount === 1 && text[start - 1] === ' ') { - start--; - } - // In the case of `x /* :: ?: ... */` we want to ignore the whitespace between `x` and `/*`. - else if (colonCount === 2 && text[end] === '?' && text[end + 1] === ':' && text[start - 1] === ' ') { - start--; - } - // In the case of `x /* :: !: ... */` we want to ignore the whitespace between `x` and `/*`. - else if (colonCount === 2 && text[end] === '!' && text[end + 1] === ':' && text[start - 1] === ' ') { - start--; - } - // In cases such as `import { x /*::, type y */ }` we want to ignore the whitespace between `x` and the comma. - else if (colonCount === 2 && text[colonPos + 2] === ',' && text[start - 1] === ' ') { - start--; - } - // If `/*::` is standing on its own line, remove that line - else if (colonCount === 2 && text[end] === '\n') { - const maybeStartOfLineIndex = tryMoveToStartOfLine(text, start, /*moveOnIndex*/ false); - // We remove the newline after the comment instead of before, - // because there might not be a newline before the comment if this - // is at the start of the file. - if (maybeStartOfLineIndex !== undefined) { - start = maybeStartOfLineIndex; - end++; - } - } - spansToRemove.push({ start, end }); - } - // Handle the closing comment - // In the case of `/*:: ... /* ... */`, we only want to remove the opening delimiter, not the closing one. - if (!containedInnerOpeningBlockComment) { - let start = close; - const end = close + 2; - if (text[start - 1] === ' ') { - start--; - } - // If `*/` is standing on its own line, remove that line - if (colonCount === 2 && [undefined, '\n'].includes(text[end])) { - const maybeStartOfLineIndex = tryMoveToStartOfLine(text, start, /*moveOnIndex*/ true); - if (maybeStartOfLineIndex !== undefined) { - start = maybeStartOfLineIndex; - } - } - spansToRemove.push({ start, end }); - } - } - } - - // Generated updated file text - let newText = ''; - { - spansToRemove.sort((a, b) => a.start - b.start); - let curPos = 0; - for (const spanToRemove of spansToRemove) { - newText += text.slice(curPos, spanToRemove.start); - curPos = Math.max(curPos, spanToRemove.end); - } - newText += text.slice(curPos); - } - - // Can't think of any reason for this condition to not be true, - // but we'll still handled it in case it can happen. - const newPath = sourceFile.path.endsWith('.js') - ? sourceFile.path.slice(0, -'.js'.length) + '.ts' - : sourceFile.path + '.ts'; - - fs.writeFileSync(newPath, newText); - fs.unlinkSync(sourceFile.path); - } - - function convertToBuildlessFile( - sourceFile: SourceFile, - tsSyntaxTracker: TsSyntaxTracker, - blockCommentDelimiters: Map - ) { - const fs = require('fs'); - const text = sourceFile.text; - - interface TSSyntaxRange { - start: number - end: number - openCommentIsFirstOnLine: boolean - closeCommentIsLastOnLine: boolean - multiline: boolean - openCommentIndent: string | undefined - } - - interface TextInsert { - at: number - value: string - } - - // Add metadata to the TS node positions - const tsSyntaxRanges: TSSyntaxRange[] = [...tsSyntaxTracker.getSyntaxRanges()] - .filter(syntaxRange => syntaxRange.type === SyntaxRangeType.typeScript) - .map(tsSyntaxRange => { - const maybeStartOfLineIndex = tryMoveToStartOfLine(text, tsSyntaxRange.start, /*moveOnIndex*/ false); - const openCommentIsFirstOnLine = maybeStartOfLineIndex !== undefined; - const openCommentIndent = maybeStartOfLineIndex !== undefined - ? text.slice(maybeStartOfLineIndex, tsSyntaxRange.start) - : undefined; - - const closeCommentIsLastOnLine = tryMoveToEndOfLine(text, tsSyntaxRange.end) !== undefined; - - const multiline = text.slice(tsSyntaxRange.start, tsSyntaxRange.end).includes('\n'); - - return { ...tsSyntaxRange, openCommentIsFirstOnLine, closeCommentIsLastOnLine, multiline, openCommentIndent }; - }); - - // Merge node-positions that are next to each other. - const tsSyntaxRanges2: TSSyntaxRange[] = []; - if (tsSyntaxRanges.length > 0) { - tsSyntaxRanges2.push(tsSyntaxRanges.shift()!); - for (const tsNodePosition of tsSyntaxRanges) { - const lastNodePosition = tsSyntaxRanges2.at(-1)!; - const textInBetween = text.slice(lastNodePosition.end, tsNodePosition.start); - let merged = false; - - const mergeNodePositionWithPrevious = () => { - lastNodePosition.end = tsNodePosition.end; - lastNodePosition.multiline ||= tsNodePosition.multiline || textInBetween.includes('\n'); - lastNodePosition.closeCommentIsLastOnLine = tsNodePosition.closeCommentIsLastOnLine; - merged = true; - }; - - if (lastNodePosition.openCommentIsFirstOnLine && tsNodePosition.closeCommentIsLastOnLine) { - if (textInBetween.match(/^[ \t\n]*$/g)) { - mergeNodePositionWithPrevious(); - } - } - if (!merged && !lastNodePosition.multiline && !tsNodePosition.multiline) { - if (textInBetween.match(/^[ ]*$/g)) { - mergeNodePositionWithPrevious(); - } - } - - if (!merged) { - tsSyntaxRanges2.push(tsNodePosition); - } - } - } - - // Calculate text to insert - const textInserts: TextInsert[] = []; - for (const tsSyntaxRange of tsSyntaxRanges2) { - let moveDelimitersOnTheirOwnLine = tsSyntaxRange.multiline && tsSyntaxRange.openCommentIsFirstOnLine && tsSyntaxRange.closeCommentIsLastOnLine; - const maybeColonIndex = lookForChar(text, ':', [' '], tsSyntaxRange.start, tsSyntaxRange.end); - if (maybeColonIndex !== undefined) { - moveDelimitersOnTheirOwnLine = false; // `/*:-style comments won't go on a line of their own - textInserts.push({ - at: maybeColonIndex, - value: ' /*' // `/*` with the colon right after makes `/*:`. - }) - } - else if (moveDelimitersOnTheirOwnLine) { - textInserts.push({ - at: tsSyntaxRange.start, - value: `/*::\n${tsSyntaxRange.openCommentIndent}`, - }); - } - else { - const maybeQuestionMarkIndex = lookForChar(text, '?', [' '], tsSyntaxRange.start, tsSyntaxRange.end); - const maybeExclamationMarkIndex = lookForChar(text, '!', [' '], tsSyntaxRange.start, tsSyntaxRange.end); - // Is this a "?:" or "!:" annotation? If so, we'll handle white-space a little differently. - const isTypeAnnotation = ( - (maybeQuestionMarkIndex !== undefined && text[maybeQuestionMarkIndex + 1] === ':') || - (maybeExclamationMarkIndex !== undefined && text[maybeExclamationMarkIndex + 1] === ':') - ); - const spaceBefore = text[tsSyntaxRange.start - 1] === ' '; - textInserts.push({ - at: tsSyntaxRange.start, - value: isTypeAnnotation && !spaceBefore ? ' /*:: ' : '/*:: ', - }); - } - - // Special handling for inner block comments. - // If the range we want to comment out contains a `/* ... */`, we'll - // want to add a `/*::` after the `*/` to bring us back into TS-comment land. - for (let i = tsSyntaxRange.start + 1; i < tsSyntaxRange.end - 1; i++) { - const delimiter = blockCommentDelimiters.get(i); - if (delimiter === BlockCommentDelimiterType.close) { - textInserts.push({ - at: i + 2, - value: '/*::' - }); - } - } - - if (moveDelimitersOnTheirOwnLine) { - textInserts.push({ - at: tsSyntaxRange.end, - value: `\n${tsSyntaxRange.openCommentIndent}*/`, - }); - } - else { - textInserts.push({ - at: tsSyntaxRange.end, - value: ' */', - }); - } - } - - // Generated updated file text - let newText = ''; - { - let curPos = 0; - for (const textInsert of textInserts) { - newText += text.slice(curPos, textInsert.at) + textInsert.value; - curPos = textInsert.at; - } - newText += text.slice(curPos); - } - - // Can't think of any reason for this condition to not be true, - // but we'll still handled it in case it can happen. - const newPath = sourceFile.path.endsWith('.ts') - ? sourceFile.path.slice(0, -'.ts'.length) + '.js' - : sourceFile.path + '.js'; - - fs.writeFileSync(newPath, newText); - fs.unlinkSync(sourceFile.path); - } - - function lookForChar(text: string, charToFind: string, charsToNotBreakOn: string[], startAt: number, stopAt: number): number | undefined { - for (let i = startAt; i < stopAt; i++) { - const char = text[i]; - if (char === charToFind) return i; - if (charsToNotBreakOn.includes(char)) continue; - return undefined; - } - return undefined; - } - - /** - * Returns an index if you can make it to the start by passing through only whitespace. - * Returns undefined if there are other characters in the way. - * The character that "index" is currently on will be ignored, only characters before it will be considered.. - * @param moveOnIndex if false, returns the index right after the new line. - */ - function tryMoveToStartOfLine(text: string, index: number, moveOnIndex: boolean) { - while (true) { - index--; - if (text[index] === ' ' || text[index] === '\t') continue; - if (text[index] === undefined) return index + 1; - if (text[index] === '\n') return moveOnIndex ? index : index + 1; - return undefined; - } - } - - /** - * Returns an index if you can make it to the end by passing through only whitespace. - * Returns undefined if there are other characters in the way. - */ - function tryMoveToEndOfLine(text: string, index: number) { - index--; - while (true) { - index++; - if (text[index] === ' ' || text[index] === '\t') continue; - if (text[index] === undefined) return index - 1; - if (text[index] === '\n') return index; - return undefined; - } - } - // BUILDLESS: }); } diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 6a3149a39abaf..4b20b00eb6789 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -2,8 +2,8 @@ import { append, arraysEqual, binarySearch, - BlockCommentDelimiterPositions, // BUILDLESS: added - BlockCommentDelimiterType, // BUILDLESS: added + BlockCommentDelimiterPositions, + BlockCommentDelimiterType, CharacterCodes, CommentDirective, CommentDirectiveType, @@ -35,7 +35,7 @@ import { SyntaxKind, TextRange, TokenFlags, - TsCommentScannerState, // BUILDLESS: added + TsCommentScannerState, } from "./_namespaces/ts.js"; export type ErrorCallback = (message: DiagnosticMessage, length: number, arg0?: any) => void; @@ -131,7 +131,6 @@ export interface Scanner { tryScan(callback: () => T): T; } -// BUILDLESS: export interface TsCommentPosition { open: number; colonPos: number; @@ -162,7 +161,6 @@ export interface TsCommentScanner { scanUntilNextSignificantChar(max: number): boolean; scanUntilAtChar(char: string | string[], max: number): void; scanUntilPastChar(char: string | string[], max: number): void; - getBlockCommentDelimiters(): Map; getTSCommentPositions(): TsCommentPosition[]; } @@ -170,7 +168,6 @@ export interface TsSyntaxTracker { skipWhitespaceThenMarkRangeAsTS(start: number, end: number): void; getSyntaxRanges(): Generator } -// BUILDLESS: /** @internal */ export const textToKeywordObj: MapLike = { @@ -724,7 +721,6 @@ export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boole } if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { pos += 2; - // BUILDLESS: let isTsComment = false; while (pos < text.length) { if (text.charCodeAt(pos) === CharacterCodes.space) { @@ -764,7 +760,6 @@ export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boole pos += 3; } } - // BUILDLESS: while (pos < text.length) { if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { pos += 2; @@ -802,6 +797,10 @@ export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boole canConsumeStar = false; continue; } + if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { + pos += 2; + continue; + } break; default: @@ -966,7 +965,6 @@ function iterateCommentRanges(reduce: boolean, text: string, pos: number, } } else { - // BUILDLESS: let isTsComment = false; while (pos < text.length) { if (text.charCodeAt(pos) === CharacterCodes.space) { @@ -1005,7 +1003,6 @@ function iterateCommentRanges(reduce: boolean, text: string, pos: number, pos += 3; } } - // BUILDLESS: while (pos < text.length) { if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { pos += 2; @@ -2087,6 +2084,15 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean pos++; return token = SyntaxKind.CloseParenToken; case CharacterCodes.asterisk: + if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { + pos += 2; + // Unset the "enteringTSComment" bit. + // It may be set in the case where we entered and exited a TS comment without + // the comment actually having any body, like this: `/*:: */`. + // In this case, we just want to skip over it as if the comment wasn't there. + tokenFlags &= ~TokenFlags.enteringTSComment; + continue; + } if (charCodeUnchecked(pos + 1) === CharacterCodes.equals) { return pos += 2, token = SyntaxKind.AsteriskEqualsToken; } @@ -2165,7 +2171,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean pos += 2; const isJSDoc = charCodeUnchecked(pos) === CharacterCodes.asterisk && charCodeUnchecked(pos + 1) !== CharacterCodes.slash; - // BUILDLESS: let isTsComment = false; while (pos < end) { if (text.charCodeAt(pos) === CharacterCodes.space) { @@ -2206,7 +2211,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean pos += 3; } } - // BUILDLESS: let commentClosed = false; let lastLineStart = tokenStart; @@ -4085,7 +4089,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean } } -// BUILDLESS: export function createTSCommentScanner( { text }: SourceFileLike, _pos?: number, @@ -4107,7 +4110,6 @@ export function createTSCommentScanner( * (with some exceptions for tracking information that's needed for code transformations) */ clone() { - // console.log(`CLONING SCANNER - expect duplicate scans past this point.`) // BUILDLESS: DEBUG: comment-scan return createTSCommentScanner({ text }, pos, state, _delimiterPositions); }, /** @@ -4123,8 +4125,7 @@ export function createTSCommentScanner( * @returns true if the just-scanned node was inside a TS comment, otherwise returns false. * Also returns false when given a node that couldn't be scanned, because the scanner already passed that point. */ - scanThroughLeafNode(node: { kind: SyntaxKind, pos: number, end: number }): boolean { - // console.log(`SCAN ${SyntaxKind[(node as any).kind]} from:${pos} to-node-start:${node.pos} hard-limit-at-node-end:${node.end}${node.end < pos ? ' FAIL' : ''}`); // BUILDLESS: DEBUG: comment-scan + scanThroughLeafNode(node: { pos: number, end: number }): boolean { if (node.end < pos) return false; commentScan(text, pos, node.pos, /*stopOnSignificantText*/ false, onCommentDelimiter); commentScan(text, node.pos, node.end, /*stopOnSignificantText*/ true, onCommentDelimiter); @@ -4141,7 +4142,6 @@ export function createTSCommentScanner( * Also returns false when nothing could be scanned, because the scanner already passed that point. */ scanTo(end: number): boolean { - // console.log(`SCAN to ${end}.${end < pos ? ' FAIL' : ''}`) // BUILDLESS: DEBUG: comment-scan if (end < pos) return false; const significantTextAt = commentScan(text, pos, end, /*stopOnSignificantText*/ true, onCommentDelimiter); let allInTSComment = state === TsCommentScannerState.inTSBlockComment; @@ -4160,7 +4160,6 @@ export function createTSCommentScanner( * Otherwise, returns false. Also returns false when nothing could be scanned, because the scanner already passed that point. */ scanUntilNextSignificantChar(max: number) { - // console.log(`SCAN UNTIL NEXT SIGNIFICANT CHAR (max: ${max})${max < pos ? ' FAIL' : ''}`); // BUILDLESS: DEBUG: comment-scan if (max < pos) return false; commentScan(text, pos, max, /*stopOnSignificantText*/ true, onCommentDelimiter); return state === TsCommentScannerState.inTSBlockComment; @@ -4170,7 +4169,6 @@ export function createTSCommentScanner( * @param max Mostly a failsafe - if something goes wrong, we'll stop jumping at this point. */ scanUntilAtChar(char: string | string[], max: number) { - // console.log(`SCAN UNTIL AT ${JSON.stringify(char)}, (max: ${max})}`); // BUILDLESS: DEBUG: comment-scan const charList: string[] = Array.isArray(char) ? char : [char]; while (pos < max) { commentScan(text, pos, max, /*stopOnSignificantText*/ true, onCommentDelimiter) @@ -4183,7 +4181,6 @@ export function createTSCommentScanner( * @param max Mostly a failsafe - if something goes wrong, we'll stop jumping at this point. */ scanUntilPastChar(char: string | string[], max: number) { - // console.log(`SCAN UNTIL PAST ${JSON.stringify(char)}, (max: ${max})}`); // BUILDLESS: DEBUG: comment-scan const charList: string[] = Array.isArray(char) ? char : [char]; while (pos < max) { commentScan(text, pos, max, /*stopOnSignificantText*/ true, onCommentDelimiter) @@ -4195,9 +4192,6 @@ export function createTSCommentScanner( pos++; } }, - getBlockCommentDelimiters(): Map { - return delimiterPositions; - }, /** * Returns all TS-comment pairs in the file. * Prerequisites: @@ -4274,12 +4268,10 @@ export function createTSSyntaxTracker({ text }: SourceFileLike): TsSyntaxTracker */ skipWhitespaceThenMarkRangeAsTS(start: number, end: number) { // Skip past whitespace/comments - // console.log('(iterating as part of mark-range-as-ts - this does not alter a scanner instance)'); // BUILDLESS: DEBUG: comment-scan const realStart = commentScan(text, start, end, /*stopOnSignificantText*/ true); if (start < realStart) { whitespaceSyntaxRanges.push({ start, end: realStart, type: SyntaxRangeType.whitespace }); } - // console.log(`Mark TypeScript range - from ${realStart} to ${end} (original start: ${start})`); // BUILDLESS: DEBUG: comment-scan if (realStart < end) { tsSyntaxRanges.push({ start: realStart, end, type: SyntaxRangeType.typeScript }); } @@ -4351,12 +4343,10 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant if (pos >= end) { return pos; } - // console.log(` char-scan ${JSON.stringify(text[pos])} (at ${pos})${pos >= end ? ' (past soft limit)' : ''}`) // BUILDLESS: DEBUG: comment-scan if (pos === 0) { // Special handling for shebang if (ch === CharacterCodes.hash && isShebangTrivia(text, pos)) { - // console.log(' she-bang skip'); // BUILDLESS: DEBUG: comment-scan pos = scanShebangTrivia(text, pos); Debug.assertLessThan(pos, end); continue; @@ -4395,7 +4385,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant case CharacterCodes.asterisk: if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { onCommentDelimiter?.(pos, BlockCommentDelimiterType.close) - // console.log(' closing comment skip'); // BUILDLESS: DEBUG: comment-scan pos += 2; continue; } @@ -4414,7 +4403,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant } pos++; } - // console.log(' single line comment skip'); // BUILDLESS: DEBUG: comment-scan continue; } // Multi-line comment @@ -4452,7 +4440,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant : BlockCommentDelimiterType.doubleColonForOpenTsComment ); pos += colonCount; - // console.log(' ts opening comment skip'); // BUILDLESS: DEBUG: comment-scan continue; } else { // `/*::: ... */` @@ -4463,7 +4450,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant } onCommentDelimiter?.(saveStartPos, BlockCommentDelimiterType.openNonTsComment) - // console.log(' non-ts block comment skip'); // BUILDLESS: DEBUG: comment-scan while (pos < end) { const ch = text.charCodeAt(pos); @@ -4489,7 +4475,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant } } } -// BUILDLESS: /** @internal */ function codePointAt(s: string, i: number): number { diff --git a/src/compiler/types.ts b/src/compiler/types.ts index d376a8008eacb..26fed35e68d90 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -2812,6 +2812,7 @@ export const enum TokenFlags { ContainsLeadingZero = 1 << 13, // e.g. `0888` /** @internal */ ContainsInvalidSeparator = 1 << 14, // e.g. `0_1` + enteringTSComment = 1 << 15, // This bit is set if, just before this token, we entered a TS comment. /** @internal */ BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier, /** @internal */ @@ -9933,8 +9934,6 @@ export const enum ListFormat { JSDocComment = MultiLine | AsteriskDelimited, } -// BUILDLESS: - export const enum BlockCommentDelimiterType { openNonTsComment, // A `/*` not followed by colons openTsCommentWithoutColons, // Just the `/*` part of `/*:` or `/*::`. @@ -9953,7 +9952,6 @@ export const enum TsCommentScannerState { inBlockComment, inTSBlockComment, } -// BUILDLESS: /** @internal */ export const enum PragmaKindFlags { diff --git a/src/compiler/utilities.ts b/src/compiler/utilities.ts index 4cf5f0082cd20..29984046ed407 100644 --- a/src/compiler/utilities.ts +++ b/src/compiler/utilities.ts @@ -4969,13 +4969,13 @@ export function getClassExtendsHeritageElement(node: ClassLikeDeclaration | Inte /** @internal */ export function getEffectiveImplementsTypeNodes(node: ClassLikeDeclaration): undefined | readonly ExpressionWithTypeArguments[] { - if (isInJSFile(node)) { + const heritageClause = getHeritageClause(node.heritageClauses, SyntaxKind.ImplementsKeyword); + const result = heritageClause?.types; + // fall back to jsdocs if its a TypeScript file + if (result === undefined && isInJSFile(node)) { return getJSDocImplementsTags(node).map(n => n.class); } - else { - const heritageClause = getHeritageClause(node.heritageClauses, SyntaxKind.ImplementsKeyword); - return heritageClause?.types; - } + return result; } /** From 064a7ab327b18a0ddc5b5d0cc8cec026bf449f3e Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Tue, 21 May 2024 00:26:50 -0600 Subject: [PATCH 09/12] Fix minor issues from earlier merge conflict resolution --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6197eb4408588..3314c58f49221 100644 --- a/README.md +++ b/README.md @@ -47,4 +47,4 @@ with any additional questions or comments. ## Roadmap -For details on our planned features and future direction, please refer to our [roadmap](https://github.com/microsoft/TypeScript/wiki/Roadmap). \ No newline at end of file +For details on our planned features and future direction, please refer to our [roadmap](https://github.com/microsoft/TypeScript/wiki/Roadmap). diff --git a/package.json b/package.json index 35d8d9abe52a6..e479356ebb9a5 100644 --- a/package.json +++ b/package.json @@ -110,4 +110,4 @@ "node": "20.1.0", "npm": "8.19.4" } -} \ No newline at end of file +} From 5ed05febab489d57ca821e7e9fcc0340399b2e3b Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Tue, 21 May 2024 00:46:28 -0600 Subject: [PATCH 10/12] Fix some more merge-conflict-related issues --- src/compiler/program.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index b9ae4290c2e92..08e90bc75dd35 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -320,6 +320,7 @@ import { toPath as ts_toPath, trace, tracing, + tryCast, TsCommentPosition, TsCommentScanner, TsConfigSourceFile, From f770cf192f8d377f0843e455d5fa998f57c8eaf0 Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Tue, 21 May 2024 00:54:43 -0600 Subject: [PATCH 11/12] More conflict fixes --- src/compiler/scanner.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 4b20b00eb6789..c2741c3786e40 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -4476,7 +4476,6 @@ function commentScan(text: string, start: number, end: number, stopOnSignificant } } -/** @internal */ function codePointAt(s: string, i: number): number { // TODO(jakebailey): this is wrong and should have ?? 0; but all users are okay with it return s.codePointAt(i)!; From 6926b46f73157bdabefcf418da2d2653e133f713 Mon Sep 17 00:00:00 2001 From: Scotty Jamison Date: Tue, 21 May 2024 00:58:09 -0600 Subject: [PATCH 12/12] Remove comment about a feature that was removed --- src/compiler/program.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 08e90bc75dd35..fe5f02d879dfb 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -3111,9 +3111,6 @@ export function createProgram(rootNamesOrOptions: readonly string[] | CreateProg which goes and traverses the subtree of our current node, looking for all leaves, then it'll run the scanner across those leaves, asking the scanner at each step if the leaf was inside or outside of a TS comment. If all leaves were inside a TS comment, we'll skip reporting a "this can only be used in JS files" error. - - There's also a fair amount of work that goes into tracking the exact boundaries of TypeScript syntax, - so if we wish to convert a TypeScript file to JavaScript+comment, we'd be able to do so. */ function getJSSyntacticDiagnosticsForFile(sourceFile: SourceFile): DiagnosticWithLocation[] {