Add dts bundling

Author: jakebailey

Diff for: .eslintignore

@@ -20,3 +20,4 @@
 /scripts/eslint/built/**
 /internal/**
 /coverage/**
+/dtstesting/**

Diff for: .gitignore

@@ -47,6 +47,7 @@ scripts/configurePrerelease.js
 scripts/configureLanguageServiceBuild.js
 scripts/open-user-pr.js
 scripts/open-cherry-pick-pr.js
+scripts/dtsBundler.js
 scripts/processDiagnosticMessages.d.ts
 scripts/processDiagnosticMessages.js
 scripts/produceLKG.js

Diff for: Gulpfile.js

@@ -127,6 +127,33 @@ const buildSrc = () => buildProject("src");
 // But, if we are bundling, we are running only d.ts emit, so maybe this is fast?
 task("build-src", series(preSrc, buildSrc));
 
+// TODO(jakebailey): split this in two, only build the relavent parts, move down before tests.
+const dtsBundler = async () => {
+    /**
+     * @param {string} projectPath
+     * @param {string} entrypoint
+     * @param {string} output
+     */
+    async function runDtsBundler(projectPath, entrypoint, output) {
+        await exec(process.execPath, ["./scripts/dtsBundler.js", projectPath, entrypoint, output]);
+    }
+
+    // TODO(jakebailey): prepend copyright notice, replace const enums with regular enums
+    if (needsUpdate(["./built/local/typescript/tsconfig.tsbuildinfo", "./scripts/dtsBundler.js"], "./built/local/typescript.d.ts")) {
+        await runDtsBundler("./src/typescript/tsconfig.json", "./built/local/typescript/typescript.d.ts", "./built/local/typescript.d.ts");
+    }
+
+    if (needsUpdate(["./built/local/tsserverlibrary/tsconfig.tsbuildinfo", "./scripts/dtsBundler.js"], "./built/local/tsserverlibrary.d.ts")) {
+        await runDtsBundler("./src/tsserverlibrary/tsconfig.json", "./built/local/tsserverlibrary/tsserverlibrary.d.ts", "./built/local/tsserverlibrary.d.ts");
+    }
+};
+
+// TODO(jakebailey): Some tests depend on the extracted output.
+// TODO(jakebailey): implicit dependency on scripts
+const dts = series(preSrc, buildSrc, dtsBundler);
+dts.displayName = "dts";
+task("dts", dts);
+
 /** @type {string | undefined} */
 let copyrightHeader;
 function getCopyrightHeader() {
@@ -474,7 +501,7 @@ const preTest = parallel(buildTsc, buildTests, buildServices, buildLssl);
 preTest.displayName = "preTest";
 
 const runTests = () => runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
-task("runtests", series(preBuild, preTest, runTests));
+task("runtests", series(preBuild, preTest, dts, runTests));
 task("runtests").description = "Runs the tests using the built run.js file.";
 task("runtests").flags = {
     "-t --tests=<regex>": "Pattern for tests to run.",
@@ -493,7 +520,7 @@ task("runtests").flags = {
 };
 
 const runTestsParallel = () => runConsoleTests(testRunner, "min", /*runInParallel*/ cmdLineOptions.workers > 1, /*watchMode*/ false);
-task("runtests-parallel", series(preBuild, preTest, runTestsParallel));
+task("runtests-parallel", series(preBuild, preTest, dts, runTestsParallel));
 task("runtests-parallel").description = "Runs all the tests in parallel using the built run.js file.";
 task("runtests-parallel").flags = {
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster).",
@@ -597,8 +624,7 @@ const produceLKG = async () => {
     }
 };
 
-// TODO(jakebailey): dependencies on dts
-task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs), produceLKG));
+task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts), produceLKG));
 task("LKG").description = "Makes a new LKG out of the built js files";
 task("LKG").flags = {
     "   --built": "Compile using the built version of the compiler.",

Diff for: scripts/dtsBundler.ts

@@ -0,0 +1,131 @@
+import * as assert from "assert";
+import * as fs from "fs";
+import * as path from "path";
+import * as ts from "../lib/typescript";
+
+// (ts as any).Debug.enableDebugInfo();
+
+const [
+    projectPath,
+    entrypoint,
+    output,
+] = process.argv.slice(2);
+
+assert(projectPath);
+assert(entrypoint);
+assert(output);
+
+console.log(`Bundling ${entrypoint} to ${output}, using project ${projectPath}`);
+
+const newLineKind = ts.NewLineKind.LineFeed;
+const newLine = newLineKind === ts.NewLineKind.LineFeed ? "\n" : "\r\n";
+
+function isDeclarationStatement(node: ts.Node): node is ts.DeclarationStatement {
+    return (ts as any).isDeclarationStatement(node);
+}
+
+const compareStringsCaseSensitive: (a: string | undefined, b: string | undefined) => number = (ts as any).compareStringsCaseSensitive;
+
+
+const tsconfigContents = ts.readJsonConfigFile(path.join(projectPath, "tsconfig.json"), ts.sys.readFile);
+const tsconfig = ts.parseJsonConfigFileContent(tsconfigContents, ts.sys, projectPath);
+const program = ts.createProgram([entrypoint], tsconfig.options);
+
+const typeChecker = program.getTypeChecker();
+
+const sourceFile = program.getSourceFile(entrypoint);
+assert(sourceFile);
+const moduleSymbol = typeChecker.getSymbolAtLocation(sourceFile);
+assert(moduleSymbol);
+
+const printer = ts.createPrinter({ newLine: newLineKind });
+
+
+const lines: string[] = [];
+const indent = "    ";
+let currentIndent = "";
+
+function increaseIndent() {
+    currentIndent += indent;
+}
+
+function decreaseIndent() {
+    currentIndent = currentIndent.slice(indent.length);
+}
+
+function write(s = "") {
+    if (!s) {
+        lines.push("");
+    }
+    else {
+        lines.push(...s.split(/\r?\n/).map((line) => (currentIndent + line).trimEnd()));
+    }
+}
+
+const copyrightNotice = fs.readFileSync(path.join(__dirname, "..", "CopyrightNotice.txt"), "utf-8");
+write(copyrightNotice);
+
+function emitAsNamespace(name: string, moduleSymbol: ts.Symbol) {
+    if (name === "ts") {
+        // We will write `export = ts` at the end.
+        write(`declare namespace ${name} {`);
+    }
+    else {
+        write(`export namespace ${name} {`);
+    }
+    increaseIndent();
+
+    let moduleExports = typeChecker.getExportsOfModule(moduleSymbol);
+    moduleExports = moduleExports.slice(0).sort((a, b) => compareStringsCaseSensitive(a.name, b.name));
+
+    for (const me of moduleExports) {
+        if (!me.declarations?.length) {
+            continue;
+        }
+
+        if (me.flags & ts.SymbolFlags.Alias) {
+            const resolved = typeChecker.getAliasedSymbol(me);
+            assert(resolved.flags & ts.SymbolFlags.ValueModule);
+            emitAsNamespace(me.name, resolved);
+            continue;
+        }
+
+        for (const decl of me.declarations) {
+            let output: ts.Node | undefined;
+
+            if (ts.isVariableDeclaration(decl)) {
+                output = ts.factory.createVariableStatement(
+                    [ts.factory.createToken(ts.SyntaxKind.ExportKeyword)],
+                    ts.factory.createVariableDeclarationList([decl], ts.getCombinedNodeFlags(decl))
+                );
+            }
+            else if (isDeclarationStatement(decl)) {
+                output = decl;
+            }
+
+            if (output) {
+                let printed = printer.printNode(ts.EmitHint.Unspecified, output, decl.getSourceFile());
+                // TODO(jakebailey): This could be less hacky.
+                printed = printed.replace("export declare ", "export ");
+                printed = printed.replace("export const enum ", "export enum ");
+                write(printed);
+                write();
+            }
+            else {
+                const declSourceFile = decl.getSourceFile();
+                const lc = declSourceFile.getLineAndCharacterOfPosition(decl.pos);
+                throw new Error(`Unhandled declaration for ${me.name} at ${declSourceFile.fileName}:${lc.line}:${lc.character}`);
+            }
+        }
+    }
+
+    decreaseIndent();
+    write(`}`);
+    write();
+}
+
+emitAsNamespace("ts", moduleSymbol);
+
+write("export = ts;");
+
+fs.writeFileSync(output, lines.join(newLine));

Diff for: src/compiler/commandLineParser.ts

@@ -2081,7 +2081,7 @@ function getTsconfigRootOptionsMap() {
 }
 
 /** @internal */
-interface JsonConversionNotifier {
+export interface JsonConversionNotifier {
     /**
      * Notifies parent option object is being set with the optionKey and a valid optionValue
      * Currently it notifies only if there is element with type object (parentOption) and

Diff for: src/compiler/factory/utilities.ts

@@ -2,11 +2,12 @@ import {
     AccessorDeclaration, addEmitFlags, AdditiveOperator, AdditiveOperatorOrHigher, AssertionLevel,
     AssignmentOperatorOrHigher, BinaryExpression, BinaryOperator, BinaryOperatorToken, BindingOrAssignmentElement,
     BindingOrAssignmentElementRestIndicator, BindingOrAssignmentElementTarget, BindingOrAssignmentPattern,
-    BitwiseOperator, BitwiseOperatorOrHigher, BooleanLiteral, CharacterCodes, CommaListExpression,
+    BitwiseOperator, BitwiseOperatorOrHigher, Block, BooleanLiteral, CharacterCodes, CommaListExpression,
     compareStringsCaseSensitive, CompilerOptions, Debug, Declaration, EmitFlags, EmitHelperFactory, EmitHost,
     EmitResolver, EntityName, EqualityOperator, EqualityOperatorOrHigher, ExclamationToken, ExponentiationOperator,
     ExportDeclaration, Expression, ExpressionStatement, externalHelpersModuleNameText, first, firstOrUndefined,
     ForInitializer, GeneratedIdentifier, GeneratedIdentifierFlags, GeneratedNamePart, GeneratedPrivateIdentifier,
+    GetAccessorDeclaration,
     getAllAccessorDeclarations, getEmitFlags, getEmitHelpers, getEmitModuleKind, getESModuleInterop,
     getExternalModuleName, getExternalModuleNameFromPath, getJSDocType, getJSDocTypeTag, getModifiers,
     getNamespaceDeclarationNode, getOrCreateEmitNode, getOriginalNode, getParseTreeNode,
@@ -26,7 +27,7 @@ import {
     NumericLiteral, ObjectLiteralElementLike, ObjectLiteralExpression, or, OuterExpression, OuterExpressionKinds,
     outFile, parseNodeFactory, PlusToken, PostfixUnaryExpression, PrefixUnaryExpression, PrivateIdentifier,
     PropertyAssignment, PropertyDeclaration, PropertyName, pushIfUnique, QuestionToken, ReadonlyKeyword,
-    RelationalOperator, RelationalOperatorOrHigher, setOriginalNode, setParent, setStartsOnNewLine, setTextRange,
+    RelationalOperator, RelationalOperatorOrHigher, SetAccessorDeclaration, setOriginalNode, setParent, setStartsOnNewLine, setTextRange,
     ShiftOperator, ShiftOperatorOrHigher, ShorthandPropertyAssignment, some, SourceFile, Statement, StringLiteral,
     SyntaxKind, TextRange, ThisTypeNode, Token, TypeNode, TypeParameterDeclaration,
 } from "../_namespaces/ts";
@@ -185,7 +186,7 @@ export function createForOfBindingStatement(factory: NodeFactory, node: ForIniti
 }
 
 /** @internal */
-export function insertLeadingStatement(factory: NodeFactory, dest: Statement, source: Statement) {
+export function insertLeadingStatement(factory: NodeFactory, dest: Statement, source: Statement): Block {
     if (isBlock(dest)) {
         return factory.updateBlock(dest, setTextRange(factory.createNodeArray([source, ...dest.statements]), dest.statements));
     }
@@ -1456,7 +1457,7 @@ export function createAccessorPropertyBackingField(factory: NodeFactory, node: P
  *
  * @internal
  */
-export function createAccessorPropertyGetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName) {
+export function createAccessorPropertyGetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName): GetAccessorDeclaration {
     return factory.createGetAccessorDeclaration(
         modifiers,
         name,
@@ -1478,7 +1479,7 @@ export function createAccessorPropertyGetRedirector(factory: NodeFactory, node:
  *
  * @internal
  */
-export function createAccessorPropertySetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName) {
+export function createAccessorPropertySetRedirector(factory: NodeFactory, node: PropertyDeclaration, modifiers: ModifiersArray | undefined, name: PropertyName): SetAccessorDeclaration {
     return factory.createSetAccessorDeclaration(
         modifiers,
         name,

Diff for: src/compiler/moduleNameResolver.ts

@@ -128,7 +128,7 @@ function createResolvedModuleWithFailedLookupLocations(
 }
 
 /** @internal */
-interface ModuleResolutionState {
+export interface ModuleResolutionState {
     host: ModuleResolutionHost;
     compilerOptions: CompilerOptions;
     traceEnabled: boolean;
@@ -146,7 +146,7 @@ interface ModuleResolutionState {
  *
  * @internal
  */
-interface PackageJsonPathFields {
+export interface PackageJsonPathFields {
     typings?: string;
     types?: string;
     typesVersions?: MapLike<MapLike<string[]>>;
@@ -226,7 +226,7 @@ function readPackageJsonTypesVersionsField(jsonContent: PackageJson, state: Modu
 }
 
 /** @internal */
-interface VersionPaths {
+export interface VersionPaths {
     version: string;
     paths: MapLike<string[]>;
 }
@@ -1281,7 +1281,7 @@ export function resolveJSModule(moduleName: string, initialDir: string, host: Mo
 }
 
 /** @internal */
-enum NodeResolutionFeatures {
+export enum NodeResolutionFeatures {
     None = 0,
     // resolving `#local` names in your own package.json
     Imports = 1 << 1,

Diff for: src/compiler/parser.ts

@@ -9831,7 +9831,7 @@ export function processCommentPragmas(context: PragmaContext, sourceText: string
 }
 
 /** @internal */
-type PragmaDiagnosticReporter = (pos: number, length: number, message: DiagnosticMessage) => void;
+export type PragmaDiagnosticReporter = (pos: number, length: number, message: DiagnosticMessage) => void;
 
 /** @internal */
 export function processPragmasIntoFields(context: PragmaContext, reportDiagnostic: PragmaDiagnosticReporter): void {

Diff for: src/compiler/perfLogger.ts

@@ -1,6 +1,38 @@
 import { noop } from "./_namespaces/ts";
 
-type PerfLogger = typeof import("@microsoft/typescript-etw");
+/** @internal */
+export interface PerfLogger {
+    logEvent(msg: string): void;
+    logErrEvent(msg: string): void;
+    logPerfEvent(msg: string): void;
+    logInfoEvent(msg: string): void;
+    logStartCommand(command: string, msg: string): void;
+    logStopCommand(command: string, msg: string): void;
+    logStartUpdateProgram(msg: string): void;
+    logStopUpdateProgram(msg: string): void;
+    logStartUpdateGraph(): void;
+    logStopUpdateGraph(): void;
+    logStartResolveModule(name: string): void;
+    logStopResolveModule(success: string): void;
+    logStartParseSourceFile(filename: string): void;
+    logStopParseSourceFile(): void;
+    logStartReadFile(filename: string): void;
+    logStopReadFile(): void;
+    logStartBindFile(filename: string): void;
+    logStopBindFile(): void;
+    logStartScheduledOperation(operationId: string): void;
+    logStopScheduledOperation(): void;
+}
+
+type ImportedPerfLogger = typeof import("@microsoft/typescript-etw");
+
+// Assert that our PerfLogger type is compatible with the library.
+// Doing this keeps the above import expression out of our emitted d.ts file.
+const _perfLoggerCorrectType: PerfLogger extends ImportedPerfLogger ? true : false = true;
+
+// eslint-disable-next-line @typescript-eslint/no-unused-expressions
+_perfLoggerCorrectType;
+
 const nullLogger: PerfLogger = {
     logEvent: noop,
     logErrEvent: noop,

Diff for: src/compiler/program.ts

@@ -210,7 +210,7 @@ export function createCompilerHostWorker(options: CompilerOptions, setParentNode
 }
 
 /** @internal */
-interface CompilerHostLikeForCache {
+export interface CompilerHostLikeForCache {
     fileExists(fileName: string): boolean;
     readFile(fileName: string, encoding?: string): string | undefined;
     directoryExists?(directory: string): boolean;
@@ -570,7 +570,7 @@ export interface SourceFileImportsList {
  * Calculates the resulting resolution mode for some reference in some file - this is generally the explicitly
  * provided resolution mode in the reference, unless one is not present, in which case it is the mode of the containing file.
  */
-export function getModeForFileReference(ref: FileReference | string, containingFileMode: SourceFile["impliedNodeFormat"]) {
+export function getModeForFileReference(ref: FileReference | string, containingFileMode: SourceFile["impliedNodeFormat"]): ModuleKind.CommonJS | ModuleKind.ESNext | undefined {
     return (isString(ref) ? containingFileMode : ref.resolutionMode) || containingFileMode;
 }
 
@@ -612,7 +612,7 @@ export function isExclusivelyTypeOnlyImportOrExport(decl: ImportDeclaration | Ex
  * @param usage The module reference string
  * @returns The final resolution mode of the import
  */
-export function getModeForUsageLocation(file: {impliedNodeFormat?: SourceFile["impliedNodeFormat"]}, usage: StringLiteralLike) {
+export function getModeForUsageLocation(file: {impliedNodeFormat?: SourceFile["impliedNodeFormat"]}, usage: StringLiteralLike): ModuleKind.CommonJS | ModuleKind.ESNext | undefined {
     if (file.impliedNodeFormat === undefined) return undefined;
     if ((isImportDeclaration(usage.parent) || isExportDeclaration(usage.parent))) {
         const isTypeOnly = isExclusivelyTypeOnlyImportOrExport(usage.parent);
@@ -4318,7 +4318,7 @@ export function filterSemanticDiagnostics(diagnostic: readonly Diagnostic[], opt
 }
 
 /** @internal */
-interface CompilerHostLike {
+export interface CompilerHostLike {
     useCaseSensitiveFileNames(): boolean;
     getCurrentDirectory(): string;
     fileExists(fileName: string): boolean;