Add build via esbuild

This configures the existing build tasks to use esbuild by defualt. If
using the plain files is desired, passing `--bundle=false` will build
using plain files and still produce a runnable system.

Author: jakebailey

Gulpfile.js

@@ -69,6 +69,7 @@
         "chalk": "^4.1.2",
         "del": "^6.1.1",
         "diff": "^5.1.0",
+        "esbuild": "^0.15.8",
         "eslint": "^8.22.0",
         "eslint-formatter-autolinkable-stylish": "^1.2.0",
         "eslint-plugin-import": "^2.26.0",

package-lock.json

@@ -6,7 +6,7 @@ const ci = ["1", "true"].includes(process.env.CI);
 
 /** @type {CommandLineOptions} */
 module.exports = minimist(process.argv.slice(2), {
-    boolean: ["dirty", "light", "colors", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built", "ci"],
+    boolean: ["dirty", "light", "colors", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built", "ci", "bundle"],
     string: ["browser", "tests", "break", "host", "reporter", "stackTraceLimit", "timeout", "shards", "shardId"],
     alias: {
         /* eslint-disable quote-props */
@@ -41,6 +41,7 @@ module.exports = minimist(process.argv.slice(2), {
         dirty: false,
         built: false,
         ci,
+        bundle: true
     }
 });
 
@@ -49,7 +50,7 @@ if (module.exports.built) {
 }
 
 /**
- * @typedef TypedOptions
+ * @typedef CommandLineOptions
  * @property {boolean} dirty
  * @property {boolean} light
  * @property {boolean} colors
@@ -69,7 +70,9 @@ if (module.exports.built) {
  * @property {boolean} failed
  * @property {boolean} keepFailed
  * @property {boolean} ci
- *
- * @typedef {import("minimist").ParsedArgs & TypedOptions} CommandLineOptions
+ * @property {boolean} bundle
+ * @property {string} shards
+ * @property {string} shardId
+ * @property {string} break
  */
 void 0;

package.json

@@ -2,67 +2,57 @@
 const { exec, Debouncer } = require("./utils");
 const { resolve } = require("path");
 const { findUpRoot } = require("./findUpDir");
+const cmdLineOptions = require("./options");
 
 class ProjectQueue {
     /**
-     * @param {(projects: string[], lkg: boolean, force: boolean) => Promise<any>} action
+     * @param {(projects: string[]) => Promise<any>} action
      */
     constructor(action) {
-        /** @type {{ lkg: boolean, force: boolean, projects?: string[], debouncer: Debouncer }[]} */
-        this._debouncers = [];
-        this._action = action;
+        /** @type {string[] | undefined} */
+        this._projects = undefined;
+        this._debouncer = new Debouncer(100, async () => {
+            const projects = this._projects;
+            if (projects) {
+                this._projects = undefined;
+                await action(projects);
+            }
+        });
     }
 
     /**
      * @param {string} project
-     * @param {object} options
      */
-    enqueue(project, { lkg = true, force = false } = {}) {
-        let entry = this._debouncers.find(entry => entry.lkg === lkg && entry.force === force);
-        if (!entry) {
-            const debouncer = new Debouncer(100, async () => {
-                const projects = entry.projects;
-                if (projects) {
-                    entry.projects = undefined;
-                    await this._action(projects, lkg, force);
-                }
-            });
-            this._debouncers.push(entry = { lkg, force, debouncer });
-        }
-        if (!entry.projects) entry.projects = [];
-        entry.projects.push(project);
-        return entry.debouncer.enqueue();
+    enqueue(project) {
+        if (!this._projects) this._projects = [];
+        this._projects.push(project);
+        return this._debouncer.enqueue();
     }
 }
 
-const execTsc = (/** @type {boolean} */ lkg, /** @type {string[]} */ ...args) =>
+const execTsc = (/** @type {string[]} */ ...args) =>
     exec(process.execPath,
-         [resolve(findUpRoot(), lkg ? "./lib/tsc" : "./built/local/tsc"),
+         [resolve(findUpRoot(), cmdLineOptions.lkg ? "./lib/tsc" : "./built/local/tsc"),
           "-b", ...args],
          { hidePrompt: true });
 
-const projectBuilder = new ProjectQueue((projects, lkg, force) => execTsc(lkg, ...(force ? ["--force"] : []), ...projects));
+const projectBuilder = new ProjectQueue((projects) => execTsc(...projects));
 
 /**
  * @param {string} project
- * @param {object} options
- * @param {boolean} [options.lkg=true]
- * @param {boolean} [options.force=false]
  */
-exports.buildProject = (project, { lkg, force } = {}) => projectBuilder.enqueue(project, { lkg, force });
+exports.buildProject = (project) => projectBuilder.enqueue(project);
 
-const projectCleaner = new ProjectQueue((projects, lkg) => execTsc(lkg, "--clean", ...projects));
+const projectCleaner = new ProjectQueue((projects) => execTsc("--clean", ...projects));
 
 /**
  * @param {string} project
  */
 exports.cleanProject = (project) => projectCleaner.enqueue(project);
 
-const projectWatcher = new ProjectQueue((projects) => execTsc(/*lkg*/ true, "--watch", ...projects));
+const projectWatcher = new ProjectQueue((projects) => execTsc("--watch", ...projects));
 
 /**
  * @param {string} project
- * @param {object} options
- * @param {boolean} [options.lkg=true]
  */
-exports.watchProject = (project, { lkg } = {}) => projectWatcher.enqueue(project, { lkg });
+exports.watchProject = (project) => projectWatcher.enqueue(project);

scripts/build/options.js

@@ -123,17 +123,17 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode)
             errorStatus = exitCode;
             error = new Error(`Process exited with status code ${errorStatus}.`);
         }
-        else if (cmdLineOptions.ci) {
-            // finally, do a sanity check and build the compiler with the built version of itself
-            log.info("Starting sanity check build...");
-            // Cleanup everything except lint rules (we'll need those later and would rather not waste time rebuilding them)
-            await exec("gulp", ["clean-tsc", "clean-services", "clean-tsserver", "clean-lssl", "clean-tests"]);
-            const { exitCode } = await exec("gulp", ["local", "--lkg=false"]);
-            if (exitCode !== 0) {
-                errorStatus = exitCode;
-                error = new Error(`Sanity check build process exited with status code ${errorStatus}.`);
-            }
-        }
+        // else if (cmdLineOptions.ci) {
+        //     // finally, do a sanity check and build the compiler with the built version of itself
+        //     log.info("Starting sanity check build...");
+        //     // Cleanup everything except lint rules (we'll need those later and would rather not waste time rebuilding them)
+        //     await exec("gulp", ["clean-tsc", "clean-services", "clean-tsserver", "clean-lssl", "clean-tests"]);
+        //     const { exitCode } = await exec("gulp", ["local", "--lkg=false"]);
+        //     if (exitCode !== 0) {
+        //         errorStatus = exitCode;
+        //         error = new Error(`Sanity check build process exited with status code ${errorStatus}.`);
+        //     }
+        // }
     }
     catch (e) {
         errorStatus = undefined;

scripts/build/prepend.js

@@ -7,7 +7,6 @@
 const fs = require("fs");
 const path = require("path");
 const log = require("fancy-log");
-const mkdirp = require("mkdirp");
 const del = require("del");
 const File = require("vinyl");
 const ts = require("../../lib/typescript");
@@ -225,90 +224,6 @@ function getDirSize(root) {
 }
 exports.getDirSize = getDirSize;
 
-/**
- * Flattens a project with project references into a single project.
- * @param {string} projectSpec The path to a tsconfig.json file or its containing directory.
- * @param {string} flattenedProjectSpec The output path for the flattened tsconfig.json file.
- * @param {FlattenOptions} [options] Options used to flatten a project hierarchy.
- *
- * @typedef FlattenOptions
- * @property {string} [cwd] The path to use for the current working directory. Defaults to `process.cwd()`.
- * @property {import("../../lib/typescript").CompilerOptions} [compilerOptions] Compiler option overrides.
- * @property {boolean} [force] Forces creation of the output project.
- * @property {string[]} [exclude] Files to exclude (relative to `cwd`)
- */
-function flatten(projectSpec, flattenedProjectSpec, options = {}) {
-    const cwd = normalizeSlashes(options.cwd ? path.resolve(options.cwd) : process.cwd());
-    const files = [];
-    const resolvedOutputSpec = path.resolve(cwd, flattenedProjectSpec);
-    const resolvedOutputDirectory = path.dirname(resolvedOutputSpec);
-    const resolvedProjectSpec = resolveProjectSpec(projectSpec, cwd, /*referrer*/ undefined);
-    const project = readJson(resolvedProjectSpec);
-    const skipProjects = /**@type {Set<string>}*/(new Set());
-    const skipFiles = new Set(options && options.exclude && options.exclude.map(file => normalizeSlashes(path.resolve(cwd, file))));
-    recur(resolvedProjectSpec, project);
-
-    if (options.force || needsUpdate(files, resolvedOutputSpec)) {
-        const config = {
-            extends: normalizeSlashes(path.relative(resolvedOutputDirectory, resolvedProjectSpec)),
-            compilerOptions: options.compilerOptions || {},
-            files: files.map(file => normalizeSlashes(path.relative(resolvedOutputDirectory, file)))
-        };
-        mkdirp.sync(resolvedOutputDirectory);
-        fs.writeFileSync(resolvedOutputSpec, JSON.stringify(config, undefined, 2), "utf8");
-    }
-
-    /**
-     * @param {string} projectSpec
-     * @param {object} project
-     */
-    function recur(projectSpec, project) {
-        if (skipProjects.has(projectSpec)) return;
-        skipProjects.add(project);
-        if (project.references) {
-            for (const ref of project.references) {
-                const referencedSpec = resolveProjectSpec(ref.path, cwd, projectSpec);
-                const referencedProject = readJson(referencedSpec);
-                recur(referencedSpec, referencedProject);
-            }
-        }
-        if (project.include) {
-            throw new Error("Flattened project may not have an 'include' list.");
-        }
-        if (!project.files) {
-            throw new Error("Flattened project must have an explicit 'files' list.");
-        }
-        const projectDirectory = path.dirname(projectSpec);
-        for (let file of project.files) {
-            file = normalizeSlashes(path.resolve(projectDirectory, file));
-            if (skipFiles.has(file)) continue;
-            skipFiles.add(file);
-            files.push(file);
-        }
-    }
-}
-exports.flatten = flatten;
-
-/**
- * @param {string} file
- */
-function normalizeSlashes(file) {
-    return file.replace(/\\/g, "/");
-}
-
-/**
- * @param {string} projectSpec
- * @param {string} cwd
- * @param {string | undefined} referrer
- * @returns {string}
- */
-function resolveProjectSpec(projectSpec, cwd, referrer) {
-    const projectPath = normalizeSlashes(path.resolve(cwd, referrer ? path.dirname(referrer) : "", projectSpec));
-    const stats = fs.statSync(projectPath);
-    if (stats.isFile()) return normalizeSlashes(projectPath);
-    return normalizeSlashes(path.resolve(cwd, projectPath, "tsconfig.json"));
-}
-
 /**
  * @param {string | ((file: File) => string) | { cwd?: string }} [dest]
  * @param {{ cwd?: string }} [opts]

scripts/build/projects.js

@@ -62,7 +62,6 @@ async function copyScriptOutputs() {
     await copyWithCopyright("cancellationToken.js");
     await copyWithCopyright("tsc.release.js", "tsc.js");
     await copyWithCopyright("tsserver.js");
-    await copyWithCopyright("dynamicImportCompat.js");
     await copyFromBuiltLocal("tsserverlibrary.js"); // copyright added by build
     await copyFromBuiltLocal("typescript.js"); // copyright added by build
     await copyFromBuiltLocal("typescriptServices.js"); // copyright added by build

scripts/build/tests.js

@@ -1,7 +1,9 @@
 {
     "extends": "../tsconfig-base",
     "compilerOptions": {
-        "outDir": "../../built/local/cancellationToken",
+        "outDir": "../../built/local",
+        "tsBuildInfoFile": "../../built/local/cancellationToken.tsbuildinfo",
+        "rootDir": ".",
         "module": "commonjs",
         "types": [
             "node"

scripts/build/utils.js

@@ -721,7 +721,7 @@ export namespace Debug {
         try {
             if (sys && sys.require) {
                 const basePath = getDirectoryPath(resolvePath(sys.getExecutingFilePath()));
-                const result = sys.require(basePath, "./compiler-debug") as RequireResult<ExtendedDebugModule>;
+                const result = sys.require(basePath, "./compilerDebug") as RequireResult<ExtendedDebugModule>;
                 if (!result.error) {
                     result.module.init(ts);
                     extendedDebugModule = result.module;

scripts/produceLKG.ts

@@ -1466,6 +1466,13 @@ export let sys: System = (() => {
         const useCaseSensitiveFileNames = isFileSystemCaseSensitive();
         const fsRealpath = !!_fs.realpathSync.native ? process.platform === "win32" ? fsRealPathHandlingLongPath : _fs.realpathSync.native : _fs.realpathSync;
 
+        // If our filename is "sys.js", then we are executing unbundled on the raw tsc output.
+        // In that case, simulate a faked path in the directory where a bundle would normally
+        // appear (e.g. the directory containing lib.*.d.ts files).
+        //
+        // Note that if we ever emit as files like cjs/mjs, this check will be wrong.
+        const executingFilePath = __filename.endsWith("sys.js") ? _path.join(_path.dirname(__dirname), "__fake__.js") : __filename;
+
         const fsSupportsRecursiveFsWatch = isNode4OrLater && (process.platform === "win32" || process.platform === "darwin");
         const getCurrentDirectory = memoize(() => process.cwd());
         const { watchFile, watchDirectory } = createSystemWatchFunctions({
@@ -1525,14 +1532,7 @@ export let sys: System = (() => {
                 }
             },
             getExecutingFilePath() {
-                // This function previously returned a path like `built/local/tsc.js`.
-                // Now, with a module output, this file is now `built/local/compiler/sys.js`.
-                // We want to return a file that looks like the old one, so that callers
-                // can locate other assets like the lib.d.ts files.
-                //
-                // TODO(jakebailey): replace this function with one that returns the path
-                // to the lib folder (or package path)?.
-                return _path.join(_path.dirname(__dirname), "fake.js");
+                return executingFilePath;
             },
             getCurrentDirectory,
             getDirectories,
@@ -1571,7 +1571,10 @@ export let sys: System = (() => {
             debugMode: !!process.env.NODE_INSPECTOR_IPC || !!process.env.VSCODE_INSPECTOR_OPTIONS || some(process.execArgv as string[], arg => /^--(inspect|debug)(-brk)?(=\d+)?$/i.test(arg)),
             tryEnableSourceMapsForHost() {
                 try {
-                    require("source-map-support").install();
+                    // Trick esbuild into not eagerly resolving a path to a JS file.
+                    // See: https://github.com/evanw/esbuild/issues/1958
+                    const moduleName = "source-map-support" as const;
+                    (require(moduleName) as typeof import("source-map-support")).install();
                 }
                 catch {
                     // Could not enable source maps.