Add upper limit for the program size to prevent tsserver from crashing

src/compiler/commandLineParser.ts

@@ -331,6 +331,10 @@ namespace ts {
             name: "noImplicitUseStrict",
             type: "boolean",
             description: Diagnostics.Do_not_emit_use_strict_directives_in_module_output
+        },
+        {
+            name: "disableSizeLimit",
+            type: "boolean"
         }
     ];
 
@@ -622,7 +626,7 @@ namespace ts {
                 }
                 else {
                     // by default exclude node_modules, and any specificied output directory
-                    exclude = ["node_modules"];
+                    exclude = ["node_modules", "bower_components"];
                     const outDir = json["compilerOptions"] && json["compilerOptions"]["outDir"];
                     if (outDir) {
                         exclude.push(outDir);
@@ -633,10 +637,22 @@ namespace ts {
                 const supportedExtensions = getSupportedExtensions(options);
                 Debug.assert(indexOf(supportedExtensions, ".ts") < indexOf(supportedExtensions, ".d.ts"), "Changed priority of extensions to pick");
 
+                const potentialFiles: string[] = [];
+                if (host.readDirectoryWithMultipleExtensions) {
+                    addRange(potentialFiles, host.readDirectoryWithMultipleExtensions(basePath, supportedExtensions, exclude));
+                }
+                else {
+                    for (const extension of supportedExtensions) {
+                        addRange(potentialFiles, host.readDirectory(basePath, extension, exclude));
+                    }
+                }
                 // Get files of supported extensions in their order of resolution
                 for (const extension of supportedExtensions) {
-                    const filesInDirWithExtension = host.readDirectory(basePath, extension, exclude);
-                    for (const fileName of filesInDirWithExtension) {
+                    for (const fileName of potentialFiles) {
+                        if (!fileExtensionIs(fileName, extension)) {
+                            continue;
+                        }
+
                         // .ts extension would read the .d.ts extension files too but since .d.ts is lower priority extension,
                         // lets pick them when its turn comes up
                         if (extension === ".ts" && fileExtensionIs(fileName, ".d.ts")) {
@@ -657,8 +673,10 @@ namespace ts {
                             }
                         }
 
-                        filesSeen[fileName] = true;
-                        fileNames.push(fileName);
+                        if (!filesSeen[fileName]) {
+                            filesSeen[fileName] = true;
+                            fileNames.push(fileName);
+                        }
                     }
                 }
             }

src/compiler/diagnosticMessages.json

@@ -2807,5 +2807,9 @@
     "Unknown typing option '{0}'.": {
         "category": "Error",
         "code": 17010
+    },
+    "Too many JavaScript files in the project. Consider specifying the 'exclude' setting in project configuration to limit included source folders. The likely folder to exclude is '{0}'. To disable the project size limit, set the 'disableSizeLimit' compiler option to 'true'.": {
+        "category": "Error",
+        "code": 17012
     }
 }

src/compiler/program.ts

@@ -7,6 +7,7 @@ namespace ts {
     /* @internal */ export let emitTime = 0;
     /* @internal */ export let ioReadTime = 0;
     /* @internal */ export let ioWriteTime = 0;
+    /* @internal */ export const maxProgramSizeForNonTsFiles = 20 * 1024 * 1024;
 
     /** The version of the TypeScript compiler release */
 
@@ -696,6 +697,8 @@ namespace ts {
         let diagnosticsProducingTypeChecker: TypeChecker;
         let noDiagnosticsTypeChecker: TypeChecker;
         let classifiableNames: Map<string>;
+        let programSizeLimitExceeded = false;
+        let programSizeForNonTsFiles = 0;
 
         let skipDefaultLib = options.noLib;
         const supportedExtensions = getSupportedExtensions(options);
@@ -742,7 +745,8 @@ namespace ts {
                 (oldOptions.target !== options.target) ||
                 (oldOptions.noLib !== options.noLib) ||
                 (oldOptions.jsx !== options.jsx) ||
-                (oldOptions.allowJs !== options.allowJs)) {
+                (oldOptions.allowJs !== options.allowJs) ||
+                (oldOptions.disableSizeLimit !== options.disableSizeLimit)) {
                 oldProgram = undefined;
             }
         }
@@ -790,6 +794,11 @@ namespace ts {
 
         return program;
 
+        function exceedProgramSizeLimit() {
+            return !options.disableSizeLimit && programSizeLimitExceeded;
+        }
+
+
         function getCommonSourceDirectory() {
             if (typeof commonSourceDirectory === "undefined") {
                 if (options.rootDir && checkSourceFilesBelongToPath(files, options.rootDir)) {
@@ -1415,7 +1424,7 @@ namespace ts {
                 }
             }
 
-            if (diagnostic) {
+            if (diagnostic && !exceedProgramSizeLimit()) {
                 if (refFile !== undefined && refEnd !== undefined && refPos !== undefined) {
                     fileProcessingDiagnostics.add(createFileDiagnostic(refFile, refPos, refEnd - refPos, diagnostic, ...diagnosticArgument));
                 }
@@ -1452,6 +1461,11 @@ namespace ts {
                 return file;
             }
 
+            const isNonTsFile = !hasTypeScriptFileExtension(fileName);
+            if (isNonTsFile && exceedProgramSizeLimit()) {
+                return undefined;
+            }
+
             // We haven't looked for this file, do so now and cache result
             const file = host.getSourceFile(fileName, options.target, hostErrorMessage => {
                 if (refFile !== undefined && refPos !== undefined && refEnd !== undefined) {
@@ -1463,6 +1477,25 @@ namespace ts {
                 }
             });
 
+            if (!options.disableSizeLimit && file && file.text && !hasTypeScriptFileExtension(file.fileName)) {
+                programSizeForNonTsFiles += file.text.length;
+                if (programSizeForNonTsFiles > maxProgramSizeForNonTsFiles) {
+                    // If the program size limit was reached when processing a file, this file is
+                    // likely in the problematic folder than contains too many files.
+                    // Normally the folder is one level down from the commonSourceDirectory, for example,
+                    // if the commonSourceDirectory is "/src/", and the last processed path was "/src/node_modules/a/b.js",
+                    // we should show in the error message "/src/node_modules/".
+                    const commonSourceDirectory = getCommonSourceDirectory();
+                    let rootLevelDirectory = path.substring(0, Math.max(commonSourceDirectory.length, path.indexOf(directorySeparator, commonSourceDirectory.length)));
+                    if (rootLevelDirectory[rootLevelDirectory.length - 1] !== directorySeparator) {
+                        rootLevelDirectory += directorySeparator;
+                    }
+                    programDiagnostics.add(createCompilerDiagnostic(Diagnostics.Too_many_JavaScript_files_in_the_project_Consider_specifying_the_exclude_setting_in_project_configuration_to_limit_included_source_folders_The_likely_folder_to_exclude_is_0_To_disable_the_project_size_limit_set_the_disableSizeLimit_compiler_option_to_true, rootLevelDirectory));
+                    programSizeLimitExceeded = true;
+                    return undefined;
+                }
+            }
+
             filesByName.set(path, file);
             if (file) {
                 file.wasReferenced = file.wasReferenced || isReference;

src/compiler/sys.ts

@@ -10,6 +10,7 @@ namespace ts {
         useCaseSensitiveFileNames: boolean;
         write(s: string): void;
         readFile(path: string, encoding?: string): string;
+        getFileSize?(path: string): number;
         writeFile(path: string, data: string, writeByteOrderMark?: boolean): void;
         watchFile?(path: Path, callback: FileWatcherCallback): FileWatcher;
         watchDirectory?(path: string, callback: DirectoryWatcherCallback, recursive?: boolean): FileWatcher;
@@ -20,6 +21,7 @@ namespace ts {
         getExecutingFilePath(): string;
         getCurrentDirectory(): string;
         readDirectory(path: string, extension?: string, exclude?: string[]): string[];
+        readDirectoryWithMultipleExtensions?(path: string, extensions: string[], exclude?: string[]): string[];
         getMemoryUsage?(): number;
         exit(exitCode?: number): void;
     }
@@ -74,7 +76,7 @@ namespace ts {
         watchDirectory?(path: string, callback: DirectoryWatcherCallback, recursive?: boolean): FileWatcher;
     };
 
-    export var sys: System = (function () {
+    export var sys: System = (function() {
 
         function getWScriptSystem(): System {
 
@@ -404,8 +406,8 @@ namespace ts {
             const watchedFileSet = createWatchedFileSet();
 
             function isNode4OrLater(): boolean {
-                 return parseInt(process.version.charAt(1)) >= 4;
-             }
+                return parseInt(process.version.charAt(1)) >= 4;
+            }
 
             const platform: string = _os.platform();
             // win32\win64 are case insensitive platforms, MacOS (darwin) by default is also case insensitive
@@ -489,34 +491,59 @@ namespace ts {
                 return fileSystemEntryExists(path, FileSystemEntryKind.Directory);
             }
 
-            function readDirectory(path: string, extension?: string, exclude?: string[]): string[] {
-                const result: string[] = [];
-                exclude = map(exclude, s => getCanonicalPath(combinePaths(path, s)));
-                visitDirectory(path);
-                return result;
-                function visitDirectory(path: string) {
-                    const files = _fs.readdirSync(path || ".").sort();
-                    const directories: string[] = [];
-                    for (const current of files) {
-                        const name = combinePaths(path, current);
-                        if (!contains(exclude, getCanonicalPath(name))) {
+            function visitDirectory(path: string, result: string[], extension: string | string[], exclude: string[]) {
+                const files = _fs.readdirSync(path || ".").sort();
+                const directories: string[] = [];
+                for (const current of files) {
+                    const name = combinePaths(path, current);
+                    if (!contains(exclude, getCanonicalPath(name))) {
+                        // fs.statSync would throw an exception if the file is a symlink
+                        // whose linked file doesn't exist.
+                        try {
                             const stat = _fs.statSync(name);
                             if (stat.isFile()) {
-                                if (!extension || fileExtensionIs(name, extension)) {
+                                if (checkExtension(name)) {
                                     result.push(name);
                                 }
                             }
                             else if (stat.isDirectory()) {
                                 directories.push(name);
                             }
                         }
+                        catch (e) { }
+                    }
+                }
+                for (const current of directories) {
+                    visitDirectory(current, result, extension, exclude);
+                }
+
+                function checkExtension(name: string) {
+                    if (!extension) {
+                        return true;
                     }
-                    for (const current of directories) {
-                        visitDirectory(current);
+                    if (typeof extension === "string") {
+                        return fileExtensionIs(name, extension);
+                    }
+                    else {
+                        return forEach(extension, ext => fileExtensionIs(name, ext));
                     }
                 }
             }
 
+            function readDirectoryWithMultipleExtensions(path: string, extensions: string[], exclude?: string[]): string[] {
+                const result: string[] = [];
+                exclude = map(exclude, s => getCanonicalPath(combinePaths(path, s)));
+                visitDirectory(path, result, extensions, exclude);
+                return result;
+            }
+
+            function readDirectory(path: string, extension?: string, exclude?: string[]): string[] {
+                const result: string[] = [];
+                exclude = map(exclude, s => getCanonicalPath(combinePaths(path, s)));
+                visitDirectory(path, result, extension, exclude);
+                return result;
+            }
+
             return {
                 args: process.argv.slice(2),
                 newLine: _os.EOL,
@@ -532,7 +559,7 @@ namespace ts {
                     // and https://github.com/Microsoft/TypeScript/issues/4643), therefore
                     // if the current node.js version is newer than 4, use `fs.watch` instead.
                     const watchSet = isNode4OrLater() ? watchedFileSet : pollingWatchedFileSet;
-                    const watchedFile =  watchSet.addFile(filePath, callback);
+                    const watchedFile = watchSet.addFile(filePath, callback);
                     return {
                         close: () => watchSet.removeFile(watchedFile)
                     };
@@ -562,7 +589,7 @@ namespace ts {
                         }
                     );
                 },
-                resolvePath: function (path: string): string {
+                resolvePath: function(path: string): string {
                     return _path.resolve(path);
                 },
                 fileExists,
@@ -579,12 +606,23 @@ namespace ts {
                     return process.cwd();
                 },
                 readDirectory,
+                readDirectoryWithMultipleExtensions,
                 getMemoryUsage() {
                     if (global.gc) {
                         global.gc();
                     }
                     return process.memoryUsage().heapUsed;
                 },
+                getFileSize(path) {
+                    try {
+                        const stat = _fs.statSync(path);
+                        if (stat.isFile()) {
+                            return stat.size;
+                        }
+                    }
+                    catch (e) { }
+                    return 0;
+                },
                 exit(exitCode?: number): void {
                     process.exit(exitCode);
                 }

src/compiler/types.ts

@@ -1581,6 +1581,7 @@ namespace ts {
 
     export interface ParseConfigHost {
         readDirectory(rootDir: string, extension: string, exclude: string[]): string[];
+        readDirectoryWithMultipleExtensions?(rootDir: string, extensions: string[], exclude: string[]): string[];
     }
 
     export interface WriteFileCallback {
@@ -2439,6 +2440,7 @@ namespace ts {
         allowSyntheticDefaultImports?: boolean;
         allowJs?: boolean;
         noImplicitUseStrict?: boolean;
+        disableSizeLimit?: boolean;
         lib?: string[];
         /* @internal */ stripInternal?: boolean;
 

src/compiler/utilities.ts

@@ -2484,6 +2484,10 @@ namespace ts {
         return forEach(supportedJavascriptExtensions, extension => fileExtensionIs(fileName, extension));
     }
 
+    export function hasTypeScriptFileExtension(fileName: string) {
+        return forEach(supportedTypeScriptExtensions, extension => fileExtensionIs(fileName, extension));
+    }
+
     /**
      * Replace each instance of non-ascii characters by one, two, three, or four escape sequences
      * representing the UTF-8 encoding of the character, and return the expanded char code list.