Port microsoft#7486 to release-1.8

Author: zhengbli

src/compiler/commandLineParser.ts

@@ -305,6 +305,11 @@ namespace ts {
             name: "noCustomAsyncPromise",
             type: "boolean",
             experimental: true
+        },
+        {
+            name: "disableSizeLimit",
+            type: "boolean",
+            description: Diagnostics.Disable_the_upper_limit_for_the_total_file_size_of_a_project
         }
     ];
 
@@ -534,7 +539,7 @@ namespace ts {
                 }
                 else {
                     // by default exclude node_modules, and any specificied output directory
-                    exclude = ["node_modules"];
+                    exclude = ["node_modules", "bower_components"];
                     const outDir = json["compilerOptions"] && json["compilerOptions"]["outDir"];
                     if (outDir) {
                         exclude.push(outDir);

src/compiler/diagnosticMessages.json

@@ -2458,6 +2458,10 @@
         "category": "Message",
         "code": 6112
     },
+    "Disable the upper limit for the total file size of a project.": {
+        "category": "Message",
+        "code": 6113
+    },
     "Variable '{0}' implicitly has an '{1}' type.": {
         "category": "Error",
         "code": 7005
@@ -2661,5 +2665,9 @@
     "Unknown typing option '{0}'.": {
         "category": "Error",
         "code": 17010
+    },
+    "Too many JavaScript files in the project. Use an exact 'files' list, or use the 'exclude' setting in project configuration to limit included source folders. The likely folder to exclude is '{0}'. To disable the project size limit, set the 'disableSizeLimit' compiler option to 'true'": {
+        "category": "Error",
+        "code": 17012
     }
 }

src/compiler/program.ts

@@ -394,13 +394,41 @@ namespace ts {
                 (oldOptions.target !== options.target) ||
                 (oldOptions.noLib !== options.noLib) ||
                 (oldOptions.jsx !== options.jsx) ||
-                (oldOptions.allowJs !== options.allowJs)) {
+                (oldOptions.allowJs !== options.allowJs) ||
+                (oldOptions.disableSizeLimit !== options.disableSizeLimit)) {
                 oldProgram = undefined;
             }
         }
 
         if (!tryReuseStructureFromOldProgram()) {
-            forEach(rootNames, name => processRootFile(name, /*isDefaultLib*/ false));
+            if (options.disableSizeLimit === true) {
+                forEach(rootNames, name => processRootFile(name, /*isDefaultLib*/ false));
+            }
+            else {
+                let programSize = 0;
+                for (const name of rootNames) {
+                    const path = toPath(name, currentDirectory, getCanonicalFileName);
+                    if (programSize <= maxProgramSize) {
+                        processRootFile(name, /*isDefaultLib*/ false);
+                        const file = filesByName.get(path);
+                        if (!hasTypeScriptFileExtension(name) && file && file.text) {
+                            programSize += file.text.length;
+                        }
+                    }
+                    else {
+                        // If the program size limit was reached when processing a file, this file is
+                        // likely in the problematic folder than contains too many files
+                        const commonSourceDirectory = getCommonSourceDirectory();
+                        let rootLevelDirectory = path.substring(0, Math.max(commonSourceDirectory.length, path.indexOf(directorySeparator, commonSourceDirectory.length)));
+                        if (rootLevelDirectory[rootLevelDirectory.length - 1] !== directorySeparator) {
+                            rootLevelDirectory += directorySeparator;
+                        }
+                        programDiagnostics.add(createCompilerDiagnostic(Diagnostics.Too_many_JavaScript_files_in_the_project_Use_an_exact_files_list_or_use_the_exclude_setting_in_project_configuration_to_limit_included_source_folders_The_likely_folder_to_exclude_is_0_To_disable_the_project_size_limit_set_the_disableSizeLimit_compiler_option_to_true, rootLevelDirectory));
+                        break;
+                    }
+                }
+            }
+
             // Do not process the default library if:
             //  - The '--noLib' flag is used.
             //  - A 'no-default-lib' reference comment is encountered in

src/compiler/sys.ts

@@ -74,7 +74,7 @@ namespace ts {
         watchDirectory?(path: string, callback: DirectoryWatcherCallback, recursive?: boolean): FileWatcher;
     };
 
-    export var sys: System = (function () {
+    export var sys: System = (function() {
 
         function getWScriptSystem(): System {
 
@@ -407,8 +407,8 @@ namespace ts {
             const watchedFileSet = createWatchedFileSet();
 
             function isNode4OrLater(): boolean {
-                 return parseInt(process.version.charAt(1)) >= 4;
-             }
+                return parseInt(process.version.charAt(1)) >= 4;
+            }
 
             const platform: string = _os.platform();
             // win32\win64 are case insensitive platforms, MacOS (darwin) by default is also case insensitive
@@ -477,15 +477,20 @@ namespace ts {
                     for (const current of files) {
                         const name = combinePaths(path, current);
                         if (!contains(exclude, getCanonicalPath(name))) {
-                            const stat = _fs.statSync(name);
-                            if (stat.isFile()) {
-                                if (!extension || fileExtensionIs(name, extension)) {
-                                    result.push(name);
+                            // fs.statSync would throw an exception if the file is a symlink
+                            // whose linked file doesn't exist.
+                            try {
+                                const stat = _fs.statSync(name);
+                                if (stat.isFile()) {
+                                    if (!extension || fileExtensionIs(name, extension)) {
+                                        result.push(name);
+                                    }
+                                }
+                                else if (stat.isDirectory()) {
+                                    directories.push(name);
                                 }
                             }
-                            else if (stat.isDirectory()) {
-                                directories.push(name);
-                            }
+                            catch (e) { }
                         }
                     }
                     for (const current of directories) {
@@ -509,7 +514,7 @@ namespace ts {
                     // and https://github.com/Microsoft/TypeScript/issues/4643), therefore
                     // if the current node.js version is newer than 4, use `fs.watch` instead.
                     const watchSet = isNode4OrLater() ? watchedFileSet : pollingWatchedFileSet;
-                    const watchedFile =  watchSet.addFile(filePath, callback);
+                    const watchedFile = watchSet.addFile(filePath, callback);
                     return {
                         close: () => watchSet.removeFile(watchedFile)
                     };
@@ -539,7 +544,7 @@ namespace ts {
                         }
                     );
                 },
-                resolvePath: function (path: string): string {
+                resolvePath: function(path: string): string {
                     return _path.resolve(path);
                 },
                 fileExists(path: string): boolean {

src/compiler/types.ts

@@ -2437,6 +2437,7 @@ namespace ts {
         allowSyntheticDefaultImports?: boolean;
         allowJs?: boolean;
         noImplicitUseStrict?: boolean;
+        disableSizeLimit?: boolean;
         /* @internal */ stripInternal?: boolean;
 
         // Skip checking lib.d.ts to help speed up tests.

src/compiler/utilities.ts

@@ -2476,6 +2476,10 @@ namespace ts {
         return forEach(supportedJavascriptExtensions, extension => fileExtensionIs(fileName, extension));
     }
 
+    export function hasTypeScriptFileExtension(fileName: string) {
+        return forEach(supportedTypeScriptExtensions, extension => fileExtensionIs(fileName, extension));
+    }
+
     /**
      * Replace each instance of non-ascii characters by one, two, three, or four escape sequences
      * representing the UTF-8 encoding of the character, and return the expanded char code list.
@@ -2858,4 +2862,6 @@ namespace ts {
     export function isParameterPropertyDeclaration(node: ParameterDeclaration): boolean {
         return node.flags & NodeFlags.AccessibilityModifier && node.parent.kind === SyntaxKind.Constructor && isClassLike(node.parent.parent);
     }
+
+    export const maxProgramSize = 20 * 1024 * 1024;
 }

src/server/editorServices.ts

@@ -1217,10 +1217,43 @@ namespace ts.server {
             }
             else {
                 const project = this.createProject(configFilename, projectOptions);
+                let programSize = 0;
+
+                // As the project openning might not be complete if there are too many files,
+                // therefore to surface the diagnostics we need to make sure the given client file is opened.
+                if (clientFileName) {
+                    if (this.host.fileExists(clientFileName)) {
+                        const currentClientFileInfo = this.openFile(clientFileName, /*openedByClient*/ true);
+                        project.addRoot(currentClientFileInfo);
+                        if (!hasTypeScriptFileExtension(currentClientFileInfo.fileName) && currentClientFileInfo.content) {
+                            programSizeForNonTsFiles += currentClientFileInfo.content.length;
+                        }
+                    }
+                    else {
+                        return { errorMsg: "specified file " + clientFileName + " not found" };
+                    }
+                }
+
                 for (const rootFilename of projectOptions.files) {
+                    if (rootFilename === clientFileName) {
+                        continue;
+                    }
+
                     if (this.host.fileExists(rootFilename)) {
-                        const info = this.openFile(rootFilename, /*openedByClient*/ clientFileName == rootFilename);
-                        project.addRoot(info);
+                        if (projectOptions.compilerOptions.disableSizeLimit) {
+                            const info = this.openFile(rootFilename, /*openedByClient*/ false);
+                            project.addRoot(info);
+                        }
+                        else if (programSizeForNonTsFiles <= maxProgramSizeForNonTsFiles) {
+                            const info = this.openFile(rootFilename, /*openedByClient*/ false);
+                            project.addRoot(info);
+                            if (!hasTypeScriptFileExtension(rootFilename)) {
+                                programSizeForNonTsFiles += info.content.length;
+                            }
+                        }
+                        else {
+                            break;
+                        }
                     }
                     else {
                         return { errorMsg: "specified file " + rootFilename + " not found" };
@@ -1251,7 +1284,10 @@ namespace ts.server {
                     return error;
                 }
                 else {
-                    const oldFileNames = project.compilerService.host.roots.map(info => info.fileName);
+                    // if the project is too large, the root files might not have been all loaded if the total
+                    // program size reached the upper limit. In that case project.projectOptions.files should 
+                    // be more precise. However this would only happen for configured project.
+                    const oldFileNames = project.projectOptions ? project.projectOptions.files : project.compilerService.host.roots.map(info => info.fileName);
                     const newFileNames = projectOptions.files;
                     const fileNamesToRemove = oldFileNames.filter(f => newFileNames.indexOf(f) < 0);
                     const fileNamesToAdd = newFileNames.filter(f => oldFileNames.indexOf(f) < 0);