Skip to content

Commit 8bf3d54

Browse files
billtibillti
authored
Merge pull request #7075 from Microsoft/loadJsFromModules
Load JavaScript modules from Node packages
2 parents f9338c6 + ddad35a commit 8bf3d54

File tree

80 files changed

+721
-62
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+721
-62
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,4 @@ internal/
4949
**/.vscode
5050
!**/.vscode/tasks.json
5151
!tests/cases/projects/projectOption/**/node_modules
52+
!tests/cases/projects/NodeModulesSearch/**/*

‎src/compiler/commandLineParser.ts

+5
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,11 @@ namespace ts {
374374
type: "boolean",
375375
description: Diagnostics.Do_not_emit_use_strict_directives_in_module_output
376376
},
377+
{
378+
name: "maxNodeModuleJsDepth",
379+
type: "number",
380+
description: Diagnostics.The_maximum_dependency_depth_to_search_under_node_modules_and_load_JavaScript_files
381+
},
377382
{
378383
name: "listEmittedFiles",
379384
type: "boolean"

‎src/compiler/diagnosticMessages.json

+8-1
Original file line numberDiff line numberDiff line change
@@ -2804,7 +2804,14 @@
28042804
"category": "Message",
28052805
"code": 6135
28062806
},
2807-
2807+
"The maximum dependency depth to search under node_modules and load JavaScript files": {
2808+
"category": "Message",
2809+
"code": 6136
2810+
},
2811+
"No types specified in 'package.json' but 'allowJs' is set, so returning 'main' value of '{0}'": {
2812+
"category": "Message",
2813+
"code": 6137
2814+
},
28082815
"Variable '{0}' implicitly has an '{1}' type.": {
28092816
"category": "Error",
28102817
"code": 7005

‎src/compiler/program.ts

+69-11
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,10 @@ namespace ts {
130130
}
131131

132132
function tryReadTypesSection(packageJsonPath: string, baseDirectory: string, state: ModuleResolutionState): string {
133-
let jsonContent: { typings?: string, types?: string };
133+
let jsonContent: { typings?: string, types?: string, main?: string };
134134
try {
135135
const jsonText = state.host.readFile(packageJsonPath);
136-
jsonContent = jsonText ? <{ typings?: string, types?: string }>JSON.parse(jsonText) : {};
136+
jsonContent = jsonText ? <{ typings?: string, types?: string, main?: string }>JSON.parse(jsonText) : {};
137137
}
138138
catch (e) {
139139
// gracefully handle if readFile fails or returns not JSON
@@ -173,6 +173,14 @@ namespace ts {
173173
}
174174
return typesFilePath;
175175
}
176+
// Use the main module for inferring types if no types package specified and the allowJs is set
177+
if (state.compilerOptions.allowJs && jsonContent.main && typeof jsonContent.main === "string") {
178+
if (state.traceEnabled) {
179+
trace(state.host, Diagnostics.No_types_specified_in_package_json_but_allowJs_is_set_so_returning_main_value_of_0, jsonContent.main);
180+
}
181+
const mainFilePath = normalizePath(combinePaths(baseDirectory, jsonContent.main));
182+
return mainFilePath;
183+
}
176184
return undefined;
177185
}
178186

@@ -610,7 +618,7 @@ namespace ts {
610618
failedLookupLocations, supportedExtensions, state);
611619

612620
let isExternalLibraryImport = false;
613-
if (!resolvedFileName) {
621+
if (!resolvedFileName) {
614622
if (moduleHasNonRelativeName(moduleName)) {
615623
if (traceEnabled) {
616624
trace(host, Diagnostics.Loading_module_0_from_node_modules_folder, moduleName);
@@ -740,12 +748,13 @@ namespace ts {
740748
const nodeModulesFolder = combinePaths(directory, "node_modules");
741749
const nodeModulesFolderExists = directoryProbablyExists(nodeModulesFolder, state.host);
742750
const candidate = normalizePath(combinePaths(nodeModulesFolder, moduleName));
743-
// Load only typescript files irrespective of allowJs option if loading from node modules
744-
let result = loadModuleFromFile(candidate, supportedTypeScriptExtensions, failedLookupLocations, !nodeModulesFolderExists, state);
751+
const supportedExtensions = getSupportedExtensions(state.compilerOptions);
752+
753+
let result = loadModuleFromFile(candidate, supportedExtensions, failedLookupLocations, !nodeModulesFolderExists, state);
745754
if (result) {
746755
return result;
747756
}
748-
result = loadNodeModuleFromDirectory(supportedTypeScriptExtensions, candidate, failedLookupLocations, !nodeModulesFolderExists, state);
757+
result = loadNodeModuleFromDirectory(supportedExtensions, candidate, failedLookupLocations, !nodeModulesFolderExists, state);
749758
if (result) {
750759
return result;
751760
}
@@ -1057,6 +1066,23 @@ namespace ts {
10571066
let resolvedTypeReferenceDirectives: Map<ResolvedTypeReferenceDirective> = {};
10581067
let fileProcessingDiagnostics = createDiagnosticCollection();
10591068

1069+
// The below settings are to track if a .js file should be add to the program if loaded via searching under node_modules.
1070+
// This works as imported modules are discovered recursively in a depth first manner, specifically:
1071+
// - For each root file, findSourceFile is called.
1072+
// - This calls processImportedModules for each module imported in the source file.
1073+
// - This calls resolveModuleNames, and then calls findSourceFile for each resolved module.
1074+
// As all these operations happen - and are nested - within the createProgram call, they close over the below variables.
1075+
// The current resolution depth is tracked by incrementing/decrementing as the depth first search progresses.
1076+
const maxNodeModulesJsDepth = typeof options.maxNodeModuleJsDepth === "number" ? options.maxNodeModuleJsDepth : 2;
1077+
let currentNodeModulesJsDepth = 0;
1078+
1079+
// If a module has some of its imports skipped due to being at the depth limit under node_modules, then track
1080+
// this, as it may be imported at a shallower depth later, and then it will need its skipped imports processed.
1081+
const modulesWithElidedImports: Map<boolean> = {};
1082+
1083+
// Track source files that are JavaScript files found by searching under node_modules, as these shouldn't be compiled.
1084+
const jsFilesFoundSearchingNodeModules: Map<boolean> = {};
1085+
10601086
const start = new Date().getTime();
10611087

10621088
host = host || createCompilerHost(options);
@@ -1211,6 +1237,7 @@ namespace ts {
12111237
(oldOptions.rootDir !== options.rootDir) ||
12121238
(oldOptions.configFilePath !== options.configFilePath) ||
12131239
(oldOptions.baseUrl !== options.baseUrl) ||
1240+
(oldOptions.maxNodeModuleJsDepth !== options.maxNodeModuleJsDepth) ||
12141241
!arrayIsEqualTo(oldOptions.typeRoots, oldOptions.typeRoots) ||
12151242
!arrayIsEqualTo(oldOptions.rootDirs, options.rootDirs) ||
12161243
!mapIsEqualTo(oldOptions.paths, options.paths)) {
@@ -1335,6 +1362,7 @@ namespace ts {
13351362
getSourceFile: program.getSourceFile,
13361363
getSourceFileByPath: program.getSourceFileByPath,
13371364
getSourceFiles: program.getSourceFiles,
1365+
getFilesFromNodeModules: () => jsFilesFoundSearchingNodeModules,
13381366
writeFile: writeFileCallback || (
13391367
(fileName, data, writeByteOrderMark, onError, sourceFiles) => host.writeFile(fileName, data, writeByteOrderMark, onError, sourceFiles)),
13401368
isEmitBlocked,
@@ -1869,6 +1897,14 @@ namespace ts {
18691897
reportFileNamesDifferOnlyInCasingError(fileName, file.fileName, refFile, refPos, refEnd);
18701898
}
18711899

1900+
// See if we need to reprocess the imports due to prior skipped imports
1901+
if (file && lookUp(modulesWithElidedImports, file.path)) {
1902+
if (currentNodeModulesJsDepth < maxNodeModulesJsDepth) {
1903+
modulesWithElidedImports[file.path] = false;
1904+
processImportedModules(file, getDirectoryPath(fileName));
1905+
}
1906+
}
1907+
18721908
return file;
18731909
}
18741910

@@ -2007,16 +2043,38 @@ namespace ts {
20072043
for (let i = 0; i < moduleNames.length; i++) {
20082044
const resolution = resolutions[i];
20092045
setResolvedModule(file, moduleNames[i], resolution);
2046+
const resolvedPath = resolution ? toPath(resolution.resolvedFileName, currentDirectory, getCanonicalFileName) : undefined;
2047+
20102048
// add file to program only if:
20112049
// - resolution was successful
20122050
// - noResolve is falsy
20132051
// - module name comes from the list of imports
2014-
const shouldAddFile = resolution &&
2015-
!options.noResolve &&
2016-
i < file.imports.length;
2052+
// - it's not a top level JavaScript module that exceeded the search max
2053+
const isJsFileUnderNodeModules = resolution && resolution.isExternalLibraryImport &&
2054+
hasJavaScriptFileExtension(resolution.resolvedFileName);
2055+
2056+
if (isJsFileUnderNodeModules) {
2057+
jsFilesFoundSearchingNodeModules[resolvedPath] = true;
2058+
currentNodeModulesJsDepth++;
2059+
}
2060+
2061+
const elideImport = isJsFileUnderNodeModules && currentNodeModulesJsDepth > maxNodeModulesJsDepth;
2062+
const shouldAddFile = resolution && !options.noResolve && i < file.imports.length && !elideImport;
2063+
2064+
if (elideImport) {
2065+
modulesWithElidedImports[file.path] = true;
2066+
}
2067+
else if (shouldAddFile) {
2068+
findSourceFile(resolution.resolvedFileName,
2069+
resolvedPath,
2070+
/*isDefaultLib*/ false, /*isReference*/ false,
2071+
file,
2072+
skipTrivia(file.text, file.imports[i].pos),
2073+
file.imports[i].end);
2074+
}
20172075

2018-
if (shouldAddFile) {
2019-
findSourceFile(resolution.resolvedFileName, toPath(resolution.resolvedFileName, currentDirectory, getCanonicalFileName), /*isDefaultLib*/ false, /*isReference*/ false, file, skipTrivia(file.text, file.imports[i].pos), file.imports[i].end);
2076+
if (isJsFileUnderNodeModules) {
2077+
currentNodeModulesJsDepth--;
20202078
}
20212079
}
20222080
}

‎src/compiler/types.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -2533,6 +2533,7 @@ namespace ts {
25332533
declaration?: boolean;
25342534
declarationDir?: string;
25352535
/* @internal */ diagnostics?: boolean;
2536+
disableSizeLimit?: boolean;
25362537
emitBOM?: boolean;
25372538
emitDecoratorMetadata?: boolean;
25382539
experimentalDecorators?: boolean;
@@ -2548,6 +2549,7 @@ namespace ts {
25482549
/*@internal*/listFiles?: boolean;
25492550
locale?: string;
25502551
mapRoot?: string;
2552+
maxNodeModuleJsDepth?: number;
25512553
module?: ModuleKind;
25522554
moduleResolution?: ModuleResolutionKind;
25532555
newLine?: NewLineKind;
@@ -2586,7 +2588,6 @@ namespace ts {
25862588
/* @internal */ suppressOutputPathCheck?: boolean;
25872589
target?: ScriptTarget;
25882590
traceResolution?: boolean;
2589-
disableSizeLimit?: boolean;
25902591
types?: string[];
25912592
/** Paths used to used to compute primary types search locations */
25922593
typeRoots?: string[];

‎src/compiler/utilities.ts

+14-6
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ namespace ts {
3535
export interface EmitHost extends ScriptReferenceHost {
3636
getSourceFiles(): SourceFile[];
3737

38+
/* @internal */
39+
getFilesFromNodeModules(): Map<boolean>;
40+
3841
getCommonSourceDirectory(): string;
3942
getCanonicalFileName(fileName: string): string;
4043
getNewLine(): string;
@@ -2274,8 +2277,10 @@ namespace ts {
22742277
}
22752278
else {
22762279
const sourceFiles = targetSourceFile === undefined ? host.getSourceFiles() : [targetSourceFile];
2280+
const nodeModulesFiles = host.getFilesFromNodeModules();
22772281
for (const sourceFile of sourceFiles) {
2278-
if (!isDeclarationFile(sourceFile)) {
2282+
// Don't emit if source file is a declaration file, or was located under node_modules
2283+
if (!isDeclarationFile(sourceFile) && !lookUp(nodeModulesFiles, sourceFile.path)) {
22792284
onSingleFileEmit(host, sourceFile);
22802285
}
22812286
}
@@ -2307,11 +2312,14 @@ namespace ts {
23072312
}
23082313

23092314
function onBundledEmit(host: EmitHost) {
2310-
// Can emit only sources that are not declaration file and are either non module code or module with --module or --target es6 specified
2311-
const bundledSources = filter(host.getSourceFiles(), sourceFile =>
2312-
!isDeclarationFile(sourceFile) // Not a declaration file
2313-
&& (!isExternalModule(sourceFile) || !!getEmitModuleKind(options))); // and not a module, unless module emit enabled
2314-
2315+
// Can emit only sources that are not declaration file and are either non module code or module with
2316+
// --module or --target es6 specified. Files included by searching under node_modules are also not emitted.
2317+
const nodeModulesFiles = host.getFilesFromNodeModules();
2318+
const bundledSources = filter(host.getSourceFiles(),
2319+
sourceFile => !isDeclarationFile(sourceFile) &&
2320+
!lookUp(nodeModulesFiles, sourceFile.path) &&
2321+
(!isExternalModule(sourceFile) ||
2322+
!!getEmitModuleKind(options)));
23152323
if (bundledSources.length) {
23162324
const jsFilePath = options.outFile || options.out;
23172325
const emitFileNames: EmitFileNames = {

‎src/harness/runnerbase.ts

+3-9
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,12 @@ abstract class RunnerBase {
3131

3232
/** Replaces instances of full paths with fileNames only */
3333
static removeFullPaths(path: string) {
34-
let fixedPath = path;
35-
36-
// full paths either start with a drive letter or / for *nix, shouldn't have \ in the path at this point
37-
const fullPath = /(\w+:|\/)?([\w+\-\.]|\/)*\.tsx?/g;
38-
const fullPathList = fixedPath.match(fullPath);
39-
if (fullPathList) {
40-
fullPathList.forEach((match: string) => fixedPath = fixedPath.replace(match, Harness.Path.getFileName(match)));
41-
}
34+
// If its a full path (starts with "C:" or "/") replace with just the filename
35+
let fixedPath = /^(\w:|\/)/.test(path) ? Harness.Path.getFileName(path) : path;
4236

4337
// when running in the browser the 'full path' is the host name, shows up in error baselines
4438
const localHost = /http:\/localhost:\d+/g;
4539
fixedPath = fixedPath.replace(localHost, "");
4640
return fixedPath;
4741
}
48-
}
42+
}

‎tests/baselines/reference/declarationEmit_invalidReference2.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ tests/cases/compiler/declarationEmit_invalidReference2.ts(1,1): error TS6053: Fi
44
==== tests/cases/compiler/declarationEmit_invalidReference2.ts (1 errors) ====
55
/// <reference path="invalid.ts" />
66
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
7-
!!! error TS6053: File 'invalid.ts' not found.
7+
!!! error TS6053: File 'tests/cases/compiler/invalid.ts' not found.
88
var x = 0;

‎tests/baselines/reference/declarationFileOverwriteError.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
error TS5055: Cannot write file 'tests/cases/compiler/a.d.ts' because it would overwrite input file.
22

33

4-
!!! error TS5055: Cannot write file 'a.d.ts' because it would overwrite input file.
4+
!!! error TS5055: Cannot write file 'tests/cases/compiler/a.d.ts' because it would overwrite input file.
55
==== tests/cases/compiler/a.d.ts (0 errors) ====
66

77
declare class c {

‎tests/baselines/reference/declarationFileOverwriteErrorWithOut.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
error TS5055: Cannot write file 'tests/cases/compiler/out.d.ts' because it would overwrite input file.
22

33

4-
!!! error TS5055: Cannot write file 'out.d.ts' because it would overwrite input file.
4+
!!! error TS5055: Cannot write file 'tests/cases/compiler/out.d.ts' because it would overwrite input file.
55
==== tests/cases/compiler/out.d.ts (0 errors) ====
66

77
declare class c {

‎tests/baselines/reference/exportStarFromEmptyModule.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ tests/cases/compiler/exportStarFromEmptyModule_module4.ts(4,5): error TS2339: Pr
1414
==== tests/cases/compiler/exportStarFromEmptyModule_module3.ts (1 errors) ====
1515
export * from "./exportStarFromEmptyModule_module2";
1616
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
17-
!!! error TS2306: File 'exportStarFromEmptyModule_module2.ts' is not a module.
17+
!!! error TS2306: File 'tests/cases/compiler/exportStarFromEmptyModule_module2.ts' is not a module.
1818
export * from "./exportStarFromEmptyModule_module1";
1919

2020
export class A {

‎tests/baselines/reference/importNonExternalModule.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ tests/cases/conformance/externalModules/foo_1.ts(1,22): error TS2306: File 'test
44
==== tests/cases/conformance/externalModules/foo_1.ts (1 errors) ====
55
import foo = require("./foo_0");
66
~~~~~~~~~
7-
!!! error TS2306: File 'foo_0.ts' is not a module.
7+
!!! error TS2306: File 'tests/cases/conformance/externalModules/foo_0.ts' is not a module.
88
// Import should fail. foo_0 not an external module
99
if(foo.answer === 42){
1010

‎tests/baselines/reference/invalidTripleSlashReference.errors.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ tests/cases/compiler/invalidTripleSlashReference.ts(2,1): error TS6053: File 'te
55
==== tests/cases/compiler/invalidTripleSlashReference.ts (2 errors) ====
66
/// <reference path='filedoesnotexist.ts'/>
77
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
8-
!!! error TS6053: File 'filedoesnotexist.ts' not found.
8+
!!! error TS6053: File 'tests/cases/compiler/filedoesnotexist.ts' not found.
99
/// <reference path='otherdoesnotexist.d.ts'/>
1010
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
11-
!!! error TS6053: File 'otherdoesnotexist.d.ts' not found.
11+
!!! error TS6053: File 'tests/cases/compiler/otherdoesnotexist.d.ts' not found.
1212

1313
// this test doesn't actually give the errors you want due to the way the compiler reports errors
1414
var x = 1;

‎tests/baselines/reference/jsFileCompilationWithDeclarationEmitPathSameAsInput.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
error TS5055: Cannot write file 'tests/cases/compiler/a.d.ts' because it would overwrite input file.
22

33

4-
!!! error TS5055: Cannot write file 'a.d.ts' because it would overwrite input file.
4+
!!! error TS5055: Cannot write file 'tests/cases/compiler/a.d.ts' because it would overwrite input file.
55
==== tests/cases/compiler/a.ts (0 errors) ====
66
class c {
77
}

‎tests/baselines/reference/jsFileCompilationWithOutDeclarationFileNameSameAsInputJsFile.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
error TS5055: Cannot write file 'tests/cases/compiler/b.d.ts' because it would overwrite input file.
22

33

4-
!!! error TS5055: Cannot write file 'b.d.ts' because it would overwrite input file.
4+
!!! error TS5055: Cannot write file 'tests/cases/compiler/b.d.ts' because it would overwrite input file.
55
==== tests/cases/compiler/a.ts (0 errors) ====
66
class c {
77
}

‎tests/baselines/reference/library-reference-5.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
==== /node_modules/bar/index.d.ts (1 errors) ====
1919
/// <reference types="alpha" />
2020
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
21-
!!! message TS4090: Conflicting library definitions for 'alpha' found at 'index.d.ts' and 'index.d.ts'. Copy the correct file to the 'typings' folder to resolve this conflict.
21+
!!! message TS4090: Conflicting library definitions for 'alpha' found at '/node_modules/bar/node_modules/alpha/index.d.ts' and '/node_modules/foo/node_modules/alpha/index.d.ts'. Copy the correct file to the 'typings' folder to resolve this conflict.
2222
declare var bar: any;
2323

2424
==== /node_modules/bar/node_modules/alpha/index.d.ts (0 errors) ====

‎tests/baselines/reference/parserRealSource1.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ tests/cases/conformance/parser/ecmascript5/parserRealSource1.ts(4,1): error TS60
77

88
///<reference path='typescript.ts' />
99
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
10-
!!! error TS6053: File 'typescript.ts' not found.
10+
!!! error TS6053: File 'tests/cases/conformance/parser/ecmascript5/typescript.ts' not found.
1111

1212
module TypeScript {
1313
export module CompilerDiagnostics {

‎tests/baselines/reference/parserRealSource10.errors.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ tests/cases/conformance/parser/ecmascript5/parserRealSource10.ts(449,40): error
348348

349349
///<reference path='typescript.ts' />
350350
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
351-
!!! error TS6053: File 'typescript.ts' not found.
351+
!!! error TS6053: File 'tests/cases/conformance/parser/ecmascript5/typescript.ts' not found.
352352

353353
module TypeScript {
354354
export enum TokenID {

0 commit comments

Comments
 (0)