From 684425dcfbfadb6fa687911c55558e047cd0c9e1 Mon Sep 17 00:00:00 2001 From: Jaap Frolich Date: Thu, 11 Apr 2024 11:05:12 +0200 Subject: [PATCH 1/6] :sparkles: - Support rewatch for incremental compilation --- server/src/incrementalCompilation.ts | 170 ++++++++++++++++++--------- 1 file changed, 112 insertions(+), 58 deletions(-) diff --git a/server/src/incrementalCompilation.ts b/server/src/incrementalCompilation.ts index 3e2b57a97..7578440d1 100644 --- a/server/src/incrementalCompilation.ts +++ b/server/src/incrementalCompilation.ts @@ -20,6 +20,11 @@ function debug() { const INCREMENTAL_FOLDER_NAME = "___incremental"; const INCREMENTAL_FILE_FOLDER_LOCATION = `lib/bs/${INCREMENTAL_FOLDER_NAME}`; +type RewatchCompilerArgs = { + compiler_args: Array; + parser_args: Array; +}; + type IncrementallyCompiledFileInfo = { file: { /** File type. */ @@ -44,6 +49,10 @@ type IncrementallyCompiledFileInfo = { /** The raw, extracted needed info from build.ninja. Needs processing. */ rawExtracted: Array; } | null; + /** Cache for rewatch compiler args. */ + buildRewatch: { + compilerArgs: RewatchCompilerArgs; + } | null; /** Info of the currently active incremental compilation. `null` if no incremental compilation is active. */ compilation: { /** The timeout of the currently active compilation for this incremental file. */ @@ -176,18 +185,28 @@ export function cleanUpIncrementalFiles( } function getBscArgs( entry: IncrementallyCompiledFileInfo -): Promise | null> { +): Promise | RewatchCompilerArgs | null> { const buildNinjaPath = path.resolve( entry.project.rootPath, "lib/bs/build.ninja" ); + const rewatchLockfile = path.resolve( + entry.project.rootPath, + "lib/rewatch.lock" + ); + let buildSystem: "bsb" | "rewatch" | null = null; + let stat: fs.Stats | null = null; try { stat = fs.statSync(buildNinjaPath); - } catch { - if (debug()) { - console.log("Did not find build.ninja, cannot proceed.."); - } + buildSystem = "bsb"; + } catch {} + try { + stat = fs.statSync(rewatchLockfile); + buildSystem = "rewatch"; + } catch {} + if (buildSystem == null) { + console.log("Did not find build.ninja or rewatch.lock, cannot proceed.."); return Promise.resolve(null); } const cacheEntry = entry.buildNinja; @@ -199,8 +218,8 @@ function getBscArgs( return Promise.resolve(cacheEntry.rawExtracted); } return new Promise((resolve, _reject) => { - function resolveResult(result: Array) { - if (stat != null) { + function resolveResult(result: Array | RewatchCompilerArgs) { + if (stat != null && Array.isArray(result)) { entry.buildNinja = { fileMtime: stat.mtimeMs, rawExtracted: result, @@ -208,64 +227,82 @@ function getBscArgs( } resolve(result); } - const fileStream = fs.createReadStream(buildNinjaPath); - const rl = readline.createInterface({ - input: fileStream, - crlfDelay: Infinity, - }); - let captureNextLine = false; - let done = false; - let stopped = false; - const captured: Array = []; - rl.on("line", (line) => { - if (stopped) { - return; - } - if (captureNextLine) { - captured.push(line); - captureNextLine = false; - } - if (done) { - fileStream.destroy(); - rl.close(); + + if (buildSystem === "bsb") { + const fileStream = fs.createReadStream(buildNinjaPath); + const rl = readline.createInterface({ + input: fileStream, + crlfDelay: Infinity, + }); + let captureNextLine = false; + let done = false; + let stopped = false; + const captured: Array = []; + rl.on("line", (line) => { + if (stopped) { + return; + } + if (captureNextLine) { + captured.push(line); + captureNextLine = false; + } + if (done) { + fileStream.destroy(); + rl.close(); + resolveResult(captured); + stopped = true; + return; + } + if (line.startsWith("rule astj")) { + captureNextLine = true; + } + if (line.startsWith("rule mij")) { + captureNextLine = true; + done = true; + } + }); + rl.on("close", () => { resolveResult(captured); - stopped = true; - return; - } - if (line.startsWith("rule astj")) { - captureNextLine = true; - } - if (line.startsWith("rule mij")) { - captureNextLine = true; - done = true; + }); + } else if (buildSystem === "rewatch") { + try { + let rewatchPath = path.resolve( + entry.project.rootPath, + "../rewatch/target/debug/rewatch" + ); + const compilerArgs = JSON.parse( + cp + .execFileSync(rewatchPath, [ + "--rescript-version", + entry.project.rescriptVersion, + "--get-compiler-args", + entry.file.sourceFilePath, + ]) + .toString() + .trim() + ) as RewatchCompilerArgs; + resolveResult(compilerArgs); + } catch (e) { + console.error(e); } - }); - rl.on("close", () => { - resolveResult(captured); - }); + } }); } -function argsFromCommandString(cmdString: string): Array> { - const s = cmdString - .trim() - .split("command = ")[1] - .split(" ") - .map((v) => v.trim()) - .filter((v) => v !== ""); - const args: Array> = []; - for (let i = 0; i <= s.length - 1; i++) { - const item = s[i]; +function argCouples(argList: string[]): string[][] { + let args: string[][] = []; + for (let i = 0; i <= argList.length - 1; i++) { + const item = argList[i]; const nextIndex = i + 1; - const nextItem = s[nextIndex] ?? ""; + const nextItem = argList[nextIndex] ?? ""; if (item.startsWith("-") && nextItem.startsWith("-")) { // Single entry arg args.push([item]); } else if (item.startsWith("-") && nextItem.startsWith("'")) { // Quoted arg, take until ending ' const arg = [nextItem.slice(1)]; - for (let x = nextIndex + 1; x <= s.length - 1; x++) { - let subItem = s[x]; + for (let x = nextIndex + 1; x <= argList.length - 1; x++) { + let subItem = argList[x]; let break_ = false; if (subItem.endsWith("'")) { subItem = subItem.slice(0, subItem.length - 1); @@ -284,6 +321,17 @@ function argsFromCommandString(cmdString: string): Array> { } return args; } + +function argsFromCommandString(cmdString: string): Array> { + const argList = cmdString + .trim() + .split("command = ")[1] + .split(" ") + .map((v) => v.trim()) + .filter((v) => v !== ""); + + return argCouples(argList); +} function removeAnsiCodes(s: string): string { const ansiEscape = /\x1B[@-_][0-?]*[ -/]*[@-~]/g; return s.replace(ansiEscape, ""); @@ -368,6 +416,7 @@ function triggerIncrementalCompilationOfFile( incrementalFolderPath, rescriptVersion, }, + buildRewatch: null, buildNinja: null, compilation: null, killCompilationListeners: [], @@ -419,11 +468,16 @@ function verifyTriggerToken(filePath: string, triggerToken: number): boolean { async function figureOutBscArgs(entry: IncrementallyCompiledFileInfo) { const res = await getBscArgs(entry); if (res == null) return null; - const [astBuildCommand, fullBuildCommand] = res; - - const astArgs = argsFromCommandString(astBuildCommand); - const buildArgs = argsFromCommandString(fullBuildCommand); - + let astArgs: Array> = []; + let buildArgs: Array> = []; + if (Array.isArray(res)) { + const [astBuildCommand, fullBuildCommand] = res; + astArgs = argsFromCommandString(astBuildCommand); + buildArgs = argsFromCommandString(fullBuildCommand); + } else { + astArgs = argCouples(res.parser_args); + buildArgs = argCouples(res.compiler_args); + } let callArgs: Array = []; if (config.extensionConfiguration.incrementalTypechecking?.acrossFiles) { From 013c295f31f3e437f141d0724a2a9087c953061d Mon Sep 17 00:00:00 2001 From: Jaap Frolich Date: Thu, 11 Apr 2024 11:09:18 +0200 Subject: [PATCH 2/6] fix path --- server/src/incrementalCompilation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/incrementalCompilation.ts b/server/src/incrementalCompilation.ts index 7578440d1..52bf3b35c 100644 --- a/server/src/incrementalCompilation.ts +++ b/server/src/incrementalCompilation.ts @@ -268,7 +268,7 @@ function getBscArgs( try { let rewatchPath = path.resolve( entry.project.rootPath, - "../rewatch/target/debug/rewatch" + "node_modules/@rolandpeelen/rewatch/rewatch" ); const compilerArgs = JSON.parse( cp From 50611a7576624d0f07f10ce85270bb1aee67344f Mon Sep 17 00:00:00 2001 From: Jaap Frolich Date: Thu, 11 Apr 2024 12:30:59 +0200 Subject: [PATCH 3/6] make it actually work :) --- server/src/incrementalCompilation.ts | 35 +++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/server/src/incrementalCompilation.ts b/server/src/incrementalCompilation.ts index 52bf3b35c..e3c687649 100644 --- a/server/src/incrementalCompilation.ts +++ b/server/src/incrementalCompilation.ts @@ -66,6 +66,8 @@ type IncrementallyCompiledFileInfo = { project: { /** The root path of the project. */ rootPath: string; + /** The root path of the workspace (if a monorepo) */ + workspaceRootPath: string; /** Computed location of bsc. */ bscBinaryLocation: string; /** The arguments needed for bsc, derived from the project configuration/build.ninja. */ @@ -191,7 +193,7 @@ function getBscArgs( "lib/bs/build.ninja" ); const rewatchLockfile = path.resolve( - entry.project.rootPath, + entry.project.workspaceRootPath, "lib/rewatch.lock" ); let buildSystem: "bsb" | "rewatch" | null = null; @@ -211,6 +213,7 @@ function getBscArgs( } const cacheEntry = entry.buildNinja; if ( + buildSystem === "bsb" && cacheEntry != null && stat != null && cacheEntry.fileMtime >= stat.mtimeMs @@ -267,15 +270,15 @@ function getBscArgs( } else if (buildSystem === "rewatch") { try { let rewatchPath = path.resolve( - entry.project.rootPath, - "node_modules/@rolandpeelen/rewatch/rewatch" + entry.project.workspaceRootPath, + "../rewatch/target/debug/rewatch" ); const compilerArgs = JSON.parse( cp .execFileSync(rewatchPath, [ "--rescript-version", entry.project.rescriptVersion, - "--get-compiler-args", + "--compiler-args", entry.file.sourceFilePath, ]) .toString() @@ -346,6 +349,9 @@ function triggerIncrementalCompilationOfFile( if (incrementalFileCacheEntry == null) { // New file const projectRootPath = utils.findProjectRootOfFile(filePath); + const workspaceRootPath = projectRootPath + ? utils.findProjectRootOfFile(projectRootPath) + : null; if (projectRootPath == null) { if (debug()) console.log("Did not find project root path for " + filePath); @@ -410,6 +416,7 @@ function triggerIncrementalCompilationOfFile( incrementalFilePath: path.join(incrementalFolderPath, moduleName + ext), }, project: { + workspaceRootPath: workspaceRootPath ?? projectRootPath, rootPath: projectRootPath, callArgs: Promise.resolve([]), bscBinaryLocation, @@ -470,6 +477,7 @@ async function figureOutBscArgs(entry: IncrementallyCompiledFileInfo) { if (res == null) return null; let astArgs: Array> = []; let buildArgs: Array> = []; + let isBsb = Array.isArray(res); if (Array.isArray(res)) { const [astBuildCommand, fullBuildCommand] = res; astArgs = argsFromCommandString(astBuildCommand); @@ -489,10 +497,21 @@ async function figureOutBscArgs(entry: IncrementallyCompiledFileInfo) { buildArgs.forEach(([key, value]: Array) => { if (key === "-I") { - callArgs.push( - "-I", - path.resolve(entry.project.rootPath, "lib/bs", value) - ); + if (isBsb) { + callArgs.push( + "-I", + path.resolve(entry.project.rootPath, "lib/bs", value) + ); + } else { + if (value === ".") { + callArgs.push( + "-I", + path.resolve(entry.project.rootPath, "lib/ocaml") + ); + } else { + callArgs.push("-I", value); + } + } } else if (key === "-bs-v") { callArgs.push("-bs-v", Date.now().toString()); } else if (key === "-bs-package-output") { From 68d3ba845a8e29dd1dc52c5b1868187073745d3d Mon Sep 17 00:00:00 2001 From: Jaap Frolich Date: Thu, 11 Apr 2024 12:43:34 +0200 Subject: [PATCH 4/6] add basic cache entry --- server/src/incrementalCompilation.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/server/src/incrementalCompilation.ts b/server/src/incrementalCompilation.ts index e3c687649..c740266dd 100644 --- a/server/src/incrementalCompilation.ts +++ b/server/src/incrementalCompilation.ts @@ -51,6 +51,7 @@ type IncrementallyCompiledFileInfo = { } | null; /** Cache for rewatch compiler args. */ buildRewatch: { + lastFile: string; compilerArgs: RewatchCompilerArgs; } | null; /** Info of the currently active incremental compilation. `null` if no incremental compilation is active. */ @@ -211,14 +212,23 @@ function getBscArgs( console.log("Did not find build.ninja or rewatch.lock, cannot proceed.."); return Promise.resolve(null); } - const cacheEntry = entry.buildNinja; + const bsbCacheEntry = entry.buildNinja; + const rewatchCacheEntry = entry.buildRewatch; + if ( buildSystem === "bsb" && - cacheEntry != null && + bsbCacheEntry != null && stat != null && - cacheEntry.fileMtime >= stat.mtimeMs + bsbCacheEntry.fileMtime >= stat.mtimeMs ) { - return Promise.resolve(cacheEntry.rawExtracted); + return Promise.resolve(bsbCacheEntry.rawExtracted); + } + if ( + buildSystem === "rewatch" && + rewatchCacheEntry != null && + rewatchCacheEntry.lastFile === entry.file.sourceFilePath + ) { + return Promise.resolve(rewatchCacheEntry.compilerArgs); } return new Promise((resolve, _reject) => { function resolveResult(result: Array | RewatchCompilerArgs) { @@ -227,6 +237,11 @@ function getBscArgs( fileMtime: stat.mtimeMs, rawExtracted: result, }; + } else if (!Array.isArray(result)) { + entry.buildRewatch = { + lastFile: entry.file.sourceFilePath, + compilerArgs: result, + }; } resolve(result); } From 2859dfbd6ec5508f6c941207b83edc8e10c6d29d Mon Sep 17 00:00:00 2001 From: Jaap Frolich Date: Thu, 11 Apr 2024 12:51:10 +0200 Subject: [PATCH 5/6] fix path --- server/src/incrementalCompilation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/incrementalCompilation.ts b/server/src/incrementalCompilation.ts index c740266dd..285b6a5b5 100644 --- a/server/src/incrementalCompilation.ts +++ b/server/src/incrementalCompilation.ts @@ -286,7 +286,7 @@ function getBscArgs( try { let rewatchPath = path.resolve( entry.project.workspaceRootPath, - "../rewatch/target/debug/rewatch" + "node_modules/@rolandpeelen/rewatch/rewatch" ); const compilerArgs = JSON.parse( cp From d6f6e10d60c9112c170249330779124967001a6e Mon Sep 17 00:00:00 2001 From: Jaap Frolich Date: Fri, 12 Apr 2024 10:47:32 +0200 Subject: [PATCH 6/6] add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fd7a16b0..b2282a94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ - Make sure doc strings are always on top in hovers. https://github.com/rescript-lang/rescript-vscode/pull/956 +#### :rocket: New Feature + +- Add support for the rewatch build system for incremental compilation. https://github.com/rescript-lang/rescript-vscode/pull/965 + ## 1.50.0 #### :rocket: New Feature