[compiler][be] Test runner (snap) now uses tsup bundled plugin

Currently, `babel-plugin-react-compiler` is bundled with (almost) all external dependencies. This is because babel traversal and ast logic is not forward-compatible. Since `babel-plugin-react-compiler` needs to be compatible with babel pipelines across a wide semvar range, we (1) set this package's babel dependency to an early version and (2) inline babel libraries into our bundle.

A few other packages in `react/compiler` depend on the compiler. This PR moves `snap`, our test fixture compiler and evaluator, to use the bundled version of `babel-plugin-react-compiler`. This decouples the babel version used by `snap` with the version used by `babel-plugin-react-compiler`, which means that `snap` now can test features from newer babel versions (see #32742).

Author: mofeiZ

compiler/packages/babel-plugin-react-compiler/package.json

@@ -12,7 +12,7 @@
     "build": "rimraf dist && tsup",
     "test": "./scripts/link-react-compiler-runtime.sh && yarn snap:ci",
     "jest": "yarn build && ts-node node_modules/.bin/jest",
-    "snap": "node ../snap/dist/main.js",
+    "snap": "yarn workspace snap run snap",
     "snap:build": "yarn workspace snap run build",
     "snap:ci": "yarn snap:build && yarn snap",
     "ts:analyze-trace": "scripts/ts-analyze-trace.sh",

compiler/packages/babel-plugin-react-compiler/src/HIR/index.ts

@@ -32,5 +32,5 @@ export {
 } from './HIRBuilder';
 export {mergeConsecutiveBlocks} from './MergeConsecutiveBlocks';
 export {mergeOverlappingReactiveScopesHIR} from './MergeOverlappingReactiveScopesHIR';
-export {printFunction, printHIR} from './PrintHIR';
+export {printFunction, printHIR, printFunctionWithOutlined} from './PrintHIR';
 export {pruneUnusedLabelsHIR} from './PruneUnusedLabelsHIR';

compiler/packages/babel-plugin-react-compiler/src/ReactiveScopes/index.ts

@@ -14,7 +14,10 @@ export {extractScopeDeclarationsFromDestructuring} from './ExtractScopeDeclarati
 export {inferReactiveScopeVariables} from './InferReactiveScopeVariables';
 export {memoizeFbtAndMacroOperandsInSameScope} from './MemoizeFbtAndMacroOperandsInSameScope';
 export {mergeReactiveScopesThatInvalidateTogether} from './MergeReactiveScopesThatInvalidateTogether';
-export {printReactiveFunction} from './PrintReactiveFunction';
+export {
+  printReactiveFunction,
+  printReactiveFunctionWithOutlined,
+} from './PrintReactiveFunction';
 export {promoteUsedTemporaries} from './PromoteUsedTemporaries';
 export {propagateEarlyReturns} from './PropagateEarlyReturns';
 export {pruneAllReactiveScopes} from './PruneAllReactiveScopes';

compiler/packages/babel-plugin-react-compiler/src/index.ts

@@ -32,13 +32,17 @@ export {
   ValueKind,
   parseConfigPragmaForTests,
   printHIR,
+  printFunctionWithOutlined,
   validateEnvironmentConfig,
   type EnvironmentConfig,
   type ExternalFunction,
   type Hook,
   type SourceLocation,
 } from './HIR';
-export {printReactiveFunction} from './ReactiveScopes';
+export {
+  printReactiveFunction,
+  printReactiveFunctionWithOutlined,
+} from './ReactiveScopes';
 declare global {
   let __DEV__: boolean | null | undefined;
 }

compiler/packages/babel-plugin-react-compiler/tsconfig.json

@@ -4,7 +4,7 @@
     "module": "ES2015",
     "moduleResolution": "Bundler",
     "rootDir": "src",
-    "outDir": "dist",
+    "noEmit": true,
     "jsx": "react-jsxdev",
     // weaken strictness from preset
     "importsNotUsedAsValues": "remove",

compiler/packages/snap/package.json

@@ -11,6 +11,7 @@
   "scripts": {
     "postinstall": "./scripts/link-react-compiler-runtime.sh && perl -p -i -e 's/react\\.element/react.transitional.element/' ../../node_modules/fbt/lib/FbtReactUtil.js && perl -p -i -e 's/didWarnAboutUsingAct = false;/didWarnAboutUsingAct = true;/' ../../node_modules/react-dom/cjs/react-dom-test-utils.development.js",
     "build": "rimraf dist && concurrently -n snap,runtime \"tsc --build\" \"yarn --silent workspace react-compiler-runtime build\"",
+    "snap": "node dist/main.js",
     "test": "echo 'no tests'",
     "prettier": "prettier --write 'src/**/*.ts'"
   },
@@ -40,8 +41,7 @@
   },
   "devDependencies": {
     "@babel/core": "^7.19.1",
-    "@babel/parser": "^7.19.1",
-    "@babel/plugin-syntax-typescript": "^7.18.6",
+    "@babel/parser": "^7.20.15",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
     "@babel/preset-react": "^7.18.6",
     "@babel/traverse": "^7.19.1",

compiler/packages/snap/src/constants.ts

@@ -9,40 +9,22 @@ import path from 'path';
 
 // We assume this is run from `babel-plugin-react-compiler`
 export const PROJECT_ROOT = path.normalize(
-  path.join(process.cwd(), '..', '..'),
+  path.join(process.cwd(), '..', 'babel-plugin-react-compiler'),
 );
-export const COMPILER_PATH = path.join(
-  process.cwd(),
-  'dist',
-  'Babel',
-  'BabelPlugin.js',
-);
-export const COMPILER_INDEX_PATH = path.join(process.cwd(), 'dist', 'index');
-export const PRINT_HIR_PATH = path.join(
-  process.cwd(),
-  'dist',
-  'HIR',
-  'PrintHIR.js',
-);
-export const PRINT_REACTIVE_IR_PATH = path.join(
-  process.cwd(),
-  'dist',
-  'ReactiveScopes',
-  'PrintReactiveFunction.js',
-);
-export const PARSE_CONFIG_PRAGMA_PATH = path.join(
-  process.cwd(),
-  'dist',
-  'HIR',
-  'Environment.js',
+
+export const PROJECT_SRC = path.normalize(
+  path.join(PROJECT_ROOT, 'dist', 'index.js'),
 );
+export const PRINT_HIR_IMPORT = 'printFunctionWithOutlined';
+export const PRINT_REACTIVE_IR_IMPORT = 'printReactiveFunction';
+export const PARSE_CONFIG_PRAGMA_IMPORT = 'parseConfigPragmaForTests';
 export const FIXTURES_PATH = path.join(
-  process.cwd(),
+  PROJECT_ROOT,
   'src',
   '__tests__',
   'fixtures',
   'compiler',
 );
 export const SNAPSHOT_EXTENSION = '.expect.md';
 export const FILTER_FILENAME = 'testfilter.txt';
-export const FILTER_PATH = path.join(process.cwd(), FILTER_FILENAME);
+export const FILTER_PATH = path.join(PROJECT_ROOT, FILTER_FILENAME);

compiler/packages/snap/src/runner-watch.ts

@@ -8,15 +8,16 @@
 import watcher from '@parcel/watcher';
 import path from 'path';
 import ts from 'typescript';
-import {FILTER_FILENAME, FIXTURES_PATH} from './constants';
+import {FILTER_FILENAME, FIXTURES_PATH, PROJECT_ROOT} from './constants';
 import {TestFilter, readTestFilter} from './fixture-utils';
+import {execSync} from 'child_process';
 
 export function watchSrc(
   onStart: () => void,
   onComplete: (isSuccess: boolean) => void,
 ): ts.WatchOfConfigFile<ts.SemanticDiagnosticsBuilderProgram> {
   const configPath = ts.findConfigFile(
-    /*searchPath*/ './',
+    /*searchPath*/ PROJECT_ROOT,
     ts.sys.fileExists,
     'tsconfig.json',
   );
@@ -26,10 +27,7 @@ export function watchSrc(
   const createProgram = ts.createSemanticDiagnosticsBuilderProgram;
   const host = ts.createWatchCompilerHost(
     configPath,
-    ts.convertCompilerOptionsFromJson(
-      {module: 'commonjs', outDir: 'dist', sourceMap: true},
-      '.',
-    ).options,
+    undefined,
     ts.sys,
     createProgram,
     () => {}, // we manually report errors in afterProgramCreate
@@ -41,9 +39,11 @@ export function watchSrc(
     onStart();
     return origCreateProgram(rootNames, options, host, oldProgram);
   };
-  const origPostProgramCreate = host.afterProgramCreate;
   host.afterProgramCreate = program => {
-    origPostProgramCreate!(program);
+    /**
+     * Avoid calling original postProgramCreate because it always emits tsc
+     * compilation output
+     */
 
     // syntactic diagnostics refer to javascript syntax
     const errors = program
@@ -172,13 +172,23 @@ function subscribeTsc(
       // Notify the user when compilation starts but don't clear the screen yet
       console.log('\nCompiling...');
     },
-    isSuccess => {
+    isTypecheckSuccess => {
+      let isCompilerBuildValid = false;
+      if (isTypecheckSuccess) {
+        try {
+          execSync('yarn build', {cwd: PROJECT_ROOT});
+          console.log('Built compiler successfully with tsup');
+          isCompilerBuildValid = true;
+        } catch (e) {
+          console.warn('Failed to build compiler with tsup:', e);
+        }
+      }
       // Bump the compiler version after a build finishes
       // and re-run tests
-      if (isSuccess) {
+      if (isCompilerBuildValid) {
         state.compilerVersion++;
       }
-      state.isCompilerBuildValid = isSuccess;
+      state.isCompilerBuildValid = isCompilerBuildValid;
       state.mode.action = RunnerAction.Test;
       onChange(state);
     },

compiler/packages/snap/src/runner-worker.ts

@@ -12,16 +12,19 @@ import type {printFunctionWithOutlined as PrintFunctionWithOutlined} from 'babel
 import type {printReactiveFunctionWithOutlined as PrintReactiveFunctionWithOutlined} from 'babel-plugin-react-compiler/src/ReactiveScopes/PrintReactiveFunction';
 import {TransformResult, transformFixtureInput} from './compiler';
 import {
-  COMPILER_PATH,
-  COMPILER_INDEX_PATH,
-  PARSE_CONFIG_PRAGMA_PATH,
-  PRINT_HIR_PATH,
-  PRINT_REACTIVE_IR_PATH,
+  PARSE_CONFIG_PRAGMA_IMPORT,
+  PRINT_HIR_IMPORT,
+  PRINT_REACTIVE_IR_IMPORT,
+  PROJECT_SRC,
 } from './constants';
 import {TestFixture, getBasename, isExpectError} from './fixture-utils';
 import {TestResult, writeOutputToString} from './reporter';
 import {runSprout} from './sprout';
-import {CompilerPipelineValue} from 'babel-plugin-react-compiler/src';
+import type {
+  CompilerPipelineValue,
+  Effect,
+  ValueKind,
+} from 'babel-plugin-react-compiler/src';
 import chalk from 'chalk';
 
 const originalConsoleError = console.error;
@@ -61,22 +64,29 @@ async function compile(
   let compileResult: TransformResult | null = null;
   let error: string | null = null;
   try {
+    const importedCompilerPlugin = require(PROJECT_SRC) as Record<
+      string,
+      unknown
+    >;
+
     // NOTE: we intentionally require lazily here so that we can clear the require cache
     // and load fresh versions of the compiler when `compilerVersion` changes.
-    const {default: BabelPluginReactCompiler} = require(COMPILER_PATH) as {
-      default: PluginObj;
-    };
-    const {Effect: EffectEnum, ValueKind: ValueKindEnum} = require(
-      COMPILER_INDEX_PATH,
-    );
-    const {printFunctionWithOutlined} = require(PRINT_HIR_PATH) as {
-      printFunctionWithOutlined: typeof PrintFunctionWithOutlined;
-    };
-    const {printReactiveFunctionWithOutlined} = require(
-      PRINT_REACTIVE_IR_PATH,
-    ) as {
-      printReactiveFunctionWithOutlined: typeof PrintReactiveFunctionWithOutlined;
-    };
+    const BabelPluginReactCompiler = importedCompilerPlugin[
+      'default'
+    ] as PluginObj;
+    const EffectEnum = importedCompilerPlugin['Effect'] as typeof Effect;
+    const ValueKindEnum = importedCompilerPlugin[
+      'ValueKind'
+    ] as typeof ValueKind;
+    const printFunctionWithOutlined = importedCompilerPlugin[
+      PRINT_HIR_IMPORT
+    ] as typeof PrintFunctionWithOutlined;
+    const printReactiveFunctionWithOutlined = importedCompilerPlugin[
+      PRINT_REACTIVE_IR_IMPORT
+    ] as typeof PrintReactiveFunctionWithOutlined;
+    const parseConfigPragmaForTests = importedCompilerPlugin[
+      PARSE_CONFIG_PRAGMA_IMPORT
+    ] as typeof ParseConfigPragma;
 
     let lastLogged: string | null = null;
     const debugIRLogger = shouldLog
@@ -106,9 +116,6 @@ async function compile(
           }
         }
       : () => {};
-    const {parseConfigPragmaForTests} = require(PARSE_CONFIG_PRAGMA_PATH) as {
-      parseConfigPragmaForTests: typeof ParseConfigPragma;
-    };
 
     // only try logging if we filtered out all but one fixture,
     // since console log order is non-deterministic

compiler/packages/snap/src/runner.ts

@@ -12,7 +12,7 @@ import * as readline from 'readline';
 import ts from 'typescript';
 import yargs from 'yargs';
 import {hideBin} from 'yargs/helpers';
-import {FILTER_PATH} from './constants';
+import {FILTER_PATH, PROJECT_ROOT} from './constants';
 import {TestFilter, getFixtures, readTestFilter} from './fixture-utils';
 import {TestResult, TestResults, report, update} from './reporter';
 import {
@@ -22,6 +22,7 @@ import {
   watchSrc,
 } from './runner-watch';
 import * as runnerWorker from './runner-worker';
+import {execSync} from 'child_process';
 
 const WORKER_PATH = require.resolve('./runner-worker.js');
 const NUM_WORKERS = cpus().length - 1;
@@ -205,23 +206,30 @@ export async function main(opts: RunnerOptions): Promise<void> {
     const tsWatch: ts.WatchOfConfigFile<ts.SemanticDiagnosticsBuilderProgram> =
       watchSrc(
         () => {},
-        async (compileSuccess: boolean) => {
-          let isSuccess = compileSuccess;
-          if (compileSuccess) {
-            const testFilter = opts.filter ? await readTestFilter() : null;
-            const results = await runFixtures(worker, testFilter, 0);
-            if (opts.update) {
-              update(results);
-            } else {
-              const testSuccess = report(results);
-              isSuccess &&= testSuccess;
-            }
-          } else {
+        async (isTypecheckSuccess: boolean) => {
+          let isSuccess = false;
+          if (!isTypecheckSuccess) {
             console.error(
-              'Found errors in Forget source code, skipping test fixtures.',
+              'Found typescript errors in Forget source code, skipping test fixtures.',
             );
+          } else {
+            try {
+              execSync('yarn build', {cwd: PROJECT_ROOT});
+              console.log('Built compiler successfully with tsup');
+              const testFilter = opts.filter ? await readTestFilter() : null;
+              const results = await runFixtures(worker, testFilter, 0);
+              if (opts.update) {
+                update(results);
+                isSuccess = true;
+              } else {
+                const testSuccess = report(results);
+                isSuccess &&= testSuccess;
+              }
+            } catch (e) {
+              console.warn('Failed to build compiler with tsup:', e);
+            }
           }
-          tsWatch.close();
+          tsWatch?.close();
           await worker.end();
           process.exit(isSuccess ? 0 : 1);
         },

compiler/yarn.lock

@@ -657,7 +657,7 @@
     js-tokens "^4.0.0"
     picocolors "^1.0.0"
 
-"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10", "@babel/parser@^7.19.1", "@babel/parser@^7.2.0":
+"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10", "@babel/parser@^7.2.0":
   version "7.19.1"
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.1.tgz#6f6d6c2e621aad19a92544cc217ed13f1aac5b4c"
   integrity sha512-h7RCSorm1DdTVGJf3P2Mhj3kdnkmF/EiysUkzS2TdgAYqyjFdMQJbVuXOBej2SBJaXan/lIVtT6KkGbyyq753A==
@@ -667,6 +667,13 @@
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.3.tgz#8dd36d17c53ff347f9e55c328710321b49479a9a"
   integrity sha512-pJ9xOlNWHiy9+FuFP09DEAFbAn4JskgRsVcc169w2xRBC3FRGuQEwjeIMMND9L2zc0iEhO/tGv4Zq+km+hxNpQ==
 
+"@babel/parser@^7.20.15":
+  version "7.27.0"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.27.0.tgz#3d7d6ee268e41d2600091cbd4e145ffee85a44ec"
+  integrity sha512-iaepho73/2Pz7w2eMS0Q5f83+0RKI7i4xmiYeBmDzfRVbQtTOG7Ts0S4HzJVsTMGI9keU8rNfuZr8DKfSt7Yyg==
+  dependencies:
+    "@babel/types" "^7.27.0"
+
 "@babel/parser@^7.20.7":
   version "7.21.2"
   resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.21.2.tgz#dacafadfc6d7654c3051a66d6fe55b6cb2f2a0b3"
@@ -1696,7 +1703,7 @@
     debug "^4.3.1"
     globals "^11.1.0"
 
-"@babel/types@7.26.3", "@babel/types@^7.0.0", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.2.0", "@babel/types@^7.2.2", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.21.2", "@babel/types@^7.21.4", "@babel/types@^7.21.5", "@babel/types@^7.22.0", "@babel/types@^7.22.3", "@babel/types@^7.22.4", "@babel/types@^7.22.5", "@babel/types@^7.24.7", "@babel/types@^7.25.0", "@babel/types@^7.25.6", "@babel/types@^7.25.9", "@babel/types@^7.26.0", "@babel/types@^7.26.10", "@babel/types@^7.26.3", "@babel/types@^7.26.9", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.4":
+"@babel/types@7.26.3", "@babel/types@^7.0.0", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.2.0", "@babel/types@^7.2.2", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.21.2", "@babel/types@^7.21.4", "@babel/types@^7.21.5", "@babel/types@^7.22.0", "@babel/types@^7.22.3", "@babel/types@^7.22.4", "@babel/types@^7.22.5", "@babel/types@^7.24.7", "@babel/types@^7.25.0", "@babel/types@^7.25.6", "@babel/types@^7.25.9", "@babel/types@^7.26.0", "@babel/types@^7.26.10", "@babel/types@^7.26.3", "@babel/types@^7.26.9", "@babel/types@^7.27.0", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.4":
   version "7.26.3"
   resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.26.3.tgz#37e79830f04c2b5687acc77db97fbc75fb81f3c0"
   integrity sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==