test(e2e): Use pnpm for e2e tests (#7930)

Author: lforst

.github/workflows/build.yml

@@ -768,6 +768,9 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ env.HEAD_COMMIT }}
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 7
       - name: Set up Node
         uses: actions/setup-node@v3
         with:

.github/workflows/canary.yml

@@ -26,6 +26,9 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ env.HEAD_COMMIT }}
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 7
       - name: Set up Node
         uses: actions/setup-node@v3
         with:

packages/e2e-tests/README.md

@@ -54,11 +54,11 @@ To get you started with the recipe, you can copy the following into `test-recipe
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "My New Test Application",
-  "buildCommand": "yarn install",
+  "buildCommand": "pnpm install",
   "tests": [
     {
       "testName": "My new test",
-      "testCommand": "yarn test",
+      "testCommand": "pnpm test",
       "timeoutSeconds": 60
     }
   ]

packages/e2e-tests/lib/buildApp.ts

@@ -1,7 +1,6 @@
 /* eslint-disable no-console */
 
 import * as fs from 'fs-extra';
-import * as os from 'os';
 import * as path from 'path';
 
 import { DEFAULT_BUILD_TIMEOUT_SECONDS } from './constants';
@@ -29,13 +28,9 @@ export async function buildApp(appDir: string, recipeInstance: RecipeInstance, e
   if (recipe.buildCommand) {
     console.log(`Running build command for test application "${label}"`);
 
-    fs.mkdirSync(path.join(os.tmpdir(), 'e2e-test-yarn-caches'), { recursive: true });
-    const tempYarnCache = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-test-yarn-caches', 'cache-'));
-
     const env = {
       ...process.env,
       ...envVars,
-      YARN_CACHE_FOLDER: tempYarnCache, // Use a separate yarn cache for each build commmand because multiple yarn commands running at the same time may corrupt the cache
     };
 
     const buildResult = await spawnAsync(recipe.buildCommand, {

packages/e2e-tests/lib/constants.ts

@@ -3,4 +3,3 @@ export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5;
 export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2;
 export const VERDACCIO_VERSION = '5.22.1';
 export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
-export const TMP_DIR = 'tmp';

packages/e2e-tests/lib/runAllTestApps.ts

@@ -1,7 +1,4 @@
 /* eslint-disable no-console */
-import * as fs from 'fs';
-import * as os from 'os';
-import * as path from 'path';
 
 import { constructRecipeInstances } from './constructRecipeInstances';
 import { buildAndTestApp } from './runTestApp';
@@ -11,7 +8,7 @@ export async function runAllTestApps(
   recipePaths: string[],
   envVarsToInject: Record<string, string | undefined>,
 ): Promise<void> {
-  const maxParallel = process.env.CI ? 1 : 1; // For now we are disabling parallel execution because it was causing problems (runners were too slow and timeouts happened)
+  const maxParallel = process.env.CI ? 3 : 6;
 
   const recipeInstances = constructRecipeInstances(recipePaths);
 
@@ -37,8 +34,6 @@ export async function runAllTestApps(
 
   const failed = results.filter(result => result.buildFailed || result.testFailed);
 
-  fs.rmSync(path.join(os.tmpdir(), 'e2e-test-yarn-caches'), { force: true, recursive: true });
-
   if (failed.length) {
     console.log(`${failed.length} test(s) failed.`);
     process.exit(1);

packages/e2e-tests/lib/runTestApp.ts

@@ -1,15 +1,11 @@
-/* eslint-disable no-console */
-
-import * as fs from 'fs-extra';
+import * as fs from 'fs';
+import * as fsExtra from 'fs-extra';
 import * as path from 'path';
 
 import { buildApp } from './buildApp';
-import { TMP_DIR } from './constants';
 import { testApp } from './testApp';
 import type { Env, RecipeInstance, RecipeTestResult } from './types';
 
-let tmpDirCount = 0;
-
 // This should never throw, we always return a result here
 export async function buildAndTestApp(
   recipeInstance: RecipeInstance,
@@ -18,9 +14,11 @@ export async function buildAndTestApp(
   const { recipe, portModulo, portGap } = recipeInstance;
   const recipeDirname = path.dirname(recipe.path);
 
-  const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${tmpDirCount++}`);
+  const tmpFolder = path.join(__dirname, '..', 'tmp');
+  await fs.promises.mkdir(tmpFolder, { recursive: true });
+  const targetDir = await fs.promises.mkdtemp(path.join(tmpFolder, 'tmp-app-'));
 
-  await fs.copy(recipeDirname, targetDir);
+  await fsExtra.copy(recipeDirname, targetDir);
 
   const env: Env = {
     ...envVarsToInject,
@@ -31,7 +29,7 @@ export async function buildAndTestApp(
   try {
     await buildApp(targetDir, recipeInstance, env);
   } catch (error) {
-    await fs.remove(targetDir);
+    await fsExtra.remove(targetDir);
 
     return {
       ...recipeInstance,
@@ -42,15 +40,17 @@ export async function buildAndTestApp(
   }
 
   // This cannot throw, we always return a result here
-  const results = await testApp(targetDir, recipeInstance, env);
-
-  // Cleanup
-  await fs.remove(targetDir);
-
-  return {
-    ...recipeInstance,
-    buildFailed: false,
-    testFailed: results.some(result => result.result !== 'PASS'),
-    tests: results,
-  };
+  return testApp(targetDir, recipeInstance, env)
+    .finally(() => {
+      // Cleanup
+      void fsExtra.remove(targetDir);
+    })
+    .then(results => {
+      return {
+        ...recipeInstance,
+        buildFailed: false,
+        testFailed: results.some(result => result.result !== 'PASS'),
+        tests: results,
+      };
+    });
 }

packages/e2e-tests/test-applications/create-next-app/test-recipe.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "create-next-app",
-  "buildCommand": "yarn install && npx playwright install && yarn build",
+  "buildCommand": "pnpm install && npx playwright install && pnpm build",
   "tests": [
     {
       "testName": "Playwright tests - Prod Mode",

packages/e2e-tests/test-applications/create-react-app/src/App.test.tsx

@@ -1,6 +1,6 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "create-react-app",
-  "buildCommand": "yarn install && yarn build",
+  "buildCommand": "pnpm install && pnpm build",
   "tests": []
 }

packages/e2e-tests/test-applications/create-react-app/test-recipe.json

@@ -16,5 +16,6 @@
     "noEmit": true,
     "jsx": "react"
   },
-  "include": ["src"]
+  "include": ["src"],
+  "exclude": ["src/**/*.test.tsx"]
 }

packages/e2e-tests/test-applications/create-react-app/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "nextjs-13-app-dir",
-  "buildCommand": "yarn install && npx playwright install && yarn build",
+  "buildCommand": "pnpm install && npx playwright install && pnpm build",
   "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts",
   "tests": [
     {

packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "node-express-app",
-  "buildCommand": "yarn install && yarn build",
+  "buildCommand": "pnpm install && pnpm build",
   "tests": [
     {
       "testName": "Test express server",

packages/e2e-tests/test-applications/node-express-app/test-recipe.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "standard-frontend-react-tracing-import",
-  "buildCommand": "yarn install && npx playwright install && yarn build",
+  "buildCommand": "pnpm install && npx playwright install && pnpm build",
   "tests": [
     {
       "testName": "Playwright tests",

packages/e2e-tests/test-applications/standard-frontend-react-tracing-import/test-recipe.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "standard-frontend-react",
-  "buildCommand": "yarn install && npx playwright install && yarn build",
+  "buildCommand": "pnpm install && npx playwright install && pnpm build",
   "tests": [
     {
       "testName": "Playwright tests",