chore: introduce test debugging for cli tests (#5319)

* chore: introduce test debugging for cli tests

This is mostly useful for dev server tests

* chore: update contributors field

* chore: threads

Co-authored-by: danez <[email protected]>

Author: danez

CONTRIBUTING.md

@@ -94,7 +94,7 @@ A good place to start is reading the base command README and looking at the comm
 
 ### Testing
 
-This repo uses [ava](https://github.com/avajs/ava) for testing. Unit tests are in the `tests/unit` folder and
+This repo uses [vitest](https://github.com/vitest-dev/vitest) for testing. Unit tests are in the `tests/unit` folder and
 integration tests are in the `tests/integration` folder. We use this convention since we split tests across multiple CI
 machines to speed up CI time. You can read about it more [here](https://github.com/netlify/cli/issues/4178).
 
@@ -105,6 +105,32 @@ We also test for a few other things:
 - Test coverage
 - Must work with Windows + Unix environments.
 
+#### Debugging tests
+
+To run a single test file you can do:
+
+```
+npm exec vitest -- run tests/unit/tests/unit/lib/account.test.mjs
+```
+
+To run a single test you can either use `test.only` inside the test file and ran the above command or run this:
+
+```
+npm exec vitest -- run tests/unit/tests/unit/lib/account.test.mjs -t 'test name'
+```
+
+Some of the tests actually start the CLI in a subprocess and therefore sometimes underlying errors are not visible in
+the tests when they fail. By default the output of the subprocess is not forwarded to the main process to keep the cli
+output clean. To debug test failures like this you can set the environment variable `DEBUG_TESTS=true` and the
+subprocess will pipe it's output to the main process for you to see.
+
+When `DEBUG_TESTS` is set the vitest reporter will be set to `tap` so the test output won't interfere with the debug
+output.
+
+```
+DEBUG_TESTS=true npm exec vitest -- run tests/unit/tests/unit/lib/account.test.mjs -t 'test name'
+```
+
 ### Lint docs per Netlify style guide
 
 1. [Install vale](https://docs.errata.ai/vale/install)

package.json

@@ -150,9 +150,9 @@
     "cesare soldini <[email protected]>",
     "chris (fool) mccraw (http://twitter.com/fool)",
     "dustincrogers",
+    "eddie",
     "ehmicky (https://twitter.com/ehmicky)",
     "internal tools netlibot",
-    "just toby",
     "kvn-shn",
     "netlibot (https://www.netlify.com)",
     "nikoladev",
@@ -211,7 +211,7 @@
     "test:init:eleventy-deps": "npm ci --prefix tests/integration/eleventy-site --no-audit",
     "test:init:hugo-deps": "npm ci --prefix tests/integration/hugo-site --no-audit",
     "test:dev:ava": "ava --verbose",
-    "test:dev:vitest": "vitest run",
+    "test:dev:vitest": "vitest run tests/unit/ && vitest run tests/integration",
     "test:ci:ava:integration": "c8 -r json ava --concurrency 1 --no-worker-threads tests/integration/",
     "test:ci:vitest:unit": "vitest run --coverage tests/unit/",
     "test:ci:vitest:integration": "vitest run --coverage --no-threads tests/integration/",

tests/integration/utils/dev-server.cjs

@@ -44,6 +44,11 @@ const startServer = async ({
     getExecaOptions({ cwd, env }),
   )
 
+  if (process.env.DEBUG_TESTS) {
+    ps.stderr.pipe(process.stderr)
+    ps.stdout.pipe(process.stdout)
+  }
+
   const promptHistory = []
 
   if (prompt) {

vitest.config.mjs

@@ -1,4 +1,6 @@
 /// <reference types="vitest" />
+import process from 'process'
+
 import { defineConfig } from 'vite'
 
 export default defineConfig({
@@ -9,6 +11,7 @@ export default defineConfig({
       external: ['**/fixtures/**', '**/node_modules/**'],
       interopDefault: false,
     },
+    reporters: [process.env.DEBUG_TESTS ? 'tap' : 'default'],
     coverage: {
       provider: 'c8',
       reporter: ['text', 'lcov'],