feat: add option to not fail on failing test suite (#4771)

* Add option fail-on-failing-test-suite to not fail test run if there were no infrastructure problem

* Add test on failOnFailingTestSuite

* Use parameters which by default is false

* Update docs/index.md

* refactor: shared clampedCode

---------

Co-authored-by: Josh Goldberg <[email protected]>

Author: ilgonmic

docs/index.md

@@ -915,6 +915,10 @@ To ensure your tests aren't leaving messes around, here are some ideas to get st
 - Try something like [wtfnode][npm-wtfnode]
 - Use [`.only`](#exclusive-tests) until you find the test that causes Mocha to hang
 
+### `--pass-on-failing-test-suite`
+
+If set to `true`, Mocha returns exit code `0` even if there are failed tests during run.
+
 ### `--fail-zero`
 
 > _New in v9.1.0_ Fail test run if no tests are encountered with `exit-code: 1`.

lib/cli/run-helpers.js

@@ -21,25 +21,24 @@ const {UnmatchedFile} = require('./collect-files');
 
 /**
  * Exits Mocha when tests + code under test has finished execution (default)
- * @param {number} code - Exit code; typically # of failures
+ * @param {number} clampedCode - Exit code; typically # of failures
  * @ignore
  * @private
  */
-const exitMochaLater = code => {
+const exitMochaLater = clampedCode => {
   process.on('exit', () => {
-    process.exitCode = Math.min(code, 255);
+    process.exitCode = clampedCode;
   });
 };
 
 /**
  * Exits Mocha when Mocha itself has finished execution, regardless of
  * what the tests or code under test is doing.
- * @param {number} code - Exit code; typically # of failures
+ * @param {number} clampedCode - Exit code; typically # of failures
  * @ignore
  * @private
  */
-const exitMocha = code => {
-  const clampedCode = Math.min(code, 255);
+const exitMocha = clampedCode => {
   let draining = 0;
 
   // Eagerly set the process's exit code in case stream.write doesn't
@@ -139,12 +138,17 @@ const handleUnmatchedFiles = (mocha, unmatchedFiles) => {
  * @param {Mocha} mocha - Mocha instance
  * @param {Options} [opts] - Command line options
  * @param {boolean} [opts.exit] - Whether or not to force-exit after tests are complete
+ * @param {boolean} [opts.passOnFailingTestSuite] - Whether or not to fail test run if tests were failed
  * @param {Object} fileCollectParams - Parameters that control test
  *   file collection. See `lib/cli/collect-files.js`.
  * @returns {Promise<Runner>}
  * @private
  */
-const singleRun = async (mocha, {exit}, fileCollectParams) => {
+const singleRun = async (
+  mocha,
+  {exit, passOnFailingTestSuite},
+  fileCollectParams
+) => {
   const fileCollectionObj = collectFiles(fileCollectParams);
 
   if (fileCollectionObj.unmatchedFiles.length > 0) {
@@ -156,7 +160,9 @@ const singleRun = async (mocha, {exit}, fileCollectParams) => {
 
   // handles ESM modules
   await mocha.loadFilesAsync();
-  return mocha.run(exit ? exitMocha : exitMochaLater);
+  return mocha.run(
+    createExitHandler({exit, passOnFailingTestSuite})
+  );
 };
 
 /**
@@ -186,7 +192,9 @@ const parallelRun = async (mocha, options, fileCollectParams) => {
   mocha.files = fileCollectionObj.files;
 
   // note that we DO NOT load any files here; this is handled by the worker
-  return mocha.run(options.exit ? exitMocha : exitMochaLater);
+  return mocha.run(
+    createExitHandler(options)
+  );
 };
 
 /**
@@ -282,3 +290,15 @@ exports.validateLegacyPlugin = (opts, pluginType, map = {}) => {
     }
   }
 };
+
+const createExitHandler = ({ exit, passOnFailingTestSuite }) => {
+  return code => {
+    const clampedCode = passOnFailingTestSuite
+      ? 0
+      : Math.min(code, 255);
+
+    return exit
+      ? exitMocha(clampedCode)
+      : exitMochaLater(clampedCode);
+  };
+};

lib/cli/run-option-metadata.js

@@ -35,6 +35,7 @@ const TYPES = (exports.types = {
     'diff',
     'dry-run',
     'exit',
+    'pass-on-failing-test-suite',
     'fail-zero',
     'forbid-only',
     'forbid-pending',

lib/cli/run.js

@@ -98,6 +98,11 @@ exports.builder = yargs =>
         requiresArg: true,
         coerce: list
       },
+      'pass-on-failing-test-suite': {
+        default: false,
+        description: 'Not fail test run if tests were failed',
+        group: GROUPS.RULES
+      },
       'fail-zero': {
         description: 'Fail test run if no test(s) encountered',
         group: GROUPS.RULES

lib/mocha.js

@@ -157,6 +157,7 @@ exports.run = function (...args) {
  * @param {boolean} [options.delay] - Delay root suite execution?
  * @param {boolean} [options.diff] - Show diff on failure?
  * @param {boolean} [options.dryRun] - Report tests without running them?
+ * @param {boolean} [options.passOnFailingTestSuite] - Fail test run if tests were failed?
  * @param {boolean} [options.failZero] - Fail test run if zero tests?
  * @param {string} [options.fgrep] - Test filter given string.
  * @param {boolean} [options.forbidOnly] - Tests marked `only` fail the suite?
@@ -216,6 +217,7 @@ function Mocha(options = {}) {
     'delay',
     'diff',
     'dryRun',
+    'passOnFailingTestSuite',
     'failZero',
     'forbidOnly',
     'forbidPending',
@@ -870,6 +872,20 @@ Mocha.prototype.failZero = function (failZero) {
   return this;
 };
 
+/**
+ * Fail test run if tests were failed.
+ *
+ * @public
+ * @see [CLI option](../#-pass-on-failing-test-suite)
+ * @param {boolean} [passOnFailingTestSuite=false] - Whether to fail test run.
+ * @return {Mocha} this
+ * @chainable
+ */
+Mocha.prototype.passOnFailingTestSuite = function(passOnFailingTestSuite) {
+  this.options.passOnFailingTestSuite = passOnFailingTestSuite === true;
+  return this;
+};
+
 /**
  * Causes tests marked `only` to fail the suite.
  *

test/integration/fixtures/failing-sync.fixture.js

@@ -0,0 +1,9 @@
+'use strict';
+
+var assert = require('assert');
+
+describe('a suite', function() {
+  it('should succeed', function() {
+    assert(false);
+  });
+});

test/integration/options/passOnFailingTestSuite.spec.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var helpers = require('../helpers');
+var runMochaJSON = helpers.runMochaJSON;
+
+describe('Enabled --pass-on-failing-test-suite', function() {
+  var args = ['--pass-on-failing-test-suite=true'];
+
+  it('Test should finish with zero code with disabled option', function(done) {
+    var fixture = 'failing-sync.fixture.js';
+    runMochaJSON(fixture, args, function(err, res) {
+      if (err) {
+        return done(err);
+      }
+
+      expect(res, 'to have passed test count', 0)
+        .and('to have test count', 1)
+        .and('to have exit code', 0);
+      done();
+    });
+  });
+});
+
+describe('Disabled --pass-on-failing-test-suite', function() {
+  var args = ['--pass-on-failing-test-suite=false'];
+
+  it('Test should return non-zero code with enabled option', function(done) {
+    var fixture = 'failing-sync.fixture.js';
+    runMochaJSON(fixture, args, function(err, res) {
+      if (err) {
+        return done(err);
+      }
+
+      expect(res, 'to have passed test count', 0)
+        .and('to have test count', 1)
+        .and('to have exit code', 1);
+      done();
+    });
+  });
+});

test/unit/mocha.spec.js

@@ -376,6 +376,28 @@ describe('Mocha', function () {
       });
     });
 
+    describe('passOnFailingTestSuite()', function() {
+      it('should set the passOnFailingTestSuite option to false', function() {
+        mocha.passOnFailingTestSuite();
+        expect(
+          mocha.options,
+          'to have property',
+          'passOnFailingTestSuite',
+          false
+        );
+      });
+
+      it('should set the passOnFailingTestSuite option to true', function() {
+        mocha.passOnFailingTestSuite(true);
+        expect(
+          mocha.options,
+          'to have property',
+          'passOnFailingTestSuite',
+          true
+        );
+      });
+    });
+
     describe('failZero()', function () {
       it('should set the failZero option to true', function () {
         mocha.failZero();