From e107a3c0c65b1dc473b2109729251e6c99560433 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Fri, 13 May 2016 15:40:00 +0800 Subject: [PATCH 01/11] add failing metadata --- lib/runner.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/runner.js b/lib/runner.js index 3b28219dc..844108e94 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -16,6 +16,7 @@ var chainableMethods = { exclusive: false, skipped: false, todo: false, + failing: false, callback: false, always: false }, @@ -26,6 +27,7 @@ var chainableMethods = { after: {type: 'after'}, skip: {skipped: true}, todo: {todo: true}, + failing: {failing: true}, only: {exclusive: true}, beforeEach: {type: 'beforeEach'}, afterEach: {type: 'afterEach'}, From e8ab22889b32defd215ca4dad06d444a7fef34dc Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Sat, 14 May 2016 16:34:22 +0800 Subject: [PATCH 02/11] flip Test._result when failing=true --- lib/test.js | 3 +++ test/test.js | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/test.js b/lib/test.js index 5964887e0..2c7aa96c4 100644 --- a/lib/test.js +++ b/lib/test.js @@ -187,6 +187,9 @@ Test.prototype.run = function () { Test.prototype._result = function () { var passed = this.assertError === undefined; + if (this.metadata.failing) { + passed = !passed; + } return {passed: passed, result: this, reason: this.assertError}; }; diff --git a/test/test.js b/test/test.js index 2b691603f..3e6cb2dd2 100644 --- a/test/test.js +++ b/test/test.js @@ -11,6 +11,12 @@ function ava(title, fn, contextRef, report) { return t; } +ava.failing = function (title, fn, contextRef, report) { + var t = new Test(title, fn, contextRef, report); + t.metadata = {callback: false, failing: true}; + return t; +}; + ava.cb = function (title, fn, contextRef, report) { var t = new Test(title, fn, contextRef, report); t.metadata = {callback: true}; @@ -34,6 +40,15 @@ test('run test', function (t) { t.end(); }); +test('expected failing test', function (t) { + var result = ava.failing('foo', function (a) { + a.fail(); + }).run(); + + t.is(result.passed, true); + t.end(); +}); + test('title is optional', function (t) { var result = ava(function (a) { a.pass(); From 24135941b48b605606c9d93150541fbe9a82f5b0 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Sat, 14 May 2016 16:59:33 +0800 Subject: [PATCH 03/11] add tests for verifying failing worked for callback --- test/test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/test.js b/test/test.js index 3e6cb2dd2..2cfe46b5a 100644 --- a/test/test.js +++ b/test/test.js @@ -23,6 +23,12 @@ ava.cb = function (title, fn, contextRef, report) { return t; }; +ava.cb.failing = function (title, fn, contextRef, report) { + var t = new Test(title, fn, contextRef, report); + t.metadata = {callback: true, failing: true}; + return t; +}; + test('must be called with new', function (t) { t.throws(function () { var test = Test; @@ -153,6 +159,17 @@ test('end can be used as callback with error', function (t) { }); }); +test('failing can work with callback', function (t) { + var err = new Error('failed'); + ava.cb.failing(function (a) { + a.end(err); + }).run().then(function (result) { + t.is(result.passed, true); + t.is(result.reason, err); + t.end(); + }); +}); + test('end can be used as callback with a non-error as its error argument', function (t) { var nonError = {foo: 'bar'}; ava.cb(function (a) { From fa322095f1d5c46fcebc0d5d7b1e9eea382494c9 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Sat, 14 May 2016 17:35:21 +0800 Subject: [PATCH 04/11] add tests verifying failing test fail if passed --- lib/test.js | 8 ++++++-- test/test.js | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/test.js b/lib/test.js index 2c7aa96c4..9873f4f78 100644 --- a/lib/test.js +++ b/lib/test.js @@ -186,11 +186,15 @@ Test.prototype.run = function () { }; Test.prototype._result = function () { - var passed = this.assertError === undefined; + var reason = this.assertError; + var passed = reason === undefined; if (this.metadata.failing) { passed = !passed; + if (!passed) { + reason = new Error('Test was expected to fail, but succeeded, you should unmark the test as failing'); + } } - return {passed: passed, result: this, reason: this.assertError}; + return {passed: passed, result: this, reason: reason}; }; Object.defineProperty(Test.prototype, 'end', { diff --git a/test/test.js b/test/test.js index 2cfe46b5a..457435bc9 100644 --- a/test/test.js +++ b/test/test.js @@ -55,6 +55,16 @@ test('expected failing test', function (t) { t.end(); }); +test('fail a failing test if it pass', function (t) { + var result = ava.failing('foo', function (a) { + a.pass(); + }).run(); + + t.is(result.passed, false); + t.is(result.reason.message, 'Test was expected to fail, but succeeded, you should unmark the test as failing'); + t.end(); +}); + test('title is optional', function (t) { var result = ava(function (a) { a.pass(); @@ -159,6 +169,16 @@ test('end can be used as callback with error', function (t) { }); }); +test('fail a failing callback test if it passed', function (t) { + ava.cb.failing(function (a) { + a.end(); + }).run().then(function (result) { + t.is(result.passed, false); + t.is(result.reason.message, 'Test was expected to fail, but succeeded, you should unmark the test as failing'); + t.end(); + }); +}); + test('failing can work with callback', function (t) { var err = new Error('failed'); ava.cb.failing(function (a) { From 25b4979d072017ed844212b702556d23efcf8bce Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Sat, 14 May 2016 18:08:31 +0800 Subject: [PATCH 05/11] add tests for failing test returns a resolved promise --- test/test.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/test.js b/test/test.js index 457435bc9..544e167e0 100644 --- a/test/test.js +++ b/test/test.js @@ -5,6 +5,8 @@ var delay = require('delay'); var isPromise = require('is-promise'); var Test = require('../lib/test'); +var failingTestHint = 'Test was expected to fail, but succeeded, you should unmark the test as failing'; + function ava(title, fn, contextRef, report) { var t = new Test(title, fn, contextRef, report); t.metadata = {callback: false}; @@ -61,7 +63,7 @@ test('fail a failing test if it pass', function (t) { }).run(); t.is(result.passed, false); - t.is(result.reason.message, 'Test was expected to fail, but succeeded, you should unmark the test as failing'); + t.is(result.reason.message, failingTestHint); t.end(); }); @@ -174,7 +176,7 @@ test('fail a failing callback test if it passed', function (t) { a.end(); }).run().then(function (result) { t.is(result.passed, false); - t.is(result.reason.message, 'Test was expected to fail, but succeeded, you should unmark the test as failing'); + t.is(result.reason.message, failingTestHint); t.end(); }); }); @@ -639,3 +641,15 @@ test('it is an error to set context in a hook', function (t) { t.match(result.reason.message, /t\.context is not available in foo tests/); t.end(); }); + +test('failing test returns a resolved promise is failure', function (t) { + ava.failing(function (a) { + a.plan(1); + a.notThrows(delay(10), 'foo'); + return Promise.resolve(); + }).run().then(function (result) { + t.is(result.passed, false); + t.is(result.reason.message, failingTestHint); + t.end(); + }); +}); From 7d10392af94a07332dc1a9f403ed3319ee7d74c3 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Sat, 14 May 2016 18:26:42 +0800 Subject: [PATCH 06/11] add tests for failing test returns a rejected promise --- test/test.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test.js b/test/test.js index 544e167e0..00b243714 100644 --- a/test/test.js +++ b/test/test.js @@ -653,3 +653,14 @@ test('failing test returns a resolved promise is failure', function (t) { t.end(); }); }); + +test('failing test returns a rejected promise is passing', function (t) { + ava.failing(function (a) { + a.plan(1); + a.notThrows(delay(10), 'foo'); + return Promise.reject(); + }).run().then(function (result) { + t.is(result.passed, true); + t.end(); + }); +}); From 98e43e4e9a040f972a3836d0243b26d6b9e379f8 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Mon, 16 May 2016 09:02:18 +0800 Subject: [PATCH 07/11] add tests for failing test with t.throws --- test/test.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test.js b/test/test.js index 00b243714..d2f3f1b63 100644 --- a/test/test.js +++ b/test/test.js @@ -664,3 +664,22 @@ test('failing test returns a rejected promise is passing', function (t) { t.end(); }); }); + +test('failing test with t.throws(nonThrowingPromise) is passing', function (t) { + ava.failing(function (a) { + a.throws(Promise.resolve(10)); + }).run().then(function (result) { + t.is(result.passed, true); + t.end(); + }); +}); + +test('failing test with t.throws(throws) is failure', function (t) { + ava.failing(function (a) { + a.throws(Promise.resolve('foo')); + }).run().then(function (result) { + t.is(result.passed, false); + t.is(result.message, failingTestHint); + t.end(); + }); +}); From a12227e52c1d8949a56ef2b10ea001900a5ddaf4 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Mon, 16 May 2016 09:43:44 +0800 Subject: [PATCH 08/11] fix test --- test/test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.js b/test/test.js index d2f3f1b63..9fd23a972 100644 --- a/test/test.js +++ b/test/test.js @@ -676,10 +676,10 @@ test('failing test with t.throws(nonThrowingPromise) is passing', function (t) { test('failing test with t.throws(throws) is failure', function (t) { ava.failing(function (a) { - a.throws(Promise.resolve('foo')); + a.notThrows(Promise.resolve('foo')); }).run().then(function (result) { t.is(result.passed, false); - t.is(result.message, failingTestHint); + t.is(result.reason.message, failingTestHint); t.end(); }); }); From 2378a44e4e71be6d60e44e0058c8bab790f59cfc Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Mon, 16 May 2016 09:47:22 +0800 Subject: [PATCH 09/11] fix test title --- test/test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.js b/test/test.js index 9fd23a972..209ddc706 100644 --- a/test/test.js +++ b/test/test.js @@ -674,7 +674,7 @@ test('failing test with t.throws(nonThrowingPromise) is passing', function (t) { }); }); -test('failing test with t.throws(throws) is failure', function (t) { +test('failing test with t.notThrows(throws) is failure', function (t) { ava.failing(function (a) { a.notThrows(Promise.resolve('foo')); }).run().then(function (result) { From c4d1306cc881586347eeedf6824fcdcf93875a24 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Mon, 16 May 2016 10:58:20 +0800 Subject: [PATCH 10/11] update readme about failing modifier --- readme.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/readme.md b/readme.md index a672b6983..8f8dcb5e1 100644 --- a/readme.md +++ b/readme.md @@ -415,6 +415,16 @@ You can use the `.todo` modifier when you're planning to write a test. Like skip test.todo('will think about writing this later'); ``` +### Failing tests + +You can use the `.failing` modifier for submitting tests that are intended to be failures without killing CI process. And if the failing test does pass, it will fail, and it will remind you remove the failing modifier. + +```js +test.failing('this test prove something wrong', t => { + t.fail(); +}); // will count as passed test +``` + ### Before & after hooks AVA lets you register hooks that are run before and after your tests. This allows you to run setup and/or teardown code. From 4ac5ebe7e6d237f63b129077b4911861f3c6a6f3 Mon Sep 17 00:00:00 2001 From: cgcgbcbc Date: Mon, 16 May 2016 13:44:56 +0800 Subject: [PATCH 11/11] update readme --- readme.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 8f8dcb5e1..41864eef7 100644 --- a/readme.md +++ b/readme.md @@ -417,12 +417,15 @@ test.todo('will think about writing this later'); ### Failing tests -You can use the `.failing` modifier for submitting tests that are intended to be failures without killing CI process. And if the failing test does pass, it will fail, and it will remind you remove the failing modifier. +You can use the `.failing` modifier to document issues with your code that need to be fixed. Failing tests are run just like normal ones, but they are expected to fail, and will not break your build when they do. If a test marked as failing actually passes, it will be reported as an error and fail the build with a helpful message instructing you to remove the .failing modifier. + +This allows you to merge `.failing` tests before a fix is implemented without breaking CI. This is a great way to recognize good bug report PR's with a commit credit, even if the reporter is unable to actually fix the problem. ```js -test.failing('this test prove something wrong', t => { - t.fail(); -}); // will count as passed test +// See: github.com/user/repo/issues/1234 +test.failing('demonstrate some bug', t => { + t.fail(); // test will count as passed +}); ``` ### Before & after hooks