Always fail tests with an assertion error

Wrap callback errors, rejected promises or synchronously thrown values
in an `AssertionError`. Rely on the `actual` value being printed by the
reporters, which means it's still shown to the user.

Remove now obsolete `Error: ` prefix added to non-assertion errors in
`run-status.js`. Instead more helpful messages from `test.js` can be
shown.

Try to reuse the stack trace from any original error, so magic assert
can show context.

Fail `t.throws()` / `t.notThrows()` with an AssertionError if called
with an improper argument, rather than throwing a TypeError.

Author: novemberborn

lib/assert.js

@@ -122,7 +122,11 @@ function wrapAssertions(callbacks) {
 			} else if (isObservable(fn)) {
 				promise = observableToPromise(fn);
 			} else if (typeof fn !== 'function') {
-				throw new TypeError('t.throws must be called with a function, Promise, or Observable');
+				fail(this, new AssertionError({
+					actual: fn,
+					message: '`t.throws()` must be called with a function, Promise, or Observable'
+				}));
+				return;
 			}
 
 			let coreAssertThrowsErrorArg;
@@ -177,7 +181,11 @@ function wrapAssertions(callbacks) {
 			} else if (isObservable(fn)) {
 				promise = observableToPromise(fn);
 			} else if (typeof fn !== 'function') {
-				throw new TypeError('t.notThrows must be called with a function, Promise, or Observable');
+				fail(this, new AssertionError({
+					actual: fn,
+					message: '`t.notThrows()` must be called with a function, Promise, or Observable'
+				}));
+				return;
 			}
 
 			const test = fn => {

lib/run-status.js

@@ -87,10 +87,6 @@ class RunStatus extends EventEmitter {
 		test.title = this.prefixTitle(test.file) + test.title;
 
 		if (test.error) {
-			if (test.error.name !== 'AssertionError') {
-				test.error.message = `Error: ${test.error.message}`;
-			}
-
 			this.errors.push(test);
 		}
 

lib/test.js

@@ -1,5 +1,4 @@
 'use strict';
-const inspect = require('util').inspect;
 const isGeneratorFn = require('is-generator-fn');
 const maxTimeout = require('max-timeout');
 const Promise = require('bluebird');
@@ -129,8 +128,6 @@ class Test {
 		this.assertCount++;
 	}
 	_setAssertError(err) {
-		throwsHelper(err);
-
 		if (this.assertError !== undefined) {
 			return;
 		}
@@ -155,16 +152,17 @@ class Test {
 			ret = this.fn(this._createExecutionContext());
 		} catch (err) {
 			this.threwSync = true;
+			throwsHelper(err);
 
-			if (err instanceof Error) {
-				this._setAssertError(err);
-			} else {
-				this._setAssertError(new assert.AssertionError({
+			let error = err;
+			if (!(err instanceof assert.AssertionError)) {
+				error = new assert.AssertionError({
 					actual: err,
-					message: `Non-error thrown with value: ${inspect(err, {depth: null})}`,
-					operator: 'catch'
-				}));
+					message: `Error thrown in test`,
+					stack: err instanceof Error && err.stack
+				});
 			}
+			this._setAssertError(error);
 		}
 
 		return ret;
@@ -214,11 +212,13 @@ class Test {
 			return Promise.resolve(ret).then(
 				() => this.exit(),
 				err => {
-					if (!(err instanceof Error)) {
+					throwsHelper(err);
+
+					if (!(err instanceof assert.AssertionError)) {
 						err = new assert.AssertionError({
 							actual: err,
-							message: `Promise rejected with: ${inspect(err, {depth: null})}`,
-							operator: 'promise'
+							message: 'Rejected promise returned by test',
+							stack: err instanceof Error && err.stack
 						});
 					}
 
@@ -263,11 +263,10 @@ class Test {
 	}
 	_end(err) {
 		if (err) {
-			if (!(err instanceof Error)) {
+			if (!(err instanceof assert.AssertionError)) {
 				err = new assert.AssertionError({
 					actual: err,
-					message: 'Callback called with an error: ' + inspect(err, {depth: null}),
-					operator: 'callback'
+					message: 'Callback called with an error'
 				});
 			}
 

test/assert.js

@@ -412,8 +412,8 @@ test('.throws should throw if passed a bad value', t => {
 	t.throws(() => {
 		assertions.throws('not a function');
 	}, {
-		name: 'TypeError',
-		message: /t\.throws must be called with a function, Promise, or Observable/
+		name: 'AssertionError',
+		message: /`t\.throws\(\)` must be called with a function, Promise, or Observable/
 	});
 });
 
@@ -423,8 +423,8 @@ test('.notThrows should throw if passed a bad value', t => {
 	t.throws(() => {
 		assertions.notThrows('not a function');
 	}, {
-		name: 'TypeError',
-		message: /t\.notThrows must be called with a function, Promise, or Observable/
+		name: 'AssertionError',
+		message: /`t\.notThrows\(\)` must be called with a function, Promise, or Observable/
 	});
 });
 

test/promise.js

@@ -293,8 +293,8 @@ test('reject', t => {
 		});
 	}).run().then(result => {
 		t.is(result.passed, false);
-		t.is(result.reason.name, 'Error');
-		t.is(result.reason.message, 'unicorn');
+		t.is(result.reason.actual.name, 'Error');
+		t.is(result.reason.actual.message, 'unicorn');
 		t.end();
 	});
 });
@@ -303,7 +303,8 @@ test('reject with non-Error', t => {
 	ava(() => Promise.reject('failure')).run().then(result => {
 		t.is(result.passed, false);
 		t.is(result.reason.name, 'AssertionError');
-		t.is(result.reason.message, 'Promise rejected with: \'failure\'');
+		t.is(result.reason.message, 'Rejected promise returned by test');
+		t.is(result.reason.actual, 'failure');
 		t.end();
 	});
 });

test/test.js

@@ -131,14 +131,15 @@ test('run more assertions than planned', t => {
 	t.end();
 });
 
-test('handle non-assertion errors', t => {
+test('wrap non-assertion errors', t => {
+	const err = new Error();
 	const result = ava(() => {
-		throw new Error();
+		throw err;
 	}).run();
 
 	t.is(result.passed, false);
-	t.is(result.reason.name, 'Error');
-	t.true(result.reason instanceof Error);
+	t.is(result.reason.name, 'AssertionError');
+	t.is(result.reason.actual, err);
 	t.end();
 });
 
@@ -157,7 +158,9 @@ test('end can be used as callback with error', t => {
 		a.end(err);
 	}).run().then(result => {
 		t.is(result.passed, false);
-		t.is(result.reason, err);
+		t.is(result.reason.name, 'AssertionError');
+		t.is(result.reason.actual, err);
+		t.is(result.reason.message, 'Callback called with an error');
 		t.end();
 	});
 });
@@ -171,20 +174,20 @@ test('end can be used as callback with a non-error as its error argument', t =>
 		t.ok(result.reason);
 		t.is(result.reason.name, 'AssertionError');
 		t.is(result.reason.actual, nonError);
-		t.is(result.reason.message, 'Callback called with an error: { foo: \'bar\' }');
+		t.is(result.reason.message, 'Callback called with an error');
 		t.end();
 	});
 });
 
 test('handle non-assertion errors even when planned', t => {
+	const err = new Error('bar');
 	const result = ava(a => {
 		a.plan(1);
-		throw new Error('bar');
+		throw err;
 	}).run();
 
 	t.is(result.passed, false);
-	t.is(result.reason.name, 'Error');
-	t.is(result.reason.message, 'bar');
+	t.is(result.reason.actual, err);
 	t.end();
 });
 
@@ -288,7 +291,9 @@ test('fails if a bad value is passed to t.throws', t => {
 
 	t.is(result.passed, false);
 	t.ok(result.reason);
-	t.is(result.reason.name, 'TypeError');
+	t.is(result.reason.name, 'AssertionError');
+	t.is(result.reason.message, '`t.throws()` must be called with a function, Promise, or Observable');
+	t.is(result.reason.actual, 'not a function');
 	t.end();
 });
 
@@ -427,9 +432,8 @@ test('fails with thrown falsy value', t => {
 
 	t.is(result.passed, false);
 	t.is(result.reason.actual, 0);
-	t.is(result.reason.message, 'Non-error thrown with value: 0');
+	t.is(result.reason.message, 'Error thrown in test');
 	t.is(result.reason.name, 'AssertionError');
-	t.is(result.reason.operator, 'catch');
 	t.end();
 });
 
@@ -441,9 +445,8 @@ test('fails with thrown non-error object', t => {
 
 	t.is(result.passed, false);
 	t.is(result.reason.actual, obj);
-	t.is(result.reason.message, 'Non-error thrown with value: { foo: \'bar\' }');
+	t.is(result.reason.message, 'Error thrown in test');
 	t.is(result.reason.name, 'AssertionError');
-	t.is(result.reason.operator, 'catch');
 	t.end();
 });
 
@@ -489,12 +492,13 @@ test('end should not be called multiple times', t => {
 });
 
 test('cb test that throws sync', t => {
+	const err = new Error('foo');
 	const result = ava.cb(() => {
-		throw new Error('foo');
+		throw err;
 	}).run();
 
 	t.is(result.passed, false);
-	t.is(result.reason.message, 'foo');
+	t.is(result.reason.actual, err);
 	t.end();
 });