test harness: Let tests fail early by exiting non-zero or by setting a flag.

Fixes #609.

Author: danielshahaf

tests/README.md

@@ -24,11 +24,15 @@ point will not fail the test), and `$todo` is used as the explanation.
 If a test sets `$skip_test` to a non-empty string, the test will be skipped
 with the provided string as the reason.
 
-3.
+3. 
+If a test sets `$fail_test` to a non-empty string, the test will be skipped
+with the provided string as the reason.
+
+4.
 If a test sets `unsorted=1` the order of highlights in `$expected_region_highlight`
 need not match the order in `$region_highlight`.
 
-4.
+5.
 Normally, tests fail if `$expected_region_highlight` and `$region_highlight`
 have different numbers of elements.  To mark this check as expected to fail,
 tests may set `$expected_mismatch` to an explanation string (like `$todo`);

tests/test-highlighting.zsh

@@ -118,7 +118,7 @@ run_test_internal() {
   builtin cd -q -- "$tests_tempdir" || { echo >&2 "Bail out! On ${(qq)1}: cd failed: $?"; return 1 }
 
   # Load the data and prepare checking it.
-  local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test unsorted=0
+  local BUFFER CURSOR MARK PENDING PREBUFFER REGION_ACTIVE WIDGET REPLY skip_test fail_test unsorted=0
   local expected_mismatch
   local skip_mismatch
   local -a expected_region_highlight region_highlight
@@ -127,9 +127,15 @@ run_test_internal() {
   local RETURN=""
   () {
     setopt localoptions
-    . "$srcdir"/"$ARG"
 
     # WARNING: The remainder of this anonymous function will run with the test's options in effect
+    if { ! . "$srcdir"/"$ARG" } || (( $#fail_test )); then
+      print -r -- "1..1"
+      print -r -- "## ${ARG:t:r}"
+      tap_escape $fail_test; fail_test=$REPLY
+      print -r -- "not ok 1 - failed setup: $fail_test"
+      return ${RETURN:=0}
+    fi
 
     (( $#skip_test )) && {
       print -r -- "1..0 # SKIP $skip_test"