Add a flaky result type to be used for flaky scenarios.

* The flaky result type should be ok in non-strict mode and not ok in
  strict mode, so the exit code becomes zero with flaky scenarios in
  non-strict mode.
* Update the Summary report to detect and count flaky scenarios.

Author: brasmusson

lib/cucumber/core/report/summary.rb

@@ -6,6 +6,7 @@ class Summary
         attr_reader :test_cases, :test_steps
 
         def initialize(event_bus)
+          @previous_test_case = nil
           @test_cases = Test::Result::Summary.new
           @test_steps = Test::Result::Summary.new
           subscribe_to(event_bus)
@@ -19,7 +20,13 @@ def ok?(be_strict = false)
 
         def subscribe_to(event_bus)
           event_bus.on(:test_case_finished) do |event|
-            event.result.describe_to test_cases
+            if event.test_case != @previous_test_case
+              @previous_test_case = event.test_case
+              event.result.describe_to test_cases
+            elsif event.result.passed?
+              test_cases.flaky
+              test_cases.decrement_failed
+            end
           end
           event_bus.on(:test_step_finished) do |event|
             event.result.describe_to test_steps if is_step?(event.test_step)

lib/cucumber/core/test/result.rb

@@ -5,7 +5,7 @@ module Cucumber
   module Core
     module Test
       module Result
-        TYPES = [:failed, :skipped, :undefined, :pending, :passed, :unknown].freeze
+        TYPES = [:failed, :flaky, :skipped, :undefined, :pending, :passed, :unknown].freeze
 
         def self.ok?(type, be_strict = false)
           private
@@ -124,6 +124,15 @@ def with_filtered_backtrace(filter)
           end
         end
 
+        # Flaky is not used directly as an execution result, but is used as a
+        # reporting result type for test cases that fails and the passes on
+        # retry, therefore only the class method self.ok? is needed.
+        class Flaky
+          def self.ok?(be_strict = false)
+            !be_strict
+          end
+        end
+
         # Base class for exceptions that can be raised in a step definition causing
         # the step to have that result.
         class Raisable < StandardError
@@ -268,6 +277,10 @@ def total(for_status = nil)
             end
           end
 
+          def decrement_failed
+            @totals[:failed] -= 1
+          end
+          
           private
 
           def get_total(method_name)

spec/cucumber/core/report/summary_spec.rb

@@ -55,6 +55,16 @@ module Cucumber::Core::Report
         expect( @summary.test_cases.total(:undefined) ).to eq(1)
         expect( @summary.test_cases.total ).to eq(1)
       end
+
+      it "handles flaky test cases" do
+        allow(test_case).to receive(:==).and_return(false, true)
+        event_bus.send(:test_case_finished, test_case, failed_result)
+        event_bus.send(:test_case_finished, test_case, passed_result)
+
+        expect( @summary.test_cases.total(:failed) ).to eq(0)
+        expect( @summary.test_cases.total(:flaky) ).to eq(1)
+        expect( @summary.test_cases.total ).to eq(1)
+      end
     end
 
     context "test step summary" do

spec/cucumber/core/test/result_spec.rb

@@ -46,6 +46,7 @@ module Cucumber::Core::Test
       specify { expect( result ).not_to be_undefined }
       specify { expect( result ).not_to be_unknown   }
       specify { expect( result ).not_to be_skipped   }
+      specify { expect( result ).not_to be_flaky     }
 
       specify { expect( result ).to be_ok }
       specify { expect( result.ok?(false) ).to be_truthy }
@@ -105,6 +106,7 @@ module Cucumber::Core::Test
       specify { expect( result ).not_to be_undefined }
       specify { expect( result ).not_to be_unknown   }
       specify { expect( result ).not_to be_skipped   }
+      specify { expect( result ).not_to be_flaky     }
 
       specify { expect( result ).to_not be_ok }
       specify { expect( result.ok?(false) ).to be_falsey }
@@ -130,6 +132,7 @@ module Cucumber::Core::Test
       specify { expect( result ).not_to be_undefined }
       specify { expect( result ).to     be_unknown   }
       specify { expect( result ).not_to be_skipped   }
+      specify { expect( result ).not_to be_flaky     }
     end
 
     describe Result::Raisable do
@@ -196,6 +199,7 @@ module Cucumber::Core::Test
       specify { expect( result ).to     be_undefined }
       specify { expect( result ).not_to be_unknown   }
       specify { expect( result ).not_to be_skipped   }
+      specify { expect( result ).not_to be_flaky     }
 
       specify { expect( result ).to be_ok }
       specify { expect( result.ok?(false) ).to be_truthy }
@@ -218,6 +222,7 @@ module Cucumber::Core::Test
       specify { expect( result ).not_to be_undefined }
       specify { expect( result ).not_to be_unknown   }
       specify { expect( result ).to     be_skipped   }
+      specify { expect( result ).not_to be_flaky     }
 
       specify { expect( result ).to be_ok }
       specify { expect( result.ok?(false) ).to be_truthy }
@@ -240,13 +245,19 @@ module Cucumber::Core::Test
       specify { expect( result ).not_to be_undefined }
       specify { expect( result ).not_to be_unknown   }
       specify { expect( result ).not_to be_skipped   }
+      specify { expect( result ).not_to be_flaky     }
       specify { expect( result ).to     be_pending   }
 
       specify { expect( result ).to be_ok }
       specify { expect( result.ok?(false) ).to be_truthy }
       specify { expect( result.ok?(true) ).to be_falsey }
     end
 
+    describe Result::Flaky do
+      specify { expect( Result::Flaky.ok?(false) ).to be_truthy }
+      specify { expect( Result::Flaky.ok?(true) ).to be_falsey }
+    end
+
     describe Result::Summary do
       let(:summary)   { Result::Summary.new }
       let(:failed)    { Result::Failed.new(Result::Duration.new(10), exception) }
@@ -357,6 +368,12 @@ def describe_to(visitor, *args)
           expect( summary.ok? ).to be true
           expect( summary.ok?(true) ).to be false
         end
+
+        it "flaky result is ok if not strict" do
+          summary.flaky
+          expect( summary.ok? ).to be true
+          expect( summary.ok?(true) ).to be false
+        end
       end
     end