When bisecting, exit each run as soon as possible.

- When an expected failure passes (or is pending)
  we don’t care about any other results.
- When the last expected failure finishes, we don’t
  care about any later examples.

Author: myronmarston

lib/rspec/core/bisect/runner.rb

@@ -17,9 +17,7 @@ def initialize(server, original_cli_args)
         end
 
         def run(locations)
-          @server.capture_run_results do
-            system command_for(locations)
-          end
+          run_locations(locations, original_results.failed_example_ids)
         end
 
         def command_for(locations)
@@ -47,11 +45,17 @@ def repro_command_from(locations)
         end
 
         def original_results
-          @original_results ||= run(original_locations)
+          @original_results ||= run_locations(original_locations)
         end
 
       private
 
+        def run_locations(locations, *capture_args)
+          @server.capture_run_results(*capture_args) do
+            system command_for(locations)
+          end
+        end
+
         def reusable_cli_options
           @reusable_cli_options ||= begin
             opts = original_cli_args_without_locations

lib/rspec/core/bisect/server.rb

@@ -16,8 +16,8 @@ def self.run
           server.stop
         end
 
-        def capture_run_results(abort_after_example_id=nil)
-          self.abort_after_example_id = abort_after_example_id
+        def capture_run_results(expected_failures=[])
+          self.expected_failures = expected_failures
           yield
           latest_run_results
         end
@@ -36,7 +36,7 @@ def drb_port
         end
 
         # Fetched via DRb by the BisectFormatter to determine when to abort.
-        attr_accessor :abort_after_example_id
+        attr_accessor :expected_failures
 
         # Set via DRb by the BisectFormatter with the results of the run.
         attr_accessor :latest_run_results

lib/rspec/core/formatters/bisect_formatter.rb

@@ -21,11 +21,11 @@ def initialize(_output)
           @all_example_ids    = []
           @failed_example_ids = []
           @bisect_server      = DRbObject.new_with_uri(drb_uri)
-          @abort_after_id     = nil
+          @remaining_failures = []
         end
 
         def start(_notification)
-          @abort_after_id = @bisect_server.abort_after_example_id
+          @remaining_failures = Set.new(@bisect_server.expected_failures)
         end
 
         def example_started(notification)
@@ -34,15 +34,15 @@ def example_started(notification)
 
         def example_failed(notification)
           @failed_example_ids << notification.example.id
-          example_finished(notification)
+          example_finished(notification, :failed)
         end
 
         def example_passed(notification)
-          example_finished(notification)
+          example_finished(notification, :passed)
         end
 
         def example_pending(notification)
-          example_finished(notification)
+          example_finished(notification, :pending)
         end
 
         def start_dump(_notification)
@@ -55,8 +55,11 @@ def start_dump(_notification)
 
       private
 
-        def example_finished(notification)
-          return unless notification.example.id == @abort_after_id
+        def example_finished(notification, status)
+          return unless @remaining_failures.include?(notification.example.id)
+          @remaining_failures.delete(notification.example.id)
+
+          return if status == :failed && !@remaining_failures.empty?
           RSpec.world.wants_to_quit = true
         end
       end

lib/rspec/core/set.rb

@@ -16,11 +16,19 @@ def initialize(array=[])
         merge(array)
       end
 
+      def empty?
+        @values.empty?
+      end
+
       def <<(key)
         @values[key] = true
         self
       end
 
+      def delete(key)
+        @values.delete(key)
+      end
+
       def each(&block)
         @values.keys.each(&block)
         self

spec/rspec/core/bisect/runner_spec.rb

@@ -1,10 +1,32 @@
 require 'rspec/core/bisect/runner'
+require 'rspec/core/formatters/bisect_formatter'
 
 module RSpec::Core
   RSpec.describe Bisect::Runner do
     let(:server) { instance_double("RSpec::Core::Bisect::Server", :drb_port => 1234) }
     let(:runner) { described_class.new(server, original_cli_args) }
 
+    describe "#run" do
+      let(:original_cli_args) { %w[ spec/1_spec.rb ] }
+
+      it "passes the failed examples from the original run as the expected failures so the runs can abort early" do
+        original_results = Formatters::BisectFormatter::RunResults.new(
+          [], %w[ spec/failure_spec.rb[1:1] spec/failure_spec.rb[1:2] ]
+        )
+
+        expect(server).to receive(:capture_run_results).
+          with(no_args).
+          ordered.
+          and_return(original_results)
+
+        expect(server).to receive(:capture_run_results).
+          with(original_results.failed_example_ids).
+          ordered
+
+        runner.run(%w[ spec/1_spec.rb[1:1] spec/1_spec.rb[1:2] ])
+      end
+    end
+
     describe "#command_for" do
       def command_for(locations, options={})
         load_path = options.fetch(:load_path) { [] }

spec/rspec/core/bisect/server_spec.rb

@@ -67,24 +67,67 @@ def run_formatter_specs
         )
       end
 
-      it 'can abort the run early (e.g. when it is not interested in later examples)' do
-        results = server.capture_run_results("./spec/rspec/core/resources/formatter_specs.rb[2:2:1]") do
-          run_formatter_specs
-        end
-
-        expect(results).to have_attributes(
-          :all_example_ids => %w[
-            ./spec/rspec/core/resources/formatter_specs.rb[1:1]
-            ./spec/rspec/core/resources/formatter_specs.rb[2:1:1]
-            ./spec/rspec/core/resources/formatter_specs.rb[2:2:1]
-          ],
-          :failed_example_ids => %w[
+      describe "aborting the run early" do
+        it "aborts as soon as the last expected failure finishes, since we don't care about what happens after that" do
+          expected_failures = %w[
             ./spec/rspec/core/resources/formatter_specs.rb[2:2:1]
+            ./spec/rspec/core/resources/formatter_specs.rb[4:1]
           ]
-        )
-      end
 
-      # TODO: test aborting after pending vs failed vs passing example if we keep this feature.
+          results = server.capture_run_results(expected_failures) do
+            run_formatter_specs
+          end
+
+          expect(results).to have_attributes(
+            :all_example_ids => %w[
+              ./spec/rspec/core/resources/formatter_specs.rb[1:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[2:1:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[2:2:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[3:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[4:1]
+            ],
+            :failed_example_ids => %w[
+              ./spec/rspec/core/resources/formatter_specs.rb[2:2:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[4:1]
+            ]
+          )
+        end
+
+        it 'aborts after an expected failure passes instead, even when there are remaining failing examples' do
+          passing_example       = "./spec/rspec/core/resources/formatter_specs.rb[3:1]"
+          later_failing_example = "./spec/rspec/core/resources/formatter_specs.rb[4:1]"
+
+          results = server.capture_run_results([passing_example, later_failing_example]) do
+            run_formatter_specs
+          end
+
+          expect(results).to have_attributes(
+            :all_example_ids => %w[
+              ./spec/rspec/core/resources/formatter_specs.rb[1:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[2:1:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[2:2:1]
+              ./spec/rspec/core/resources/formatter_specs.rb[3:1]
+            ],
+            :failed_example_ids => %w[
+              ./spec/rspec/core/resources/formatter_specs.rb[2:2:1]
+            ]
+          )
+        end
+
+        it 'aborts after an expected failure is pending instead, even when there are remaining failing examples' do
+          pending_example       = "./spec/rspec/core/resources/formatter_specs.rb[1:1]"
+          later_failing_example = "./spec/rspec/core/resources/formatter_specs.rb[4:1]"
+
+          results = server.capture_run_results([pending_example, later_failing_example]) do
+            run_formatter_specs
+          end
+
+          expect(results).to have_attributes(
+            :all_example_ids    => %w[ ./spec/rspec/core/resources/formatter_specs.rb[1:1] ],
+            :failed_example_ids => %w[]
+          )
+        end
+      end
     end
   end
 end

spec/rspec/core/set_spec.rb

@@ -20,4 +20,17 @@
     expect(set).to be_an Enumerable
     expect { |p| set.each(&p) }.to yield_successive_args(1, 2, 3)
   end
+
+  it 'supports deletions' do
+    expect {
+      set.delete(1)
+    }.to change { set.include?(1) }.from(true).to(false)
+  end
+
+  it 'indicates if it is empty' do
+    set = RSpec::Core::Set.new
+    expect {
+      set << 1
+    }.to change { set.empty? }.from(true).to(false)
+  end
 end