Allow different bisect runners to be configured and used.

Author: myronmarston

features/command_line/bisect.feature

@@ -94,7 +94,7 @@ Feature: Bisect
     When I run `rspec --seed 1234 --bisect=verbose`
     Then bisect should succeed with output like:
       """
-      Bisect started using options: "--seed 1234"
+      Bisect started using options: "--seed 1234" and bisect runner: RSpec::Core::Bisect::ForkRunner
       Running suite to find failures... (0.16528 seconds)
        - Failing examples (1):
           - ./spec/calculator_1_spec.rb[1:1]
@@ -156,3 +156,23 @@ Feature: Bisect
       """
     When I run `rspec ./spec/calculator_10_spec.rb[1:1] ./spec/calculator_1_spec.rb[1:1] --seed 1234`
     Then the output should contain "2 examples, 1 failure"
+
+  Scenario: Pick a bisect runner via a config option
+    Given a file named "spec/spec_helper.rb" with:
+      """
+      RSpec.configure do |c|
+        c.bisect_runner = :shell
+      end
+      """
+    And a file named ".rspec" with:
+      """
+      --require spec_helper
+      """
+    When I run `rspec --seed 1234 --bisect=verbose`
+    Then bisect should succeed with output like:
+      """
+      Bisect started using options: "--seed 1234" and bisect runner: RSpec::Core::Bisect::ShellRunner
+      # ...
+      The minimal reproduction command is:
+        rspec ./spec/calculator_10_spec.rb[1:1] ./spec/calculator_1_spec.rb[1:1] --seed 1234
+      """

features/step_definitions/additional_cli_steps.rb

@@ -180,9 +180,14 @@
     "Output:\n\n#{last_process.stdout}"
 
   expected = normalize_durations(expected_output)
-  actual   = normalize_durations(last_process.stdout)
+  actual   = normalize_durations(last_process.stdout).sub(/\n+\Z/, '')
 
-  expect(actual.sub(/\n+\Z/, '')).to eq(expected)
+  if expected.include?("# ...")
+    expected_start, expected_end = expected.split("# ...")
+    expect(actual).to start_with(expected_start).and end_with(expected_end)
+  else
+    expect(actual).to eq(expected)
+  end
 end
 
 When(/^I run `([^`]+)` and abort in the middle with ctrl\-c$/) do |cmd|

lib/rspec/core/bisect/coordinator.rb

@@ -1,31 +1,30 @@
 RSpec::Support.require_rspec_core "bisect/shell_command"
-RSpec::Support.require_rspec_core "bisect/shell_runner"
 RSpec::Support.require_rspec_core "bisect/example_minimizer"
 RSpec::Support.require_rspec_core "bisect/utilities"
 RSpec::Support.require_rspec_core "formatters/bisect_progress_formatter"
 
 module RSpec
   module Core
     module Bisect
-      # @private
       # The main entry point into the bisect logic. Coordinates among:
       #   - Bisect::ShellCommand: Generates shell commands to run spec subsets
-      #   - Bisect::ShellRunner: Runs a set of examples and returns the results.
       #   - Bisect::ExampleMinimizer: Contains the core bisect logic.
-      #   - Formatters::BisectProgressFormatter: provides progress updates
-      #     to the user.
+      #   - A bisect runner: runs a set of examples and returns the results.
+      #   - A bisect formatter: provides progress updates to the user.
+      # @private
       class Coordinator
-        def self.bisect_with(original_cli_args, formatter)
-          new(original_cli_args, formatter).bisect
+        def self.bisect_with(spec_runner, original_cli_args, formatter)
+          new(spec_runner, original_cli_args, formatter).bisect
         end
 
-        def initialize(original_cli_args, formatter)
+        def initialize(spec_runner, original_cli_args, formatter)
+          @spec_runner   = spec_runner
           @shell_command = ShellCommand.new(original_cli_args)
           @notifier      = Bisect::Notifier.new(formatter)
         end
 
         def bisect
-          repro = ShellRunner.start(@shell_command) do |runner|
+          repro = start_bisect_runner do |runner|
             minimizer = ExampleMinimizer.new(@shell_command, runner, @notifier)
 
             gracefully_abort_on_sigint(minimizer)
@@ -45,6 +44,11 @@ def bisect
 
       private
 
+        def start_bisect_runner(&block)
+          klass = @spec_runner.configuration.bisect_runner_class
+          klass.start(@shell_command, @spec_runner, &block)
+        end
+
         def gracefully_abort_on_sigint(minimizer)
           trap('INT') do
             repro = minimizer.repro_command_for_currently_needed_ids

lib/rspec/core/bisect/example_minimizer.rb

@@ -111,7 +111,8 @@ def example_range(ids)
         end
 
         def prep
-          notify(:bisect_starting, :original_cli_args => shell_command.original_cli_args)
+          notify(:bisect_starting, :original_cli_args => shell_command.original_cli_args,
+                                   :bisect_runner => runner.class)
 
           _, duration = track_duration do
             original_results    = runner.original_results

lib/rspec/core/bisect/shell_runner.rb

@@ -10,7 +10,7 @@ module Bisect
       # Sets of specs are run by shelling out.
       # @private
       class ShellRunner
-        def self.start(shell_command, _spec_runner=nil)
+        def self.start(shell_command, _spec_runner)
           Server.run do |server|
             yield new(server, shell_command)
           end

lib/rspec/core/configuration.rb

@@ -413,6 +413,28 @@ def shared_context_metadata_behavior=(value)
       # return [Integer]
       add_setting :max_displayed_failure_line_count
 
+      # @macro add_setting
+      # Determines which bisect runner implementation gets used to run subsets
+      # of the suite during a bisection. Your choices are:
+      #
+      #   - `:shell`: Performs a spec run by shelling out, booting RSpec and your
+      #     application environment each time. This runner is the most widely
+      #     compatible runner, but is not as fast. On platforms that do not
+      #     support forking, this is the default.
+      #   - `:fork`: Pre-boots RSpec and your application environment in a parent
+      #     process, and then forks a child process for each spec run. This runner
+      #     tends to be significantly faster than the `:shell` runner but cannot
+      #     be used in some situations. On platforms that support forking, this
+      #     is the default. If you use this runner, you should ensure that all
+      #     of your one-time setup logic goes in a `before(:suite)` hook instead
+      #     of getting run at the top-level of a file loaded by `--require`.
+      #
+      # @note This option will only be used by `--bisect` if you set it in a file
+      #   loaded via `--require`.
+      #
+      # @return [Symbol]
+      add_setting :bisect_runner
+
       # @private
       # @deprecated Use {#color_mode} = :on, instead of {#color} with {#tty}
       add_setting :tty
@@ -437,6 +459,8 @@ def initialize
         @extend_modules  = FilterableItemRepository::QueryOptimized.new(:any?)
         @prepend_modules = FilterableItemRepository::QueryOptimized.new(:any?)
 
+        @bisect_runner = Process.respond_to?(:fork) ? :fork : :shell
+
         @before_suite_hooks = []
         @after_suite_hooks  = []
 
@@ -1962,6 +1986,21 @@ def on_example_group_definition_callbacks
         @on_example_group_definition_callbacks ||= []
       end
 
+      # @private
+      def bisect_runner_class
+        case bisect_runner
+        when :fork
+          RSpec::Support.require_rspec_core 'bisect/fork_runner'
+          Bisect::ForkRunner
+        when :shell
+          RSpec::Support.require_rspec_core 'bisect/shell_runner'
+          Bisect::ShellRunner
+        else
+          raise "Unsupported value for `bisect_runner` (#{bisect_runner.inspect}). " \
+                "Only `:fork` and `:shell` are supported."
+        end
+      end
+
     private
 
       def load_file_handling_errors(method, file)

lib/rspec/core/formatters/bisect_progress_formatter.rb

@@ -8,8 +8,7 @@ module Formatters
       class BisectProgressFormatter < BaseTextFormatter
         def bisect_starting(notification)
           @round_count = 0
-          options = notification.original_cli_args.join(' ')
-          output.puts "Bisect started using options: #{options.inspect}"
+          output.puts bisect_started_message(notification)
           output.print "Running suite to find failures..."
         end
 
@@ -76,6 +75,13 @@ def bisect_aborted(notification)
           output.puts "\n\nBisect aborted!"
           output.puts "\nThe most minimal reproduction command discovered so far is:\n  #{notification.repro}"
         end
+
+      private
+
+        def bisect_started_message(notification)
+          options = notification.original_cli_args.join(' ')
+          "Bisect started using options: #{options.inspect}"
+        end
       end
 
       # @private
@@ -126,6 +132,10 @@ def describe_ids(description, ids)
           formatted_ids = organized_ids.map { |id| "    - #{id}" }.join("\n")
           "#{description} (#{ids.size}):\n#{formatted_ids}"
         end
+
+        def bisect_started_message(notification)
+          "#{super} and bisect runner: #{notification.bisect_runner}"
+        end
       end
     end
   end

lib/rspec/core/invocations.rb

@@ -26,27 +26,28 @@ def call(options, err, out)
 
       # @private
       class Bisect
-        def call(options, _err, _out)
+        def call(options, err, out)
           RSpec::Support.require_rspec_core "bisect/coordinator"
 
           success = RSpec::Core::Bisect::Coordinator.bisect_with(
+            Runner.new(options).tap { |r| r.configure(err, out) },
             options.args,
-            bisect_formatter_for(options.options[:bisect])
+            bisect_formatter_for(options.options[:bisect], out)
           )
 
           success ? 0 : 1
         end
 
-        private
+      private
 
-        def bisect_formatter_for(argument)
+        def bisect_formatter_for(argument, output)
           klass = if argument == "verbose"
                     Formatters::BisectDebugFormatter
                   else
                     Formatters::BisectProgressFormatter
                   end
 
-          klass.new(RSpec.configuration.output_stream)
+          klass.new(output)
         end
       end
 

spec/integration/bisect_spec.rb

@@ -9,10 +9,10 @@ module RSpec::Core
     end if ENV['APPVEYOR'] && RUBY_VERSION.to_f > 2.0
 
     def bisect(cli_args, expected_status=nil)
-      RSpec.configuration.output_stream = formatter_output
+      options = ConfigurationOptions.new(cli_args)
 
       expect {
-        status = RSpec::Core::Runner.run(cli_args + ["--bisect"])
+        status = Invocations::Bisect.new.call(options, formatter_output, formatter_output)
         expect(status).to eq(expected_status) if expected_status
       }.to avoid_outputting.to_stdout_from_any_process.and avoid_outputting.to_stderr_from_any_process
 

spec/rspec/core/bisect/coordinator_spec.rb

@@ -6,7 +6,9 @@ module RSpec::Core
   RSpec.describe Bisect::Coordinator, :simulate_shell_allowing_unquoted_ids do
     include FormatterSupport
 
-    let(:fake_runner) do
+    let(:config) { instance_double(Configuration, :bisect_runner_class => fake_bisect_runner) }
+    let(:spec_runner) { instance_double(RSpec::Core::Runner, :configuration => config) }
+    let(:fake_bisect_runner) do
       FakeBisectRunner.new(
         1.upto(8).map { |i| "#{i}.rb[1:1]" },
         %w[ 2.rb[1:1] ],
@@ -15,12 +17,7 @@ module RSpec::Core
     end
 
     def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
-      allow(Bisect::Server).to receive(:run).and_yield(instance_double(Bisect::Server))
-      allow(Bisect::ShellRunner).to receive(:new).and_return(fake_runner)
-
-      Bisect::Coordinator.bisect_with([], formatter.new(output))
-    ensure
-      RSpec.reset # so that RSpec.configuration.output_stream isn't closed
+      Bisect::Coordinator.bisect_with(spec_runner, [], formatter.new(output))
     end
 
     it 'notifies the bisect progress formatter of progress and closes the output' do
@@ -52,7 +49,7 @@ def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
       output = normalize_durations(output.string)
 
       expect(output).to eq(<<-EOS.gsub(/^\s+\|/, ''))
-        |Bisect started using options: ""
+        |Bisect started using options: "" and bisect runner: FakeBisectRunner
         |Running suite to find failures... (n.nnnn seconds)
         | - Failing examples (2):
         |    - 2.rb[1:1]
@@ -100,7 +97,7 @@ def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
 
     context "with an order-independent failure" do
       it "detects the independent case and prints the minimal reproduction" do
-        fake_runner.dependent_failures = {}
+        fake_bisect_runner.dependent_failures = {}
         output = StringIO.new
         find_minimal_repro(output)
         output = normalize_durations(output.string)
@@ -120,13 +117,13 @@ def find_minimal_repro(output, formatter=Formatters::BisectProgressFormatter)
       end
 
       it "can use the debug formatter for detailed output" do
-        fake_runner.dependent_failures = {}
+        fake_bisect_runner.dependent_failures = {}
         output = StringIO.new
         find_minimal_repro(output, Formatters::BisectDebugFormatter)
         output = normalize_durations(output.string)
 
         expect(output).to eq(<<-EOS.gsub(/^\s+\|/, ''))
-          |Bisect started using options: ""
+          |Bisect started using options: "" and bisect runner: FakeBisectRunner
           |Running suite to find failures... (n.nnnn seconds)
           | - Failing examples (1):
           |    - 2.rb[1:1]

spec/rspec/core/configuration_spec.rb

@@ -1443,6 +1443,25 @@ def metadata_hash(*args)
       end
     end
 
+    describe "#bisect_runner_class" do
+      it "returns `Bisect::ForkRunner` when `bisect_runner == :fork" do
+        config.bisect_runner = :fork
+        expect(config.bisect_runner_class).to be Bisect::ForkRunner
+      end
+
+      it "returns `Bisect::ShellRunner` when `bisect_runner == :shell" do
+        config.bisect_runner = :shell
+        expect(config.bisect_runner_class).to be Bisect::ShellRunner
+      end
+
+      it "raises a clear error when `bisect_runner` is configured to an unrecognized value" do
+        config.bisect_runner = :unknown
+        expect {
+          config.bisect_runner_class
+        }.to raise_error(/Unsupported value for `bisect_runner`/)
+      end
+    end
+
     %w[formatter= add_formatter].each do |config_method|
       describe "##{config_method}" do
         it "delegates to formatters#add" do

spec/rspec/core/invocations_spec.rb

@@ -77,21 +77,20 @@ def run_invocation
     end
 
     describe Invocations::Bisect do
-      let(:bisect) { nil }
-      let(:options) { { :bisect => bisect } }
-      let(:args) { double(:args) }
+      let(:original_cli_args) { %w[--bisect --seed 1234] }
+      let(:configuration_options) { ConfigurationOptions.new(original_cli_args) }
       let(:success) { true }
 
       before do
-        allow(configuration_options).to receive_messages(:args => args, :options => options)
         allow(RSpec::Core::Bisect::Coordinator).to receive(:bisect_with).and_return(success)
       end
 
       it "starts the bisection coordinator" do
         run_invocation
 
         expect(RSpec::Core::Bisect::Coordinator).to have_received(:bisect_with).with(
-          args,
+          an_instance_of(Runner),
+          configuration_options.args,
           an_instance_of(Formatters::BisectProgressFormatter)
         )
       end
@@ -115,13 +114,14 @@ def run_invocation
       end
 
       context "and the verbose option is specified" do
-        let(:bisect) { "verbose" }
+        let(:original_cli_args) { %w[--bisect=verbose --seed 1234] }
 
         it "starts the bisection coordinator with the debug formatter" do
           run_invocation
 
           expect(RSpec::Core::Bisect::Coordinator).to have_received(:bisect_with).with(
-            args,
+            an_instance_of(Runner),
+            configuration_options.args,
             an_instance_of(Formatters::BisectDebugFormatter)
           )
         end