Implement Bisect::ForkRunner.

Author: myronmarston

lib/rspec/core/bisect/fork_runner.rb

@@ -0,0 +1,130 @@
+require 'stringio'
+RSpec::Support.require_rspec_core "formatters/base_bisect_formatter"
+RSpec::Support.require_rspec_core "bisect/utilities"
+
+module RSpec
+  module Core
+    module Bisect
+      # A Bisect runner that runs requested subsets of the suite by forking
+      # sub-processes. The master process bootstraps RSpec and the application
+      # environment (including preloading files specified via `--require`) so
+      # that the individual spec runs do not have to re-pay that cost.  Each
+      # spec run happens in a forked process, ensuring that the spec files are
+      # not loaded in the main process.
+      #
+      # For most projects, bisections that use `ForkRunner` instead of
+      # `ShellRunner` will finish significantly faster, because the `ShellRunner`
+      # pays the cost of booting RSpec and the app environment on _every_ run of
+      # a subset. In contrast, `ForkRunner` pays that cost only once.
+      #
+      # However, not all projects can use `ForkRunner`. Obviously, on platforms
+      # that do not support forking (e.g. Windows), it cannot be used. In addition,
+      # it can cause problems for some projects that put side-effectful spec
+      # bootstrapping logic that should run on every spec run directly at the top
+      # level in a file loaded by `--require`, rather than in a `before(:suite)`
+      # hook. For example, consider a project that relies on some top-level logic
+      # in `spec_helper` to boot a Redis server for the test suite, intending the
+      # Redis bootstrapping to happen on every spec run. With `ShellRunner`, the
+      # bootstrapping logic will happen for each run of any subset of the suite,
+      # but for `ForkRunner`, such logic will only get run once, when the
+      # `RunDispatcher` boots the application environment. This might cause
+      # problems. The solution is for users to move the bootstrapping logic into
+      # a `before(:suite)` hook, or use the slower `ShellRunner`.
+      #
+      # @private
+      class ForkRunner
+        def self.start(shell_command, spec_runner)
+          instance = new(shell_command, spec_runner)
+          yield instance
+        ensure
+          instance.shutdown
+        end
+
+        def initialize(shell_command, spec_runner)
+          @shell_command = shell_command
+          @channel = Channel.new
+          @run_dispatcher = RunDispatcher.new(spec_runner, @channel)
+        end
+
+        def run(locations)
+          run_descriptor = ExampleSetDescriptor.new(locations, original_results.failed_example_ids)
+          dispatch_run(run_descriptor)
+        end
+
+        def original_results
+          @original_results ||= dispatch_run(ExampleSetDescriptor.new(
+            @shell_command.original_locations, []))
+        end
+
+        def shutdown
+          @channel.close
+        end
+
+      private
+
+        def dispatch_run(run_descriptor)
+          @run_dispatcher.dispatch_specs(run_descriptor)
+          @channel.receive.tap do |result|
+            if result.is_a?(String)
+              raise BisectFailedError.for_failed_spec_run(result)
+            end
+          end
+        end
+
+        # @private
+        class RunDispatcher
+          def initialize(runner, channel)
+            @runner = runner
+            @channel = channel
+
+            @spec_output = StringIO.new
+
+            runner.configuration.tap do |c|
+              c.reset_reporter
+              c.output_stream = @spec_output
+              c.error_stream = @spec_output
+            end
+          end
+
+          def dispatch_specs(run_descriptor)
+            pid = fork { run_specs(run_descriptor) }
+            Process.waitpid(pid)
+          end
+
+        private
+
+          def run_specs(run_descriptor)
+            $stdout = $stderr = @spec_output
+            formatter = CaptureFormatter.new(run_descriptor.failed_example_ids)
+
+            @runner.configuration.tap do |c|
+              c.files_or_directories_to_run = run_descriptor.all_example_ids
+              c.formatter = formatter
+              c.load_spec_files
+            end
+
+            # `announce_filters` has the side effect of implementing the logic
+            # that honors `config.run_all_when_everything_filtered` so we need
+            # to call it here. When we remove `run_all_when_everything_filtered`
+            # (slated for RSpec 4), we can remove this call to `announce_filters`.
+            @runner.world.announce_filters
+
+            @runner.run_specs(@runner.world.ordered_example_groups)
+            latest_run_results = formatter.results
+
+            if latest_run_results.nil? || latest_run_results.all_example_ids.empty?
+              @channel.send(@spec_output.string)
+            else
+              @channel.send(latest_run_results)
+            end
+          end
+        end
+
+        class CaptureFormatter < Formatters::BaseBisectFormatter
+          attr_accessor :results
+          alias_method :notify_results, :results=
+        end
+      end
+    end
+  end
+end

lib/rspec/core/bisect/shell_runner.rb

@@ -10,7 +10,7 @@ module Bisect
       # Sets of specs are run by shelling out.
       # @private
       class ShellRunner
-        def self.start(shell_command)
+        def self.start(shell_command, _spec_runner=nil)
           Server.run do |server|
             yield new(server, shell_command)
           end

lib/rspec/core/bisect/utilities.rb

@@ -27,6 +27,32 @@ def publish(event, *args)
           @formatter.__send__(event, notification)
         end
       end
+
+      # Wraps a pipe to support sending objects between a child and
+      # parent process.
+      # @private
+      class Channel
+        def initialize
+          @read_io, @write_io = IO.pipe
+        end
+
+        def send(message)
+          packet = Marshal.dump(message)
+          @write_io.write("#{packet.bytesize}\n#{packet}")
+        end
+
+        # rubocop:disable Security/MarshalLoad
+        def receive
+          packet_size = Integer(@read_io.gets)
+          Marshal.load(@read_io.read(packet_size))
+        end
+        # rubocop:enable Security/MarshalLoad
+
+        def close
+          @read_io.close
+          @write_io.close
+        end
+      end
     end
   end
 end

lib/rspec/core/runner.rb

@@ -94,9 +94,7 @@ def run(err, out)
       # @param err [IO] error stream
       # @param out [IO] output stream
       def setup(err, out)
-        @configuration.error_stream = err
-        @configuration.output_stream = out if @configuration.output_stream == $stdout
-        @options.configure(@configuration)
+        configure(err, out)
         @configuration.load_spec_files
         @world.announce_filters
       end
@@ -122,6 +120,13 @@ def run_specs(example_groups)
         success ? 0 : @configuration.failure_exit_code
       end
 
+      # @private
+      def configure(err, out)
+        @configuration.error_stream = err
+        @configuration.output_stream = out if @configuration.output_stream == $stdout
+        @options.configure(@configuration)
+      end
+
       # @private
       def self.disable_autorun!
         @autorun_disabled = true

spec/integration/bisect_runners_spec.rb

@@ -0,0 +1,133 @@
+require 'support/aruba_support'
+require 'rspec/core/bisect/shell_command'
+require 'rspec/core/bisect/shell_runner'
+require 'rspec/core/bisect/fork_runner'
+
+module RSpec::Core
+  RSpec.shared_examples_for "a bisect runner" do
+    include_context "aruba support"
+    before { clean_current_dir }
+
+    let(:shell_command) { Bisect::ShellCommand.new([]) }
+
+    def with_runner(&block)
+      handle_current_dir_change do
+        in_current_dir do
+          options = ConfigurationOptions.new(shell_command.original_cli_args)
+          runner = Runner.new(options)
+          output = StringIO.new
+          runner.configure(output, output)
+          described_class.start(shell_command, runner, &block)
+        end
+      end
+    end
+
+    it 'runs the specs in an isolated environment and reports the results' do
+      RSpec.configuration.formatter = 'progress'
+
+      write_file 'spec/a_spec.rb', "
+        formatters = RSpec.configuration.formatter_loader.formatters
+        if formatters.any? { |f| f.is_a?(RSpec::Core::Formatters::ProgressFormatter) }
+          raise 'Leaked progress formatter from host environment'
+        end
+
+        RSpec.describe 'A group' do
+          it('passes') { expect(1).to eq 1 }
+          it('fails')  { expect(1).to eq 2 }
+        end
+      "
+
+      with_runner do |runner|
+        expect(runner.original_results).to have_attributes(
+          :all_example_ids => %w[ ./spec/a_spec.rb[1:1] ./spec/a_spec.rb[1:2] ],
+          :failed_example_ids => %w[ ./spec/a_spec.rb[1:2] ]
+        )
+
+        expect(runner.run(%w[ ./spec/a_spec.rb[1:1] ])).to have_attributes(
+          :all_example_ids => %w[ ./spec/a_spec.rb[1:1] ],
+          :failed_example_ids => %w[]
+        )
+      end
+    end
+
+    it 'honors `run_all_when_everything_filtered`' do
+      write_file 'spec/a_spec.rb', "
+        RSpec.configure do |c|
+          c.filter_run :focus
+          c.run_all_when_everything_filtered = true
+        end
+
+        RSpec.describe 'A group' do
+          it('passes') { expect(1).to eq 1 }
+          it('fails')  { expect(1).to eq 2 }
+        end
+      "
+
+      with_runner do |runner|
+        expect(runner.original_results).to have_attributes(
+          :all_example_ids => %w[ ./spec/a_spec.rb[1:1] ./spec/a_spec.rb[1:2] ],
+          :failed_example_ids => %w[ ./spec/a_spec.rb[1:2] ]
+        )
+      end
+    end
+
+    it 'raises BisectFailedError with all run output when it encounters an error loading spec files' do
+      write_file 'spec/a_spec.rb', "
+        puts 'stdout in a_spec'
+        warn 'stderr in a_spec'
+
+        RSpec.escribe 'A group' do
+          it('passes') { expect(1).to eq 1 }
+          it('fails')  { expect(1).to eq 2 }
+        end
+      "
+
+      with_runner do |runner|
+        expect {
+          runner.original_results
+        }.to raise_error(Bisect::BisectFailedError, a_string_including(
+          "undefined method `escribe' for RSpec:Module",
+          'stdout in a_spec',
+          'stderr in a_spec'
+        ))
+      end
+    end
+  end
+
+  RSpec.describe Bisect::ShellRunner, :slow do
+    include_examples 'a bisect runner'
+  end
+
+  RSpec.describe Bisect::ForkRunner, :if => Process.respond_to?(:fork) do
+    include_examples 'a bisect runner'
+
+    context 'when a `--require` option has been provided' do
+      let(:shell_command) { Bisect::ShellCommand.new(['--require', './spec/a_spec_helper']) }
+
+      it 'loads the specified file only once (rather than once per subset run)' do
+        write_file 'spec_helper_loads', ''
+        write_file 'spec/a_spec_helper.rb', "
+          File.open('spec_helper_loads', 'a') do |f|
+            f.print('.')
+          end
+        "
+
+        write_file 'spec/a_spec.rb', "
+          RSpec.describe 'A group' do
+            it('passes') { expect(1).to eq 1 }
+            it('fails')  { expect(1).to eq 2 }
+          end
+        "
+
+        with_runner do |runner|
+          runner.run(%w[ ./spec/a_spec.rb[1:1] ])
+          runner.run(%w[ ./spec/a_spec.rb[1:1] ])
+        end
+
+        in_current_dir do
+          expect(File.read('spec_helper_loads')).to eq(".")
+        end
+      end
+    end
+  end
+end

spec/rspec/core/bisect/utilities_spec.rb

@@ -23,4 +23,18 @@ def foo(notification); end
       }.not_to raise_error
     end
   end
+
+  RSpec.describe Bisect::Channel do
+    include RSpec::Support::InSubProcess
+
+    it "supports sending objects from a child process back to the parent" do
+      channel = Bisect::Channel.new
+
+      in_sub_process do
+        channel.send(:value_from_child)
+      end
+
+      expect(channel.receive).to eq :value_from_child
+    end
+  end
 end