Add error-exit-code to differentiate from failures

it can be helpful to know if RSpec fails because of an example or if it
errors out because it couldn't load a spec file, or there was an issue
in the a before(:suite) hook or etc

i've named the setting error-exit-code, and tried to add it to all the
relevant places, but i don't know rspec codebase well so there was some
guessing. in particular i'm not sure what bisect should do.

i have it fall back to the failure-exit-code before defaulting to 1 so
there's no changes if people don't opt in to the setting.

the specific use of this is our CI automatically retries failures using
the persistence file, and assumes if they pass on that retry they were
flaky. however, the persistence file isn't written to when there's an
error outside of examples, so this could mean falsely passing builds.

by checking for a different exit code, and then not running the retry we
can avoid this issue.

Author: robotdana

features/configuration/error_exit_code.feature

@@ -0,0 +1,52 @@
+Feature: error exit code
+
+  Use the `error_exit_code` option to set a custom exit code when RSpec fails outside an example.
+
+  ```ruby
+  RSpec.configure { |c| c.error_exit_code = 42 }
+  ```
+
+  Background:
+    Given a file named "spec/spec_helper.rb" with:
+      """ruby
+      RSpec.configure { |c| c.error_exit_code = 42 }
+      """
+
+  Scenario: A erroring spec with the default exit code
+    Given a file named "spec/example_spec.rb" with:
+      """ruby
+      RSpec.escribe "something" do # intentional typo
+        it "works" do
+          true
+        end
+      end
+      """
+    When I run `rspec spec/example_spec.rb`
+    Then the exit status should be 1
+
+  Scenario: A erroring spec with a custom exit code
+    Given a file named "spec/example_spec.rb" with:
+      """ruby
+      require 'spec_helper'
+      RSpec.escribe "something" do # intentional typo
+        it "works" do
+          true
+        end
+      end
+      """
+    When I run `rspec spec/example_spec.rb`
+    And  the exit status should be 42
+
+
+  Scenario: Success running specs spec with a custom error exit code defined
+    Given a file named "spec/example_spec.rb" with:
+      """ruby
+      require 'spec_helper'
+      RSpec.describe "something" do
+        it "works" do
+          true
+        end
+      end
+      """
+    When I run `rspec spec/example_spec.rb`
+    Then the exit status should be 0

features/configuration/failure_exit_code.feature

@@ -37,6 +37,32 @@ Feature: failure exit code
     When I run `rspec spec/example_spec.rb`
     Then the exit status should be 42
 
+  Scenario: An error running specs spec with a custom exit code
+    Given a file named "spec/example_spec.rb" with:
+      """ruby
+      require 'spec_helper'
+      RSpec.escribe "something" do # intentional typo
+        it "works" do
+          true
+        end
+      end
+      """
+    When I run `rspec spec/example_spec.rb`
+    Then the exit status should be 42
+
+  Scenario: Success running specs spec with a custom exit code defined
+    Given a file named "spec/example_spec.rb" with:
+      """ruby
+      require 'spec_helper'
+      RSpec.describe "something" do
+        it "works" do
+          true
+        end
+      end
+      """
+    When I run `rspec spec/example_spec.rb`
+    Then the exit status should be 0
+
   Scenario: Exit with the default exit code when an `at_exit` hook is added upstream
     Given a file named "exit_at_spec.rb" with:
       """ruby

lib/rspec/core/configuration.rb

@@ -242,6 +242,11 @@ def fail_fast=(value)
       # @return [Integer]
       add_setting :failure_exit_code
 
+      # @macro add_setting
+      # The exit code to return if there are any errors outside examples (default: failure_exit_code)
+      # @return [Integer]
+      add_setting :error_exit_code
+
       # @macro add_setting
       # Whether or not to fail when there are no RSpec examples (default: false).
       # @return [Boolean]
@@ -523,6 +528,7 @@ def initialize
         @pattern = '**{,/*/**}/*_spec.rb'
         @exclude_pattern = ''
         @failure_exit_code = 1
+        @error_exit_code = nil # so it can be overrid by failure exit code
         @fail_if_no_examples = false
         @spec_files_loaded = false
 
@@ -568,6 +574,18 @@ def force(hash)
         clear_values_derived_from_example_status_persistence_file_path
       end
 
+      # @private
+      # @return [Integer]
+      def failure_or_error_exit_code(local_world=world)
+        if local_world.non_example_failure && error_exit_code
+          error_exit_code.to_i
+        elsif failure_exit_code
+          failure_exit_code.to_i
+        else
+          1
+        end
+      end
+
       # @private
       def reset
         @spec_files_loaded = false

lib/rspec/core/drb.rb

@@ -51,6 +51,7 @@ def options
         argv << "--order"        << @submitted_options[:order]               if @submitted_options[:order]
 
         add_failure_exit_code(argv)
+        add_error_exit_code(argv)
         add_full_description(argv)
         add_filter(argv, :inclusion, @filter_manager.inclusions)
         add_filter(argv, :exclusion, @filter_manager.exclusions)
@@ -67,6 +68,12 @@ def add_failure_exit_code(argv)
         argv << "--failure-exit-code" << @submitted_options[:failure_exit_code].to_s
       end
 
+      def add_error_exit_code(argv)
+        return unless @submitted_options[:error_exit_code]
+
+        argv << "--error-exit-code" << @submitted_options[:error_exit_code].to_s
+      end
+
       def add_full_description(argv)
         return unless @submitted_options[:full_description]
 

lib/rspec/core/invocations.rb

@@ -37,7 +37,7 @@ def call(options, err, out)
             runner, options.args, formatter
           )
 
-          success ? 0 : runner.configuration.failure_exit_code
+          success ? 0 : runner.configuration.failure_or_error_exit_code
         end
 
       private

lib/rspec/core/option_parser.rb

@@ -95,6 +95,11 @@ def parser(options)
           options[:failure_exit_code] = code
         end
 
+        parser.on('--error-exit-code CODE', Integer,
+                  'Override the exit code used when there are errors loading or running specs outside of examples.') do |code|
+          options[:error_exit_code] = code
+        end
+
         parser.on('-X', '--[no-]drb', 'Run examples via DRb.') do |use_drb|
           options[:drb] = use_drb
           options[:runner] = RSpec::Core::Invocations::DRbWithFallback.new if use_drb

lib/rspec/core/runner.rb

@@ -84,7 +84,9 @@ def initialize(options, configuration=RSpec.configuration, world=RSpec.world)
       # @param out [IO] output stream
       def run(err, out)
         setup(err, out)
-        return @configuration.reporter.exit_early(@configuration.failure_exit_code) if RSpec.world.wants_to_quit
+        if RSpec.world.wants_to_quit
+          return @configuration.reporter.exit_early(@configuration.failure_or_error_exit_code)
+        end
 
         run_specs(@world.ordered_example_groups).tap do
           persist_example_statuses
@@ -122,7 +124,7 @@ def run_specs(example_groups)
           end
         end && !@world.non_example_failure
 
-        success ? 0 : @configuration.failure_exit_code
+        success ? 0 : @configuration.failure_or_error_exit_code(@world)
       end
 
       # @private

spec/integration/spec_file_load_errors_spec.rb

@@ -6,6 +6,7 @@
   include FormatterSupport
 
   let(:failure_exit_code) { rand(97) + 2 } # 2..99
+  let(:error_exit_code) { failure_exit_code + 1 } # 3..100
 
   if RSpec::Support::Ruby.jruby_9000?
     let(:spec_line_suffix) { ":in `<main>'" }
@@ -24,14 +25,15 @@
       c.filter_gems_from_backtrace "gems/aruba"
       c.backtrace_exclusion_patterns << %r{/rspec-core/spec/} << %r{rspec_with_simplecov}
       c.failure_exit_code = failure_exit_code
+      c.error_exit_code = error_exit_code
     end
   end
 
   it 'nicely handles load-time errors from --require files' do
     write_file_formatted "helper_with_error.rb", "raise 'boom'"
 
     run_command "--require ./helper_with_error"
-    expect(last_cmd_exit_status).to eq(failure_exit_code)
+    expect(last_cmd_exit_status).to eq(error_exit_code)
     output = normalize_durations(last_cmd_stdout)
     expect(output).to eq unindent(<<-EOS)
 
@@ -60,7 +62,7 @@
     "
 
     run_command "--require ./helper_with_error 1_spec.rb"
-    expect(last_cmd_exit_status).to eq(failure_exit_code)
+    expect(last_cmd_exit_status).to eq(error_exit_code)
     output = normalize_durations(last_cmd_stdout)
     expect(output).to eq unindent(<<-EOS)
 
@@ -109,7 +111,7 @@
     "
 
     run_command "1_spec.rb 2_spec.rb 3_spec.rb"
-    expect(last_cmd_exit_status).to eq(failure_exit_code)
+    expect(last_cmd_exit_status).to eq(error_exit_code)
     output = normalize_durations(last_cmd_stdout)
     expect(output).to eq unindent(<<-EOS)
 

spec/integration/suite_hooks_errors_spec.rb

@@ -6,6 +6,7 @@
   include FormatterSupport
 
   let(:failure_exit_code) { rand(97) + 2 } # 2..99
+  let(:error_exit_code) { failure_exit_code + 2 } # 4..101
 
   if RSpec::Support::Ruby.jruby_9000?
     let(:spec_line_suffix) { ":in `block in (root)'" }
@@ -24,6 +25,7 @@
       c.filter_gems_from_backtrace "gems/aruba"
       c.backtrace_exclusion_patterns << %r{/rspec-core/spec/} << %r{rspec_with_simplecov}
       c.failure_exit_code = failure_exit_code
+      c.error_exit_code = error_exit_code
     end
   end
 
@@ -41,7 +43,7 @@ def run_spec_expecting_non_zero(before_or_after)
     "
 
     run_command "the_spec.rb"
-    expect(last_cmd_exit_status).to eq(failure_exit_code)
+    expect(last_cmd_exit_status).to eq(error_exit_code)
     normalize_durations(last_cmd_stdout)
   end
 
@@ -96,7 +98,7 @@ def run_spec_expecting_non_zero(before_or_after)
     "
 
     run_command "the_spec.rb"
-    expect(last_cmd_exit_status).to eq(failure_exit_code)
+    expect(last_cmd_exit_status).to eq(error_exit_code)
     output = normalize_durations(last_cmd_stdout)
 
     expect(output).to eq unindent(<<-EOS)

spec/rspec/core/configuration_options_spec.rb

@@ -321,6 +321,18 @@
     end
   end
 
+  describe "--error-exit-code" do
+    it "sets :error_exit_code" do
+      expect(parse_options('--error-exit-code', '0')).to include(:error_exit_code => 0)
+      expect(parse_options('--error-exit-code', '1')).to include(:error_exit_code => 1)
+      expect(parse_options('--error-exit-code', '2')).to include(:error_exit_code => 2)
+    end
+
+    it "overrides previous :error_exit_code" do
+      expect(parse_options('--error-exit-code', '2', '--error-exit-code', '3')).to include(:error_exit_code => 3)
+    end
+  end
+
   describe "--dry-run" do
     it "defaults to nil" do
       expect(parse_options[:dry_run]).to be(nil)

spec/rspec/core/configuration_spec.rb

@@ -2862,6 +2862,66 @@ def emulate_not_configured_expectation_framework
       end
     end
 
+    describe '#failure_exit_code' do
+      it 'defaults to 1' do
+        expect(config.failure_exit_code).to eq 1
+      end
+
+      it 'is configurable' do
+        config.failure_exit_code = 2
+        expect(config.failure_exit_code).to eq 2
+      end
+    end
+
+    describe '#error_exit_code' do
+      it 'defaults to nil' do
+        expect(config.error_exit_code).to eq nil
+      end
+
+      it 'is configurable' do
+        config.error_exit_code = 2
+        expect(config.error_exit_code).to eq 2
+      end
+    end
+
+    describe '#failure_or_error_exit_code' do
+      it 'defaults to 1' do
+        expect(config.failure_or_error_exit_code).to eq 1
+      end
+
+      it 'is failure_exit_code by default' do
+        config.failure_exit_code = 2
+        expect(config.failure_or_error_exit_code).to eq 2
+      end
+
+      it 'is failure_exit_code when world is errored by default' do
+        config.world.non_example_failure = true
+        config.failure_exit_code = 2
+        expect(config.failure_or_error_exit_code).to eq 2
+      end
+
+      it 'is error_exit_code when world is errored by and both are defined' do
+        config.world.non_example_failure = true
+        config.failure_exit_code = 2
+        config.error_exit_code = 3
+        expect(config.failure_or_error_exit_code).to eq 3
+      end
+
+      it 'is error_exit_code when world is errored by and failure exit code is not defined' do
+        config.world.non_example_failure = true
+        config.error_exit_code = 3
+        expect(config.failure_or_error_exit_code).to eq 3
+      end
+
+      it 'can be given a world to use instead of config.world' do
+        other_world = World.new
+        other_world.non_example_failure = true
+        config.world.non_example_failure = false
+        config.error_exit_code = 3
+        expect(config.failure_or_error_exit_code(other_world)).to eq 3
+      end
+    end
+
     describe "#shared_context_metadata_behavior" do
       it "defaults to :trigger_inclusion for backwards compatibility" do
         expect(config.shared_context_metadata_behavior).to eq :trigger_inclusion