Add fail_if_no_examples option

Exit with a failure status if no RSpec examples.
Default behavior not changed. CLI option not implemented.

Author: xmik

features/.nav

@@ -43,6 +43,7 @@
   - read_options_from_file.feature
   - color.feature
   - fail_fast.feature
+  - fail_if_no_examples.feature
   - custom_settings.feature
   - alias_example_to.feature
   - default_path.feature

features/configuration/fail_if_no_examples.feature

@@ -0,0 +1,31 @@
+Feature: fail if no examples
+
+  Use the `fail_if_no_examples` option to make RSpec exit with a failure status (by default 1) if there are no examples. Using this option, it is recommended to add a `--require spec_helper` option to `.rspec` file to ensure the `fail_if_no_examples` option is set even if no spec files are loaded.
+
+  This option may be particularly useful when you happen to not run RSpec tests locally, but rely on CI to do this. This prevents from false positive builds, when you expected some RSpec examples to be run, but none were run. Such a situation may be caused by your misconfiguration or regression/major changes in RSpec.
+
+  Background:
+    Given a file named "spec/spec_helper.rb" with:
+      """ruby
+      RSpec.configure { |c| c.fail_if_no_examples = true }
+      """
+    Given a file named ".rspec" with:
+      """ruby
+      --require spec_helper
+      """
+    Given a file named "spec/some.spec.rb" with:
+      """ruby
+      RSpec.describe 'something' do
+        it 'succeeds' do
+          true
+        end
+      end
+      """
+
+  Scenario: Examples file name is not matched by RSpec pattern, thus there are no examples run
+    When I run `rspec`
+    Then it should fail with "0 examples, 0 failures"
+
+  Scenario: Examples file name is matched by RSpec pattern, 1 example is run
+    When I run `rspec --pattern spec/**/*.spec.rb`
+    Then it should pass with "1 example, 0 failures"

lib/rspec/core/configuration.rb

@@ -199,6 +199,10 @@ def only_failures_but_not_configured?
       # The exit code to return if there are any failures (default: 1).
       add_setting :failure_exit_code
 
+      # @macro add_setting
+      # Whether or not to fail when there are no RSpec examples (default: false).
+      add_setting :fail_if_no_examples
+
       # @macro define_reader
       # Indicates files configured to be required.
       define_reader :requires
@@ -425,6 +429,7 @@ def initialize
         @pattern = '**{,/*/**}/*_spec.rb'
         @exclude_pattern = ''
         @failure_exit_code = 1
+        @fail_if_no_examples = false
         @spec_files_loaded = false
 
         @backtrace_formatter = BacktraceFormatter.new

lib/rspec/core/runner.rb

@@ -108,8 +108,12 @@ def setup(err, out)
       #   or the configured failure exit code (1 by default) if specs
       #   failed.
       def run_specs(example_groups)
-        @configuration.reporter.report(@world.example_count(example_groups)) do |reporter|
+        examples_count = @world.example_count(example_groups)
+        @configuration.reporter.report(examples_count) do |reporter|
           @configuration.with_suite_hooks do
+            if examples_count == 0 && @configuration.fail_if_no_examples
+              return @configuration.failure_exit_code
+            end
             example_groups.map { |g| g.run(reporter) }.all? ? 0 : @configuration.failure_exit_code
           end
         end

spec/integration/fail_if_no_examples_spec.rb

@@ -0,0 +1,108 @@
+require 'support/aruba_support'
+
+RSpec.describe 'Fail if no examples' do
+  include_context "aruba support"
+  before { clean_current_dir }
+
+  context 'when 1 passing example' do
+    def passing_example(fail_if_no_examples)
+      "
+        RSpec.configure { |c| c.fail_if_no_examples = #{fail_if_no_examples} }
+
+        RSpec.describe 'something' do
+          it 'succeeds' do
+            true
+          end
+        end
+      "
+    end
+
+    it 'succeeds if fail_if_no_examples set to true' do
+      write_file 'spec/example_spec.rb', passing_example(true)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 0 failures")
+      expect(last_cmd_exit_status).to eq(0)
+    end
+
+    it 'succeeds if fail_if_no_examples set to false' do
+      write_file 'spec/example_spec.rb', passing_example(false)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 0 failures")
+      expect(last_cmd_exit_status).to eq(0)
+    end
+  end
+
+  context 'when 1 failing example' do
+    def failing_example(fail_if_no_examples)
+      "
+        RSpec.configure { |c| c.fail_if_no_examples = #{fail_if_no_examples} }
+
+        RSpec.describe 'something' do
+          it 'fails' do
+            fail
+          end
+        end
+      "
+    end
+
+    it 'fails if fail_if_no_examples set to true' do
+      write_file 'spec/example_spec.rb', failing_example(true)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 1 failure")
+      expect(last_cmd_exit_status).to eq(1)
+    end
+
+    it 'fails if fail_if_no_examples set to false' do
+      write_file 'spec/example_spec.rb', failing_example(false)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 1 failure")
+      expect(last_cmd_exit_status).to eq(1)
+    end
+  end
+
+  context 'when 0 examples' do
+    def no_examples(fail_if_no_examples)
+      "
+        RSpec.configure { |c| c.fail_if_no_examples = #{fail_if_no_examples} }
+
+        RSpec.describe 'something' do
+        end
+      "
+    end
+
+    it 'fails if fail_if_no_examples set to true' do
+      write_file 'spec/example_spec.rb', no_examples(true)
+      run_command ""
+      expect(last_cmd_stdout).to include("0 examples, 0 failures")
+      expect(last_cmd_exit_status).to eq(1)
+    end
+
+    it 'succeeds if fail_if_no_examples set to false' do
+      write_file 'spec/example_spec.rb', no_examples(false)
+      run_command ""
+      expect(last_cmd_stdout).to include("0 examples, 0 failures")
+      expect(last_cmd_exit_status).to eq(0)
+    end
+
+    context 'when custom failure_exit_code set' do
+      def no_examples_custom_failure_exit_code(fail_if_no_examples)
+        "
+          RSpec.configure do |c|
+            c.fail_if_no_examples = #{fail_if_no_examples}
+            c.failure_exit_code = 15
+          end
+
+          RSpec.describe 'something' do
+          end
+        "
+      end
+
+      it 'fails if fail_if_no_examples set to true' do
+        write_file 'spec/example_spec.rb', no_examples_custom_failure_exit_code(true)
+        run_command ""
+        expect(last_cmd_stdout).to include("0 examples, 0 failures")
+        expect(last_cmd_exit_status).to eq(15)
+      end
+    end
+  end
+end

spec/rspec/core/configuration_spec.rb

@@ -42,6 +42,22 @@ module RSpec::Core
       end
     end
 
+    describe 'fail_if_no_examples' do
+      it 'defaults to false' do
+        expect(RSpec::Core::Configuration.new.fail_if_no_examples).to be(false)
+      end
+
+      it 'can be set to true' do
+        config.fail_if_no_examples = true
+        expect(config.fail_if_no_examples).to eq(true)
+      end
+
+      it 'can be set to false' do
+        config.fail_if_no_examples = false
+        expect(config.fail_if_no_examples).to eq(false)
+      end
+    end
+
     describe '#deprecation_stream' do
       it 'defaults to standard error' do
         expect($rspec_core_without_stderr_monkey_patch.deprecation_stream).to eq STDERR

spec/support/aruba_support.rb

@@ -10,7 +10,7 @@ module ArubaLoader
   let(:stderr) { StringIO.new }
   let(:stdout) { StringIO.new }
 
-  attr_reader :last_cmd_stdout, :last_cmd_stderr
+  attr_reader :last_cmd_stdout, :last_cmd_stderr, :last_cmd_exit_status
 
   def run_command(cmd)
     RSpec.configuration.color = true
@@ -24,7 +24,7 @@ def run_command(cmd)
 
     handle_current_dir_change do
       in_current_dir do
-        RSpec::Core::Runner.run(cmd_parts, temp_stderr, temp_stdout)
+        @last_cmd_exit_status = RSpec::Core::Runner.run(cmd_parts, temp_stderr, temp_stdout)
       end
     end
   ensure