Add fail_if_no_examples option: better tests

Author: xmik

features/.nav

@@ -43,6 +43,7 @@
   - read_options_from_file.feature
   - color.feature
   - fail_fast.feature
+  - fail_if_no_examples.feature
   - custom_settings.feature
   - alias_example_to.feature
   - default_path.feature

features/configuration/fail_if_no_examples.feature

@@ -1,106 +1,46 @@
 Feature: fail if no examples
 
-  Use the `fail_if_no_examples` option to make RSpec return non zero exit status
-   when there are 0 examples.
-
-  Scenario: A failing spec example and `fail_if_no_examples` set to true
+  Use the `fail_if_no_examples` option to make RSpec exit with exit status 1
+  if there are 0 examples. Using this option, it is recommended to add:
+  a `--require spec_helper` option to `.rspec` file to ensure the `fail_if_no_examples`
+  option is set even if no spec files are loaded.
+
+  This option may be particularly useful when you happen to not run RSpec tests
+  locally, but rely on CI to do this. This prevents from false positive builds,
+  when you expected some RSpec examples to be run, but none were run. Such
+   a situation may be caused by your misconfiguration or regression/major changes
+  in RSpec.
+
+  Background:
     Given a file named "spec/spec_helper.rb" with:
       """ruby
       RSpec.configure { |c| c.fail_if_no_examples = true }
       """
-    Given a file named "spec/example_spec.rb" with:
-      """ruby
-      require 'spec_helper'
-
-      RSpec.describe "something" do
-        it "fails" do
-          fail
-        end
-      end
-      """
-    When I run `rspec spec/example_spec.rb`
-    Then the exit status should be 1
-
-  Scenario: A failing spec example and `fail_if_no_examples` set to false
-    Given a file named "spec/spec_helper.rb" with:
-      """ruby
-      RSpec.configure { |c| c.fail_if_no_examples = false }
-      """
-    Given a file named "spec/example_spec.rb" with:
+    Given a file named ".rspec" with:
       """ruby
-      require 'spec_helper'
-
-      RSpec.describe "something" do
-        it "fails" do
-          fail
-        end
-      end
-      """
-    When I run `rspec spec/example_spec.rb`
-    Then the exit status should be 1
-
-  Scenario: A passing spec example and `fail_if_no_examples` set to true
-    Given a file named "spec/spec_helper.rb" with:
-      """ruby
-      RSpec.configure { |c| c.fail_if_no_examples = true }
+      --require spec_helper
       """
-    Given a file named "spec/example_spec.rb" with:
+    Given a file named "spec/some.spec.rb" with:
       """ruby
-      require 'spec_helper'
-
-      RSpec.describe "something" do
-        it "succeeds" do
+      RSpec.describe 'something' do
+        it 'succeeds' do
           true
         end
       end
       """
-    When I run `rspec spec/example_spec.rb`
-    Then the exit status should be 0
 
-  Scenario: A passing spec example and `fail_if_no_examples` set to false
-    Given a file named "spec/spec_helper.rb" with:
-      """ruby
-      RSpec.configure { |c| c.fail_if_no_examples = false }
-      """
-    Given a file named "spec/example_spec.rb" with:
-      """ruby
-      require 'spec_helper'
+  Scenario: Examples file has invalid name, thus there are no examples run
+    When I run `rspec`
+    Then it should fail with "0 examples, 0 failures"
 
-      RSpec.describe "something" do
-        it "succeeds" do
+  Scenario: Examples file has valid name and 1 example is run
+    Given a file named "spec/some_spec.rb" with:
+      """ruby
+      RSpec.describe 'something' do
+        it 'succeeds' do
           true
         end
       end
       """
-    When I run `rspec spec/example_spec.rb`
-    Then the exit status should be 0
-
-  Scenario: No examples and `fail_if_no_examples` set to true
-    Given a file named "spec/spec_helper.rb" with:
-      """ruby
-      RSpec.configure { |c| c.fail_if_no_examples = true }
-      """
-    Given a file named "spec/example_spec.rb" with:
-      """ruby
-      require 'spec_helper'
-
-      RSpec.describe "something" do
-      end
-      """
-    When I run `rspec spec/example_spec.rb`
-    Then the exit status should be 1
-
-  Scenario: No examples and `fail_if_no_examples` set to false
-   Given a file named "spec/spec_helper.rb" with:
-     """ruby
-     RSpec.configure { |c| c.fail_if_no_examples = false }
-     """
-   Given a file named "spec/example_spec.rb" with:
-     """ruby
-     require 'spec_helper'
-
-     RSpec.describe "something" do
-     end
-     """
-   When I run `rspec spec/example_spec.rb`
-   Then the exit status should be 0
+    When I run `rspec --pattern spec/**/*_spec.rb`
+    Then it should pass with "1 example, 0 failures"

lib/rspec/core/runner.rb

@@ -108,9 +108,10 @@ def setup(err, out)
       #   or the configured failure exit code (1 by default) if specs
       #   failed.
       def run_specs(example_groups)
-        @configuration.reporter.report(@world.example_count(example_groups)) do |reporter|
+        examples_count = @world.example_count(example_groups)
+        @configuration.reporter.report(examples_count) do |reporter|
           @configuration.with_suite_hooks do
-            if @world.example_count(example_groups) == 0 && @configuration.fail_if_no_examples
+            if examples_count == 0 && @configuration.fail_if_no_examples
               return 1
             end
             example_groups.map { |g| g.run(reporter) }.all? ? 0 : @configuration.failure_exit_code

spec/integration/fail_if_no_examples_spec.rb

@@ -0,0 +1,87 @@
+require 'support/aruba_support'
+
+RSpec.describe 'Fail if no examples' do
+  include_context "aruba support"
+  before { clean_current_dir }
+
+  context 'when 1 passing example' do
+    def passing_example(fail_if_no_examples)
+      "
+        RSpec.configure { |c| c.fail_if_no_examples = #{fail_if_no_examples} }
+
+        RSpec.describe 'something' do
+          it 'succeeds' do
+            true
+          end
+        end
+      "
+    end
+
+    it 'succeeds if fail_if_no_examples set to true' do
+      write_file 'spec/example_spec.rb', passing_example(true)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 0 failures")
+      expect(last_cmd_exitstatus).to eq(0)
+    end
+
+    it 'succeeds if fail_if_no_examples set to false' do
+      write_file 'spec/example_spec.rb', passing_example(false)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 0 failures")
+      expect(last_cmd_exitstatus).to eq(0)
+    end
+  end
+
+  context 'when 1 failing example' do
+    def failing_example(fail_if_no_examples)
+      "
+        RSpec.configure { |c| c.fail_if_no_examples = #{fail_if_no_examples} }
+
+        RSpec.describe 'something' do
+          it 'fails' do
+            fail
+          end
+        end
+      "
+    end
+
+    it 'fails if fail_if_no_examples set to true' do
+      write_file 'spec/example_spec.rb', failing_example(true)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 1 failure")
+      expect(last_cmd_exitstatus).to eq(1)
+    end
+
+    it 'fails if fail_if_no_examples set to false' do
+      write_file 'spec/example_spec.rb', failing_example(false)
+      run_command ""
+      expect(last_cmd_stdout).to include("1 example, 1 failure")
+      expect(last_cmd_exitstatus).to eq(1)
+    end
+  end
+
+  context 'when 0 examples' do
+    def no_examples(fail_if_no_examples)
+      "
+        RSpec.configure { |c| c.fail_if_no_examples = #{fail_if_no_examples} }
+
+        RSpec.describe 'something' do
+        end
+      "
+    end
+
+    it 'fails if fail_if_no_examples set to true' do
+      write_file 'spec/example_spec.rb', no_examples(true)
+      run_command ""
+      expect(last_cmd_stdout).to include("0 examples, 0 failures")
+      expect(last_cmd_exitstatus).to eq(1)
+    end
+
+    it 'succeeds if fail_if_no_examples set to false' do
+      write_file 'spec/example_spec.rb', no_examples(false)
+      run_command ""
+      expect(last_cmd_stdout).to include("0 examples, 0 failures")
+      expect(last_cmd_exitstatus).to eq(0)
+    end
+  end
+end

spec/support/aruba_support.rb

@@ -10,7 +10,7 @@ module ArubaLoader
   let(:stderr) { StringIO.new }
   let(:stdout) { StringIO.new }
 
-  attr_reader :last_cmd_stdout, :last_cmd_stderr
+  attr_reader :last_cmd_stdout, :last_cmd_stderr, :last_cmd_exitstatus
 
   def run_command(cmd)
     RSpec.configuration.color = true
@@ -24,7 +24,7 @@ def run_command(cmd)
 
     handle_current_dir_change do
       in_current_dir do
-        RSpec::Core::Runner.run(cmd_parts, temp_stderr, temp_stdout)
+        @last_cmd_exitstatus = RSpec::Core::Runner.run(cmd_parts, temp_stderr, temp_stdout)
       end
     end
   ensure