Merge pull request #2929 from santib/rspec-random

Use random order by default

Author: JonRowe

features/command_line/fail_fast.feature

@@ -30,15 +30,15 @@ Feature: `--fail-fast` option
       """
 
   Scenario: Using `--fail-fast`
-    When I run `rspec . --fail-fast`
+    When I run `rspec . --fail-fast --order defined`
     Then the output should contain ".F"
     Then the output should not contain ".F."
 
   Scenario: Using `--fail-fast=3`
-    When I run `rspec . --fail-fast=3`
+    When I run `rspec . --fail-fast=3 --order defined`
     Then the output should contain ".FFF"
     Then the output should not contain ".FFFF."
 
   Scenario: Using `--no-fail-fast`
-    When I run `rspec . --no-fail-fast`
+    When I run `rspec . --no-fail-fast --order defined`
     Then the output should contain ".FFFF."

features/command_line/format_option.feature

@@ -48,11 +48,11 @@ Feature: `--format` option
       """
 
   Scenario: Progress bar format (default)
-    When I run `rspec --format progress example_spec.rb`
+    When I run `rspec --format progress example_spec.rb --order defined`
     Then the output should contain ".F**"
 
   Scenario: Documentation format
-    When I run `rspec example_spec.rb --format documentation`
+    When I run `rspec example_spec.rb --format documentation --order defined`
     Then the output should contain:
       """
       something
@@ -63,7 +63,7 @@ Feature: `--format` option
       """
 
   Scenario: Documentation format saved to a file
-    When I run `rspec example_spec.rb --format documentation --out rspec.txt`
+    When I run `rspec example_spec.rb --format documentation --out rspec.txt --order defined`
     Then the file "rspec.txt" should contain:
       """
       something
@@ -74,7 +74,7 @@ Feature: `--format` option
       """
 
   Scenario: Multiple formats and output targets
-    When I run `rspec example_spec.rb --format progress --format documentation --out rspec.txt`
+    When I run `rspec example_spec.rb --format progress --format documentation --out rspec.txt --order defined`
     Then the output should contain ".F**"
     And the file "rspec.txt" should contain:
       """

features/configuration/custom_settings.feature

@@ -25,7 +25,7 @@ Feature: custom settings
         end
       end
       """
-    When I run `rspec ./additional_setting_spec.rb`
+    When I run `rspec ./additional_setting_spec.rb --order defined`
     Then the examples should all pass
 
   Scenario: Default to `true`
@@ -51,7 +51,7 @@ Feature: custom settings
         end
       end
       """
-    When I run `rspec ./additional_setting_spec.rb`
+    When I run `rspec ./additional_setting_spec.rb --order defined`
     Then the examples should all pass
 
   Scenario: Overridden in a subsequent `RSpec.configure` block

features/configuration/fail_fast.feature

@@ -37,7 +37,7 @@ Feature: fail fast
         end
       end
       """
-    When I run `rspec spec/example_spec.rb -fd`
+    When I run `rspec spec/example_spec.rb -fd --order defined`
     Then the output should contain "1 example, 1 failure"
 
   Scenario: `fail_fast` with multiple files, second example failing (only runs the first two examples)
@@ -77,7 +77,7 @@ Feature: fail fast
         end
       end
       """
-    When I run `rspec spec`
+    When I run `rspec spec --order defined`
     Then the output should contain "2 examples, 1 failure"
 
 
@@ -105,5 +105,5 @@ Feature: fail fast
         end
       end
       """
-    When I run `rspec spec/example_spec.rb -fd`
+    When I run `rspec spec/example_spec.rb -fd --order defined`
     Then the output should contain "3 examples, 2 failures"

features/example_groups/aliasing.feature

@@ -36,7 +36,7 @@ Feature: aliasing
       end
     end
     """
-    When I run `rspec nested_example_group_aliases_spec.rb --tag detailed -fdoc`
+    When I run `rspec nested_example_group_aliases_spec.rb --tag detailed -fdoc --order defined`
     Then the output should contain:
       """
       a detail

features/example_groups/basic_structure.feature

@@ -45,7 +45,7 @@ Feature: basic structure (`describe`/`it`)
       end
     end
     """
-    When I run `rspec nested_example_groups_spec.rb -fdoc`
+    When I run `rspec nested_example_groups_spec.rb -fdoc --order defined`
     Then the output should contain:
       """
       something

features/example_groups/shared_examples.feature

@@ -124,7 +124,7 @@ Feature: shared examples
         it_behaves_like "a collection"
       end
       """
-    When I run `rspec collection_spec.rb --format documentation`
+    When I run `rspec collection_spec.rb --format documentation --order defined`
     Then the examples should all pass
     And the output should contain:
       """
@@ -176,7 +176,7 @@ Feature: shared examples
       end
     end
     """
-    When I run `rspec shared_example_group_spec.rb --format documentation`
+    When I run `rspec shared_example_group_spec.rb --format documentation --order defined`
     Then the examples should all pass
     And the output should contain:
       """
@@ -212,7 +212,7 @@ Feature: shared examples
       it_behaves_like "a measurable object", 6, [:size, :length]
     end
     """
-    When I run `rspec shared_example_group_params_spec.rb --format documentation`
+    When I run `rspec shared_example_group_params_spec.rb --format documentation --order defined`
     Then the examples should all pass
     And the output should contain:
       """

features/helper_methods/let.feature

@@ -27,7 +27,7 @@ Feature: let and let!
         end
       end
       """
-    When I run `rspec let_spec.rb`
+    When I run `rspec let_spec.rb --order defined`
     Then the examples should all pass
 
   Scenario: Use `let!` to define a memoized helper method that is called in a `before` hook

features/hooks/before_and_after_hooks.feature

@@ -103,7 +103,7 @@ Feature: `before` and `after` hooks
         end
       end
       """
-    When I run `rspec before_context_spec.rb`
+    When I run `rspec before_context_spec.rb --order defined`
     Then the examples should all pass
 
     When I run `rspec before_context_spec.rb:15`

features/hooks/filtering.feature

@@ -228,7 +228,7 @@ Feature: filters
         it("", :around_example) { puts "example 4" }
       end
       """
-    When I run `rspec --format progress less_verbose_metadata_filter.rb`
+    When I run `rspec --format progress less_verbose_metadata_filter.rb --order defined`
     Then the examples should all pass
     And the output should contain:
       """

features/hooks/when_first_matching_example_defined.feature

@@ -53,7 +53,7 @@ Feature: `when_first_matching_example_defined` hook
       """
 
   Scenario: Running the entire suite loads the DB setup
-    When I run `rspec`
+    When I run `rspec --order defined`
     Then it should pass with:
       """
       Bootstrapped the DB.

features/pending_and_skipped_examples/skipped_examples.feature

@@ -71,7 +71,7 @@ Feature: `skip` examples
         end
       end
       """
-    When I run `rspec temporarily_skipped_spec.rb`
+    When I run `rspec temporarily_skipped_spec.rb --order defined`
     Then the exit status should be 0
     And the output should contain "3 examples, 0 failures, 3 pending"
     And the output should contain:

features/subject/explicit_subject.feature

@@ -73,7 +73,7 @@ Feature: Explicit Subject
         end
       end
       """
-    When I run `rspec memoized_subject_spec.rb`
+    When I run `rspec memoized_subject_spec.rb --order defined`
     Then the examples should all pass
 
   Scenario: The `subject` is available in `before` hooks
@@ -165,7 +165,7 @@ Feature: Explicit Subject
         end
       end
       """
-    When I run `rspec named_subject_spec.rb`
+    When I run `rspec named_subject_spec.rb --order defined`
     Then the examples should all pass
 
   Scenario: Use `subject!(:name)` to define a helper method called before the example

lib/rspec/core/ordering.rb

@@ -85,14 +85,13 @@ def initialize(configuration)
           @configuration = configuration
           @strategies    = {}
 
-          register(:random, Random.new(configuration))
+          random = Random.new(configuration)
+          register(:random, random)
           register(:recently_modified, RecentlyModified.new)
+          register(:defined, Identity.new)
 
-          identity = Identity.new
-          register(:defined, identity)
-
-          # The default global ordering is --defined.
-          register(:global, identity)
+          # The default global ordering is --random.
+          register(:global, random)
         end
 
         def fetch(name, &fallback)

lib/rspec/core/project_initializer/spec/spec_helper.rb

@@ -66,10 +66,9 @@
   # particularly slow.
   config.profile_examples = 10
 
-  # Run specs in random order to surface order dependencies. If you find an
-  # order dependency and want to debug it, you can fix the order by providing
-  # the seed, which is printed after each run.
-  #     --seed 1234
+  # RSpec runs specs in a random order by default to surface order dependencies.
+  # We recommend this setting but it can be changed to others, such as `:defined`
+  # to run specs in the defined order. See the documentation for more details.
   config.order = :random
 
   # Seed global randomization in this process using the `--seed` CLI option.

spec/integration/bisect_runners_spec.rb

@@ -39,7 +39,7 @@ def with_runner(&block)
 
       with_runner do |runner|
         expect(runner.original_results).to have_attributes(
-          :all_example_ids => %w[ ./spec/a_spec.rb[1:1] ./spec/a_spec.rb[1:2] ],
+          :all_example_ids => match_array(%w[ ./spec/a_spec.rb[1:1] ./spec/a_spec.rb[1:2] ]),
           :failed_example_ids => %w[ ./spec/a_spec.rb[1:2] ]
         )
 

spec/integration/suite_hooks_errors_spec.rb

@@ -25,6 +25,7 @@
       c.backtrace_exclusion_patterns << %r{/rspec-core/spec/} << %r{rspec_with_simplecov}
       c.failure_exit_code = failure_exit_code
       c.error_exit_code = error_exit_code
+      c.seed = 123
     end
   end
 
@@ -50,6 +51,8 @@ def run_spec_expecting_non_zero(before_or_after)
     output = run_spec_expecting_non_zero(:before)
     expect(output).to eq unindent(<<-EOS)
 
+      Randomized with seed 123
+
       An error occurred in a `before(:suite)` hook.
       Failure/Error: raise 'boom'
 
@@ -61,12 +64,16 @@ def run_spec_expecting_non_zero(before_or_after)
       Finished in n.nnnn seconds (files took n.nnnn seconds to load)
       0 examples, 0 failures, 1 error occurred outside of examples
 
+      Randomized with seed 123
+
     EOS
   end
 
   it 'nicely formats errors in `after(:suite)` hooks and exits with non-zero' do
     output = run_spec_expecting_non_zero(:after)
     expect(output).to eq unindent(<<-EOS)
+
+      Randomized with seed 123
       .
       An error occurred in an `after(:suite)` hook.
       Failure/Error: raise 'boom'
@@ -79,6 +86,8 @@ def run_spec_expecting_non_zero(before_or_after)
       Finished in n.nnnn seconds (files took n.nnnn seconds to load)
       1 example, 0 failures, 1 error occurred outside of examples
 
+      Randomized with seed 123
+
     EOS
   end
 
@@ -115,6 +124,8 @@ def run_spec_expecting_non_zero(before_or_after)
 
     expect(output).to eq unindent(<<-EOS)
 
+      Randomized with seed 123
+
       An error occurred in a `before(:suite)` hook.
       Failure/Error: c.before(:suite) { raise 'before 1' }
 
@@ -140,6 +151,8 @@ def run_spec_expecting_non_zero(before_or_after)
       Finished in n.nnnn seconds (files took n.nnnn seconds to load)
       0 examples, 0 failures, 3 errors occurred outside of examples
 
+      Randomized with seed 123
+
     EOS
   end
 end

spec/rspec/core/example_group_spec.rb

@@ -823,6 +823,8 @@ def define_and_run_group(define_outer_example = false)
       end
 
       it "runs before_all_defined_in_config, before all, before each, example, after each, after all, after_all_defined_in_config in that order" do
+        RSpec.configuration.order = :defined
+
         order = []
 
         RSpec.configure do |c|
@@ -1442,7 +1444,11 @@ def extract_execution_results(group)
       end
 
       context "with fail_fast enabled" do
-        before { RSpec.configuration.fail_fast = true }
+        before do
+          RSpec.configuration.fail_fast = true
+          RSpec.configuration.order = :defined
+        end
+
         let(:group) { RSpec.describe }
         let(:reporter) { Reporter.new(RSpec.configuration) }
 
@@ -1466,7 +1472,11 @@ def extract_execution_results(group)
       end
 
       context "with fail_fast set to 3" do
-        before { RSpec.configuration.fail_fast = 3 }
+        before do
+          RSpec.configuration.fail_fast = 3
+          RSpec.configuration.order = :defined
+        end
+
         let(:group) { RSpec.describe }
         let(:reporter) { Reporter.new(RSpec.configuration) }
 
@@ -1628,6 +1638,8 @@ def extract_execution_results(group)
       end
 
       it "applies new metadata-based config items based on the update" do
+        RSpec.configuration.order = :defined
+
         extension = Module.new do
           def extension_method; 17; end
         end
@@ -1674,6 +1686,8 @@ def extension_method; 17; end
       end
 
       it "does not cause duplicate hooks to be added when re-configuring the group" do
+        RSpec.configuration.order = :defined
+
         sequence = []
         RSpec.configure do |c|
           c.before(:example, :foo => true) { sequence << :global_before_hook }

spec/rspec/core/example_spec.rb

@@ -913,7 +913,7 @@ def expect_pending_result(example)
       example2 = group.example("example 2") { current_examples << RSpec.current_example }
 
       group.run
-      expect(current_examples).to eq([example1, example2])
+      expect(current_examples).to match_array([example1, example2])
     end
   end
 

spec/rspec/core/formatters/documentation_formatter_spec.rb

@@ -52,6 +52,8 @@ def execution_result(values)
     end
 
     it "represents nested group using hierarchy tree" do
+      RSpec.configuration.order = :defined
+
       group = RSpec.describe("root")
       context1 = group.describe("context 1")
       context1.example("nested example 1.1"){}
@@ -112,6 +114,8 @@ def execution_result(values)
     end
 
     it "strips whitespace for each row" do
+      RSpec.configuration.order = :defined
+
       group = RSpec.describe(" root ")
       context1 = group.describe(" nested ")
       context1.example(" example 1 ") {}

spec/rspec/core/formatters/json_formatter_spec.rb

@@ -20,6 +20,8 @@
   end
 
   it "outputs expected json (brittle high level functional test)" do
+    RSpec.configuration.order = :defined
+
     its = []
     group = RSpec.describe("one apiece") do
       its.push it("succeeds") { expect(1).to eq 1 }