Merge pull request #3025 from rspec/allow-custom-ordering-on-cli

Allow custom ordering on cli

Author: JonRowe

features/command_line/order.feature

@@ -0,0 +1,119 @@
+Feature: Using the `--order` option
+
+Use the `--order` option to tell RSpec how to order the files, groups, and
+examples. The available ordering schemes are `defined` and `rand`.
+
+`defined` is the default, which executes groups and examples in the order they
+are defined as the spec files are loaded, with the caveat that each group
+runs its examples before running its nested example groups, even if the
+nested groups are defined before the examples.
+
+Use `rand` to randomize the order of groups and examples within the groups.
+Nested groups are always run from top-level to bottom-level in order to avoid
+executing `before(:context)` and `after(:context)` hooks more than once, but the
+order of groups at each level is randomized.
+
+With `rand` you can also specify a seed.
+
+Use `recently-modified` to run the most recently modified files first. You can
+combine it with `--only-failures` to find the most recent failing specs. Note
+that `recently-modified` and `rand` are mutually exclusive.
+
+** Example usage **
+
+The `defined` option is only necessary when you have `--order rand` stored in a
+config file (e.g. `.rspec`) and you want to override it from the command line.
+
+<pre><code class="bash">--order defined
+--order rand
+--order rand:123
+--seed 123 # same as --order rand:123
+--order recently-modified
+</code></pre>
+
+Scenario: Default order is `defined`
+  Given a file named "example_spec.rb" with:
+    """ruby
+    RSpec.describe "something" do
+      it "does something" do
+      end
+
+      it "in order" do
+      end
+    end
+    """
+    When I run `rspec example_spec.rb --format documentation`
+    Then the output should contain:
+      """
+      something
+        does something
+        in order
+      """
+
+Scenario: Order can be psuedo randomised (seed used here to fix the ordering for tests)
+  Given a file named "example_spec.rb" with:
+    """ruby
+    RSpec.describe "something" do
+      it "does something" do
+      end
+
+      it "in order" do
+      end
+    end
+    """
+    When I run `rspec example_spec.rb --format documentation --order rand:123`
+    Then the output should contain:
+      """
+      something
+        in order
+        does something
+      """
+
+Scenario: Configure custom ordering
+  Given a file named "example_spec.rb" with:
+    """ruby
+    RSpec.configure do |config|
+      config.register_ordering(:reverse) do |examples|
+        examples.reverse
+      end
+      config.order = :reverse
+    end
+
+    RSpec.describe "something" do
+      it "does something" do
+      end
+
+      it "in order" do
+      end
+    end
+    """
+    When I run `rspec example_spec.rb --format documentation --order reverse`
+    Then the output should contain:
+      """
+      something
+        in order
+        does something
+      """
+
+Scenario: Override order to `defined` when another order is set
+  Given a file named "example_spec.rb" with:
+    """ruby
+    RSpec.configure do |config|
+      config.order = :random
+      config.seed = 123
+    end
+    RSpec.describe "something" do
+      it "does something" do
+      end
+
+      it "in order" do
+      end
+    end
+    """
+    When I run `rspec example_spec.rb --format documentation --order defined`
+    Then the output should contain:
+      """
+      something
+        does something
+        in order
+      """

features/command_line/order.md

@@ -78,6 +78,30 @@ def order(list)
         end
       end
 
+      # @private
+      # A strategy which delays looking up the ordering until needed
+      class Delayed
+        def initialize(registry, name)
+          @registry = registry
+          @name = name
+        end
+
+        def order(list)
+          strategy.order(list)
+        end
+
+        private
+
+        def strategy
+          @strategy ||= lookup_strategy
+        end
+
+        def lookup_strategy
+          raise "Undefined ordering strategy #{@name.inspect}" unless @registry.has_strategy?(@name)
+          @registry.fetch(@name)
+        end
+      end
+
       # @private
       # Stores the different ordering strategies.
       class Registry
@@ -99,6 +123,10 @@ def fetch(name, &fallback)
           @strategies.fetch(name, &fallback)
         end
 
+        def has_strategy?(name)
+          @strategies.key?(name)
+        end
+
         def register(sym, strategy)
           @strategies[sym] = strategy
         end
@@ -143,9 +171,20 @@ def order=(type)
                             :defined
                           elsif order == 'recently-modified'
                             :recently_modified
+                          else
+                            order.to_sym
                           end
 
-          register_ordering(:global, ordering_registry.fetch(ordering_name)) if ordering_name
+          if ordering_name
+            strategy =
+              if ordering_registry.has_strategy?(ordering_name)
+                ordering_registry.fetch(ordering_name)
+              else
+                Delayed.new(ordering_registry, ordering_name)
+              end
+
+            register_ordering(:global, strategy)
+          end
         end
 
         def force(hash)

lib/rspec/core/ordering.rb

@@ -2602,6 +2602,49 @@ def use_seed_on(registry)
           expect(ordering_strategy.order(list)).to eq([1, 2, 3, 4])
         end
       end
+
+      context 'given a custom ordering strategy' do
+        before do
+          allow(RSpec).to receive_messages(:configuration => config)
+        end
+
+        it 'will lookup a previously registed ordering strategy' do
+          config.register_ordering(:custom_scheme) { |list| list.reverse }
+
+          config.order = :custom_scheme
+
+          strategy = config.ordering_registry.fetch(:global)
+          expect(strategy.order([1, 2, 3, 4])).to eq [4, 3, 2, 1]
+        end
+
+        it 'will defer lookup until running' do
+          config.order = :custom_scheme
+
+          strategy = config.ordering_registry.fetch(:global)
+          expect(strategy).to be_an_instance_of(Ordering::Delayed)
+
+          config.register_ordering(:custom_scheme) { |list| list.reverse }
+          expect(strategy.order([1, 2, 3, 4])).to eq [4, 3, 2, 1]
+        end
+
+        it 'will raise an error if ordering is not present when needed' do
+          config.order = :custom_scheme
+
+          strategy = config.ordering_registry.fetch(:global)
+          expect(strategy).to be_an_instance_of(Ordering::Delayed)
+
+          expect { strategy.order([1, 2, 3, 4]) }.to raise_error("Undefined ordering strategy :custom_scheme")
+        end
+
+        it 'will lookup schemes as symbols even if given as strings' do
+          config.order = 'custom_scheme'
+
+          config.register_ordering(:custom_scheme) { |list| list.reverse }
+
+          strategy = config.ordering_registry.fetch(:global)
+          expect(strategy.order([1, 2, 3, 4])).to eq [4, 3, 2, 1]
+        end
+      end
     end
 
     describe "#register_ordering" do

spec/rspec/core/configuration_spec.rb

@@ -104,6 +104,18 @@ def order_with(seed)
         end
       end
 
+      RSpec.describe Delayed do
+        let(:registry) { Registry.new(Configuration.new) }
+
+        it 'looks up a strategy to order the list later on' do
+          strategy = Delayed.new(registry, :reverse)
+          expect { strategy.order([1, 2, 3, 4]) }.to raise_error("Undefined ordering strategy :reverse")
+
+          registry.register(:reverse, Custom.new(proc { |list| list.reverse }))
+          expect(strategy.order([1, 2, 3, 4])).to eq([4, 3, 2, 1])
+        end
+      end
+
       RSpec.describe Registry do
         let(:configuration) { Configuration.new }
         subject(:registry) { Registry.new(configuration) }
@@ -144,6 +156,15 @@ def order_with(seed)
             end
           end
         end
+
+        describe "#has_strategy?(name)" do
+          it "returns true if the strategy was registered" do
+            expect {
+              registry.register(:reverse, Custom.new(proc { |list| list.reverse }))
+            }.to change { registry.has_strategy?(:reverse) }.from(false).to(true)
+          end
+        end
+
       end
     end
   end