Add parameterized tests (#13618)

Sometimes you want to run the same tests but with different parameters.
In ExUnit, it is possible to do so by passing a `:parameterize` key to
`ExUnit.Case`. The value must be a list of maps which will be the
parameters merged into the test context.

For example, Elixir has a module called `Registry`, which can have type
`:unique` or `:duplicate`, and can control its concurrency factor using
the `:partitions` option. If you have a number of tests that *behave the
same* across all of those values, you can parameterize those tests with:

    use ExUnit.Case,
      async: true,
      parameterize:
        for(kind <- [:unique, :duplicate],
            partitions <- [1, 8],
            do: %{kind: kind, partitions: partitions})

Then, in your tests, you can access the parameters as part of the context:

    test "starts a registry", %{kind: kind, partitions: partitions} do
      ...
    end

Use parameterized tests with care:

  * Although parameterized tests run concurrently when `async: true` is also given,
    abuse of parameterized tests may make your test suite slower

  * If you use parameterized tests and then find yourself adding conditionals
    in your tests to deal with different parameters, then parameterized tests
    may be the wrong solution to your problem. Consider creating separated
    tests and sharing logic between them using regular functions

Author: josevalim

lib/elixir/test/elixir/registry_test.exs

@@ -101,9 +101,10 @@ defmodule ExUnit do
       * `:time` - the duration in microseconds of the test's runtime
       * `:tags` - the test tags
       * `:logs` - the captured logs
+      * `:parameters` - the test parameters
 
     """
-    defstruct [:name, :case, :module, :state, time: 0, tags: %{}, logs: ""]
+    defstruct [:name, :case, :module, :state, time: 0, tags: %{}, logs: "", parameters: %{}]
 
     # TODO: Remove the `:case` field on v2.0
     @type t :: %__MODULE__{
@@ -131,8 +132,10 @@ defmodule ExUnit do
 
       * `:tests` - all tests in this module
 
+      * `:parameters` - the test module parameters
+
     """
-    defstruct [:file, :name, :state, tags: %{}, tests: []]
+    defstruct [:file, :name, :state, tags: %{}, tests: [], parameters: %{}]
 
     @type t :: %__MODULE__{
             file: binary(),
@@ -404,10 +407,12 @@ defmodule ExUnit do
     for module <- additional_modules do
       module_attributes = module.__info__(:attributes)
 
-      if true in Keyword.get(module_attributes, :ex_unit_async, []) do
-        ExUnit.Server.add_async_module(module)
-      else
-        ExUnit.Server.add_sync_module(module)
+      case Keyword.get(module_attributes, :ex_unit_module) do
+        [config] ->
+          ExUnit.Server.add_module(module, config)
+
+        _ ->
+          raise(ArgumentError, "#{inspect(module)} is not a ExUnit.Case module")
       end
     end
 

lib/ex_unit/lib/ex_unit.ex

@@ -15,6 +15,11 @@ defmodule ExUnit.Case do
     * `:register` - when `false`, does not register this module within
       ExUnit server. This means the module won't run when ExUnit suite runs.
 
+    * `:parameterize` - a list of maps to parameterize tests. If both
+      `:async` and `:parameterize` are given, the different parameters
+      run concurrently. See the "Parameterized tests" section below for
+      more information.
+
   > #### `use ExUnit.Case` {: .info}
   >
   > When you `use ExUnit.Case`, it will import the functionality
@@ -173,6 +178,41 @@ defmodule ExUnit.Case do
 
     * `:tmp_dir` - (since v1.11.0) see the "Tmp Dir" section below
 
+  ## Parameterized tests
+
+  Sometimes you want to run the same tests but with different parameters.
+  In ExUnit, it is possible to do so by passing a `:parameterize` key to
+  `ExUnit.Case`. The value must be a list of maps which will be the
+  parameters merged into the test context.
+
+  For example, Elixir has a module called `Registry`, which can have type
+  `:unique` or `:duplicate`, and can control its concurrency factor using
+  the `:partitions` option. If you have a number of tests that *behave the
+  same* across all of those values, you can parameterize those tests with:
+
+      use ExUnit.Case,
+        async: true,
+        parameterize:
+          for(kind <- [:unique, :duplicate],
+              partitions <- [1, 8],
+              do: %{kind: kind, partitions: partitions})
+
+  Then, in your tests, you can access the parameters as part of the context:
+
+      test "starts a registry", %{kind: kind, partitions: partitions} do
+        ...
+      end
+
+  Use parameterized tests with care:
+
+    * Although parameterized tests run concurrently when `async: true` is also given,
+      abuse of parameterized tests may make your test suite slower
+
+    * If you use parameterized tests and then find yourself adding conditionals
+      in your tests to deal with different parameters, then parameterized tests
+      may be the wrong solution to your problem. Consider creating separated
+      tests and sharing logic between them using regular functions
+
   ## Filters
 
   Tags can also be used to identify specific tests, which can then
@@ -278,6 +318,18 @@ defmodule ExUnit.Case do
               ~s(got: #{inspect(opts)})
     end
 
+    {register?, opts} = Keyword.pop(opts, :register, true)
+    {async?, opts} = Keyword.pop(opts, :async, false)
+    {parameterize, opts} = Keyword.pop(opts, :parameterize, nil)
+
+    unless parameterize == nil or (is_list(parameterize) and Enum.all?(parameterize, &is_map/1)) do
+      raise ArgumentError, ":parameterize must be a list of maps, got: #{inspect(parameterize)}"
+    end
+
+    if opts != [] do
+      IO.warn("unknown options given to ExUnit.Case: #{inspect(opts)}")
+    end
+
     registered? = Module.has_attribute?(module, :ex_unit_tests)
 
     unless registered? do
@@ -299,23 +351,18 @@ defmodule ExUnit.Case do
 
       Enum.each(accumulate_attributes, &Module.register_attribute(module, &1, accumulate: true))
 
-      persisted_attributes = [:ex_unit_async]
+      persisted_attributes = [:ex_unit_module]
 
       Enum.each(persisted_attributes, &Module.register_attribute(module, &1, persist: true))
 
-      if Keyword.get(opts, :register, true) do
+      if register? do
         Module.put_attribute(module, :after_compile, ExUnit.Case)
       end
 
       Module.put_attribute(module, :before_compile, ExUnit.Case)
     end
 
-    async? = opts[:async]
-
-    if is_boolean(async?) or not registered? do
-      Module.put_attribute(module, :ex_unit_async, async? || false)
-    end
-
+    Module.put_attribute(module, :ex_unit_module, {async?, parameterize})
     registered?
   end
 
@@ -498,21 +545,22 @@ defmodule ExUnit.Case do
   end
 
   @doc false
-  defmacro __before_compile__(env) do
+  defmacro __before_compile__(%{module: module} = env) do
     tests =
-      env.module
+      module
       |> Module.get_attribute(:ex_unit_tests)
       |> Enum.reverse()
       |> Macro.escape()
 
-    moduletag = Module.get_attribute(env.module, :moduletag)
+    moduletag = Module.get_attribute(module, :moduletag)
+    {async?, _parameterize} = Module.get_attribute(module, :ex_unit_module)
 
     tags =
       moduletag
       |> normalize_tags()
       |> validate_tags()
       |> Map.new()
-      |> Map.merge(%{module: env.module, case: env.module})
+      |> Map.merge(%{module: module, case: env.module, async: async?})
 
     quote do
       def __ex_unit__ do
@@ -529,17 +577,16 @@ defmodule ExUnit.Case do
   @doc false
   def __after_compile__(%{module: module}, _) do
     cond do
-      Process.whereis(ExUnit.Server) == nil ->
-        unless Code.can_await_module_compilation?() do
-          raise "cannot use ExUnit.Case without starting the ExUnit application, " <>
-                  "please call ExUnit.start() or explicitly start the :ex_unit app"
-        end
+      Process.whereis(ExUnit.Server) ->
+        config = Module.get_attribute(module, :ex_unit_module)
+        ExUnit.Server.add_module(module, config)
 
-      Module.get_attribute(module, :ex_unit_async) ->
-        ExUnit.Server.add_async_module(module)
+      Code.can_await_module_compilation?() ->
+        :ok
 
       true ->
-        ExUnit.Server.add_sync_module(module)
+        raise "cannot use ExUnit.Case without starting the ExUnit application, " <>
+                "please call ExUnit.start() or explicitly start the :ex_unit app"
     end
   end
 
@@ -577,7 +624,6 @@ defmodule ExUnit.Case do
 
     moduletag = Module.get_attribute(mod, :moduletag)
     tag = Module.delete_attribute(mod, :tag)
-    async = Module.get_attribute(mod, :ex_unit_async)
 
     {name, describe, describe_line, describetag} =
       case Module.get_attribute(mod, :ex_unit_describe) do
@@ -602,7 +648,6 @@ defmodule ExUnit.Case do
         line: line,
         file: file,
         registered: registered,
-        async: async,
         describe: describe,
         describe_line: describe_line,
         test_type: test_type

lib/ex_unit/lib/ex_unit/case.ex

@@ -137,9 +137,14 @@ defmodule ExUnit.CLIFormatter do
     {:noreply, update_test_timings(config, test)}
   end
 
-  def handle_cast({:module_started, %ExUnit.TestModule{name: name, file: file}}, config) do
+  def handle_cast({:module_started, %ExUnit.TestModule{} = module}, config) do
     if config.trace do
+      %{name: name, file: file, parameters: parameters} = module
       IO.puts("\n#{inspect(name)} [#{Path.relative_to_cwd(file)}]")
+
+      if parameters != %{} do
+        IO.puts("Parameters: #{inspect(parameters)}")
+      end
     end
 
     {:noreply, config}

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -94,6 +94,7 @@ defmodule ExUnit.Formatter do
           | :error_info
           | :test_module_info
           | :test_info
+          | :parameters_info
           | :location_info
           | :stacktrace_info
           | :blame_diff
@@ -119,7 +120,7 @@ defmodule ExUnit.Formatter do
     * `:diff_insert` and `:diff_insert_whitespace` - Should format a diff insertion,
       with or without whitespace respectively.
 
-    * `:extra_info` - Should format extra information, such as the `"code: "` label
+    * `:extra_info` - Should format optional extra labels, such as the `"code: "` label
       that precedes code to show.
 
     * `:error_info` - Should format error information.
@@ -129,6 +130,8 @@ defmodule ExUnit.Formatter do
 
     * `:test_info` - Should format test information.
 
+    * `:parameters_info` - Should format test parameters.
+
     * `:location_info` - Should format test location information.
 
     * `:stacktrace_info` - Should format stacktrace information.
@@ -266,9 +269,10 @@ defmodule ExUnit.Formatter do
         ) :: String.t()
         when failure: {atom, term, Exception.stacktrace()}
   def format_test_failure(test, failures, counter, width, formatter) do
-    %ExUnit.Test{name: name, module: module, tags: tags} = test
+    %ExUnit.Test{name: name, module: module, tags: tags, parameters: parameters} = test
 
     test_info(with_counter(counter, "#{name} (#{inspect(module)})"), formatter) <>
+      test_parameters(parameters, formatter) <>
       test_location(with_location(tags), formatter) <>
       Enum.map_join(Enum.with_index(failures), "", fn {{kind, reason, stack}, index} ->
         {text, stack} = format_kind_reason(test, kind, reason, stack, width, formatter)
@@ -305,9 +309,10 @@ defmodule ExUnit.Formatter do
         ) :: String.t()
         when failure: {atom, term, Exception.stacktrace()}
   def format_test_all_failure(test_module, failures, counter, width, formatter) do
-    name = test_module.name
+    %{name: name, parameters: parameters} = test_module
 
     test_module_info(with_counter(counter, "#{inspect(name)}: "), formatter) <>
+      test_parameters(parameters, formatter) <>
       Enum.map_join(Enum.with_index(failures), "", fn {{kind, reason, stack}, index} ->
         {text, stack} = format_kind_reason(test_module, kind, reason, stack, width, formatter)
         failure_header(failures, index) <> text <> format_stacktrace(stack, name, nil, formatter)
@@ -711,6 +716,15 @@ defmodule ExUnit.Formatter do
   defp test_info(msg, nil), do: msg <> "\n"
   defp test_info(msg, formatter), do: test_info(formatter.(:test_info, msg), nil)
 
+  defp test_parameters(params, _formatter) when params == %{}, do: ""
+  defp test_parameters(params, nil) when is_binary(params), do: "     " <> params <> "\n"
+
+  defp test_parameters(params, nil) when is_map(params),
+    do: test_parameters("Parameters: #{inspect(params)}", nil)
+
+  defp test_parameters(params, formatter),
+    do: test_parameters(formatter.(:parameters_info, params), nil)
+
   defp test_location(msg, nil), do: "     " <> msg <> "\n"
   defp test_location(msg, formatter), do: test_location(formatter.(:location_info, msg), nil)