Include max cases in ExUnit reports (#13521)

Some CI environments may report the wrong number
of cores to the Erlang VM. So we choose to print
the number of max cases along side the seed
(both which affect randomness of tests) at the
top of each suite.

Author: josevalim

lib/elixir/pages/mix-and-otp/introduction-to-mix.md

@@ -213,12 +213,11 @@ This file will be required by Mix every time before we run our tests. We can run
 $ mix test
 Compiled lib/kv.ex
 Generated kv app
+Running ExUnit with seed: 540224, max_cases: 16
 ..
 
 Finished in 0.04 seconds
 1 doctest, 1 test, 0 failures
-
-Randomized with seed 540224
 ```
 
 Notice that by running `mix test`, Mix has compiled the source files and generated the application manifest once again. This happens because Mix supports multiple environments, which we will discuss later in this chapter.

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -8,10 +8,12 @@ defmodule ExUnit.CLIFormatter do
   ## Callbacks
 
   def init(opts) do
-    print_filters(Keyword.take(opts, [:include, :exclude]))
+    IO.puts("Running ExUnit with seed: #{opts[:seed]}, max_cases: #{opts[:max_cases]}")
+    print_filters(opts, :exclude)
+    print_filters(opts, :include)
+    IO.puts("")
 
     config = %{
-      seed: opts[:seed],
       trace: opts[:trace],
       colors: colors(opts),
       width: get_terminal_width(),
@@ -356,22 +358,16 @@ defmodule ExUnit.CLIFormatter do
       true ->
         IO.puts(success(message, config))
     end
-
-    IO.puts("\nRandomized with seed #{config.seed}")
   end
 
   defp if_true(value, false, _fun), do: value
   defp if_true(value, true, fun), do: fun.(value)
 
-  defp print_filters(include: [], exclude: []) do
-    :ok
-  end
-
-  defp print_filters(include: include, exclude: exclude) do
-    if exclude != [], do: IO.puts(format_filters(exclude, :exclude))
-    if include != [], do: IO.puts(format_filters(include, :include))
-    IO.puts("")
-    :ok
+  defp print_filters(opts, key) do
+    case opts[key] do
+      [] -> :ok
+      filters -> IO.puts(format_filters(filters, key))
+    end
   end
 
   defp print_failure(formatted, config) do

lib/ex_unit/test/ex_unit_test.exs

@@ -138,8 +138,6 @@ defmodule ExUnitTest do
            Showing results so far...
 
            0 failures
-
-           Randomized with seed 0
            """
   end
 
@@ -926,7 +924,7 @@ defmodule ExUnitTest do
   end
 
   describe ":repeat_until_failure" do
-    test "default value 0" do
+    test "defaults to 0" do
       configure_and_reload_on_exit([])
       ExUnit.start(autorun: false)
       config = ExUnit.configuration()
@@ -940,7 +938,7 @@ defmodule ExUnitTest do
       assert config[:repeat_until_failure] == 5
     end
 
-    test ":repeat_until_failure repeats tests up to the configured number of times" do
+    test "repeats tests up to the configured number of times" do
       defmodule TestRepeatUntilFailureReached do
         use ExUnit.Case
 
@@ -962,12 +960,12 @@ defmodule ExUnitTest do
           assert ExUnit.run() == %{total: 5, failures: 0, skipped: 1, excluded: 1}
         end)
 
-      runs = String.split(output, "Excluding", trim: true)
+      runs = String.split(output, "Running ExUnit", trim: true)
       # 6 runs in total, 5 repeats
       assert length(runs) == 6
     end
 
-    test ":repeat_until_failure stops on failure" do
+    test "stops on failure" do
       {:ok, pid} = Agent.start_link(fn -> 0 end)
       Process.register(pid, :ex_unit_repeat_until_failure_count)
 
@@ -1002,7 +1000,7 @@ defmodule ExUnitTest do
           assert ExUnit.run() == %{total: 4, excluded: 2, failures: 1, skipped: 1}
         end)
 
-      runs = String.split(output, "Excluding", trim: true)
+      runs = String.split(output, "Running ExUnit", trim: true)
       # four runs in total, the first two repeats work fine, the third repeat (4th run)
       # fails, therefore we stop
       assert length(runs) == 4

lib/mix/lib/mix/tasks/test.ex

@@ -36,6 +36,7 @@ defmodule Mix.Tasks.Test do
   a summary at the end, as seen below:
 
       $ mix test
+      Running ExUnit with seed: 646219, max_cases: 16
       ...
 
         1) test greets the world (FooTest)
@@ -52,8 +53,6 @@ defmodule Mix.Tasks.Test do
       Finished in 0.05 seconds (0.00s async, 0.05s sync)
       1 doctest, 11 tests, 1 failure
 
-      Randomized with seed 646219
-
   For each test, the test suite will print a dot. Failed tests
   are printed immediately in the format described in the next
   section.

lib/mix/test/mix/tasks/test_test.exs

@@ -265,7 +265,7 @@ defmodule Mix.Tasks.TestTest do
       in_fixture("test_stale", fn ->
         port = mix_port(~w[test --stale --listen-on-stdin])
 
-        assert receive_until_match(port, "seed", "") =~ "2 tests"
+        assert receive_until_match(port, "0 failures", "") =~ "2 tests"
 
         Port.command(port, "\n")
 
@@ -294,7 +294,7 @@ defmodule Mix.Tasks.TestTest do
 
         Port.command(port, "\n")
 
-        assert receive_until_match(port, "seed", "") =~ "2 tests"
+        assert receive_until_match(port, "0 failures", "") =~ "2 tests"
 
         File.write!("test/b_test_stale.exs", """
         defmodule BTest do
@@ -323,7 +323,7 @@ defmodule Mix.Tasks.TestTest do
 
         Port.command(port, "\n")
 
-        assert receive_until_match(port, "seed", "") =~ "2 tests"
+        assert receive_until_match(port, "0 failures", "") =~ "2 tests"
       end)
     end
   end
@@ -477,38 +477,33 @@ defmodule Mix.Tasks.TestTest do
 
         output = mix(["test", "apps/bar/test/bar_tests.exs"])
 
-        assert output =~ """
-               ==> bar
-               ....
-               """
+        assert output =~ "==> bar"
+        assert output =~ "...."
 
         refute output =~ "==> foo"
         refute output =~ "Paths given to \"mix test\" did not match any directory/file"
 
         output = mix(["test", "./apps/bar/test/bar_tests.exs"])
 
-        assert output =~ """
-               ==> bar
-               ....
-               """
+        assert output =~ "==> bar"
+        assert output =~ "...."
 
         refute output =~ "==> foo"
         refute output =~ "Paths given to \"mix test\" did not match any directory/file"
 
         output = mix(["test", Path.expand("apps/bar/test/bar_tests.exs")])
 
-        assert output =~ """
-               ==> bar
-               ....
-               """
+        assert output =~ "==> bar"
+        assert output =~ "...."
 
         refute output =~ "==> foo"
         refute output =~ "Paths given to \"mix test\" did not match any directory/file"
 
         output = mix(["test", "apps/bar/test/bar_tests.exs:10"])
 
+        assert output =~ "==> bar"
+
         assert output =~ """
-               ==> bar
                Excluding tags: [:test]
                Including tags: [location: {"test/bar_tests.exs", 10}]