rabbitmq_ct_helpers: Find crashes in RabbitMQ logs

If we find some crashes after stopping a node, the crash reports are
logged in Common Test HTML report and an assertion is fired.

Here is an example of a crash report found:

The console output:

    * [khepri_store]
        policy_ttl (00:00.574)
        operator_policy_ttl (00:00.178)
        operator_retroactive_policy_ttl (00:00.154)
        operator_retroactive_policy_publish_ttl (00:01.677)

    policy_SUITE > khepri_store
        {error,
            {{assertEqual,
                 [{module,rabbit_ct_broker_helpers},
                  {line,1050},
                  {expression,"CrashesCount"},
                  {expected,0},
                  {value,1}]},
             [{rabbit_ct_broker_helpers,find_crashes_in_logs,1,
                  [{file,"rabbit_ct_broker_helpers.erl"},{line,1050}]},
              {rabbit_ct_broker_helpers,stop_rabbitmq_nodes,1,
                  [{file,"rabbit_ct_broker_helpers.erl"},{line,1019}]},
              {rabbit_ct_helpers,run_steps,2,
                  [{file,"rabbit_ct_helpers.erl"},{line,136}]},
              {test_server,ts_tc,3,[{file,"test_server.erl"},{line,1783}]},
              {test_server,run_test_case_eval1,6,
                  [{file,"test_server.erl"},{line,1380}]},
              {test_server,run_test_case_eval,9,
                  [{file,"test_server.erl"},{line,1224}]}]}}

The Common Test HTML (truncated) report:

    *** User 2022-06-10 14:58:50.687 ***
    Looking up any crash reports in the node log files. If we find some, they will appear below:

    *** User 2022-06-10 14:58:50.716 ***
    gen_server termination:

    2022-06-10 14:58:25.785158+02:00 [error] <0.1387.0> ** Generic server rabbit_node_monitor terminating
    2022-06-10 14:58:25.785158+02:00 [error] <0.1387.0> ** Last message in was {'DOWN',#Ref<0.2117588230.1026555905.48245>,process,
    2022-06-10 14:58:25.785158+02:00 [error] <0.1387.0>                            {rabbit,'rmq-ct-khepri_store-1-21000@localhost'},
    2022-06-10 14:58:25.785158+02:00 [error] <0.1387.0>                            normal}
    (...)

    *** User 2022-06-10 14:58:50.717 ***
    Found 1 crash reports

Author: dumbbell

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -785,10 +785,12 @@ query_node(Config, NodeConfig) ->
       [rabbit, plugins_dir]),
     {ok, EnabledPluginsFile} = rpc(Config, Nodename, application, get_env,
       [rabbit, enabled_plugins_file]),
+    LogLocations = rpc(Config, Nodename, rabbit, log_locations, []),
     Vars0 = [{pid_file, PidFile},
              {mnesia_dir, MnesiaDir},
              {plugins_dir, PluginsDir},
-             {enabled_plugins_file, EnabledPluginsFile}],
+             {enabled_plugins_file, EnabledPluginsFile},
+             {log_locations, LogLocations}],
     Vars = try
                EnabledFeatureFlagsFile = rpc(Config, Nodename,
                                              rabbit_feature_flags,
@@ -999,6 +1001,7 @@ stop_rabbitmq_nodes(Config) ->
           fun(NodeConfig) ->
                   stop_rabbitmq_node(Config, NodeConfig)
           end),
+    find_crashes_in_logs(NodeConfigs),
     proplists:delete(rmq_nodes, Config).
 
 stop_rabbitmq_node(Config, NodeConfig) ->
@@ -1019,6 +1022,69 @@ stop_rabbitmq_node(Config, NodeConfig) ->
     end,
     NodeConfig.
 
+find_crashes_in_logs(NodeConfigs) ->
+    ct:pal(
+      "Looking up any crash reports in the nodes' log files. If we find "
+      "some, they will appear below:"),
+    CrashesCount = lists:foldl(
+                     fun(NodeConfig, Total) ->
+                             Count = count_crashes_in_logs(NodeConfig),
+                             Total + Count
+                     end, 0, NodeConfigs),
+    ct:pal("Found ~b crash report(s)", [CrashesCount]),
+    ?assertEqual(0, CrashesCount).
+
+count_crashes_in_logs(NodeConfig) ->
+    LogLocations = ?config(log_locations, NodeConfig),
+    lists:foldl(
+      fun(LogLocation, Total) ->
+              Count = count_crashes_in_log(LogLocation),
+              Total + Count
+      end, 0, LogLocations).
+
+count_crashes_in_log(LogLocation) ->
+    case file:read_file(LogLocation) of
+        {ok, Content} -> count_crashes_in_content(Content);
+        _             -> 0
+    end.
+
+count_crashes_in_content(Content) ->
+    ReOpts = [multiline],
+    Lines = re:split(Content, "^", ReOpts),
+    count_gen_server_terminations(Lines).
+
+count_gen_server_terminations(Lines) ->
+    count_gen_server_terminations(Lines, 0).
+
+count_gen_server_terminations([Line | Rest], Count) ->
+    ReOpts = [{capture, all_but_first, list}],
+    Ret = re:run(
+            Line,
+            "(<[0-9.]+> )[*]{2} Generic server .+ terminating$",
+            ReOpts),
+    case Ret of
+        {match, [Prefix]} ->
+            capture_gen_server_termination(Rest, Prefix, [Line], Count + 1);
+        nomatch ->
+            count_gen_server_terminations(Rest, Count)
+    end;
+count_gen_server_terminations([], Count) ->
+    Count.
+
+capture_gen_server_termination([Line | Rest] = Lines, Prefix, Acc, Count) ->
+    ReOpts = [{capture, none}],
+    Ret = re:run(Line, Prefix ++ "( |\\*|$)", ReOpts),
+    case Ret of
+        match ->
+            capture_gen_server_termination(Rest, Prefix, [Line | Acc], Count);
+        nomatch ->
+            ct:pal("gen_server termination:~n~n~s", [lists:reverse(Acc)]),
+            count_gen_server_terminations(Lines, Count)
+    end;
+capture_gen_server_termination([] = Rest, _Prefix, Acc, Count) ->
+    ct:pal("gen_server termination:~n~n~s", [lists:reverse(Acc)]),
+    count_gen_server_terminations(Rest, Count).
+
 %% -------------------------------------------------------------------
 %% Helpers for partition simulation
 %% -------------------------------------------------------------------