Logger exchange: fix race condition during initialisation

The logger exchange needs to declare the exchange during initialisation,
which requires the metadata store to be ready.
Metadata store initalisation happens in a rabbit boot step after
logger initialisation in the second phase of the prelaunch.
The spawned process that declares the exchange, should also
wait for the store to be ready. Otherwise it enters a loop
trying to decide which store to use which generates a huge log
and delays initialisation:
'Mnesia->Khepri fallback handling: Mnesia function failed because table
 `rabbit_vhost` is missing or read-only. Migration could be in progress;
 waiting for migration to progress and trying again...'

This commit gives it 60 seconds for the metadata store to boot,
and only afterwards tries (and retries if needed) to declare the
exchange.

Author: dcorbacho

deps/rabbit/src/rabbit_db.erl

@@ -14,13 +14,16 @@
 
 -include_lib("rabbit_common/include/logging.hrl").
 
+-define(PT_KEY_INIT_FINISHED, {?MODULE, node(), initialisation_finished}).
+
 -export([init/0,
          reset/0,
          force_reset/0,
          force_load_on_next_boot/0,
          is_virgin_node/0, is_virgin_node/1,
          dir/0,
-         ensure_dir_exists/0]).
+         ensure_dir_exists/0,
+         is_init_finished/0]).
 
 %% Exported to be used by various rabbit_db_* modules
 -export([
@@ -62,6 +65,7 @@ init() ->
                "DB: initialization successeful",
                #{domain => ?RMQLOG_DOMAIN_DB}),
 
+            init_finished(),
             post_init(IsVirgin),
 
             ok;
@@ -72,6 +76,20 @@ init() ->
             Error
     end.
 
+init_finished() ->
+    %% Used during initialisation by rabbit_logger_exchange_h.erl
+    %% If an exchange logger is configured, it needs to declare the
+    %% exchange. For this, it requires the metadata store to be
+    %% initialised. The initialisation happens on a rabbit boot step,
+    %% after the second phase of the prelaunch where the logger is
+    %% configured.
+    %% Using this persistent term the logger exchange can delay
+    %% declaring the exchange until the metadata store is ready.
+    persistent_term:put(?PT_KEY_INIT_FINISHED, true).
+
+is_init_finished() ->
+    persistent_term:get(?PT_KEY_INIT_FINISHED, false).
+
 pre_init(IsVirgin) ->
     Members = rabbit_db_cluster:members(),
     OtherMembers = rabbit_nodes:nodes_excl_me(Members),

deps/rabbit/src/rabbit_logger_exchange_h.erl

@@ -124,11 +124,27 @@ start_setup_proc(#{config := InternalConfig} = Config) ->
     {ok, DefaultVHost} = application:get_env(rabbit, default_vhost),
     Exchange = rabbit_misc:r(DefaultVHost, exchange, ?LOG_EXCH_NAME),
     InternalConfig1 = InternalConfig#{exchange => Exchange},
-
-    Pid = spawn(fun() -> setup_proc(Config#{config => InternalConfig1}) end),
+    Pid = spawn(fun() ->
+                        wait_for_initial_pass(60),
+                        setup_proc(Config#{config => InternalConfig1})
+                end),
     InternalConfig2 = InternalConfig1#{setup_proc => Pid},
     Config#{config => InternalConfig2}.
 
+%% Declaring an exchange requires the metadata store to be ready
+%% which happens on a boot step after the second phase of the prelaunch.
+%% This function waits for the store initialisation.
+wait_for_initial_pass(0) ->
+    ok;
+wait_for_initial_pass(N) ->
+    case rabbit_db:is_init_finished() of
+        false ->
+            timer:sleep(1000),
+            wait_for_initial_pass(N - 1);
+        true ->
+            ok
+    end.
+
 setup_proc(
   #{config := #{exchange := #resource{name = Name,
                                       virtual_host = VHost}}} = Config) ->