Skip to content

Commit 55f724a

Browse files
committed
Logger exchange: fix race condition during initialisation
The logger exchange needs to declare the exchange during initialisation, which requires the metadata store to be ready. Metadata store initalisation happens in a rabbit boot step after logger initialisation in the second phase of the prelaunch. The spawned process that declares the exchange, should also wait for the store to be ready. Otherwise it enters a loop trying to decide which store to use which generates a huge log and delays initialisation: 'Mnesia->Khepri fallback handling: Mnesia function failed because table `rabbit_vhost` is missing or read-only. Migration could be in progress; waiting for migration to progress and trying again...' This commit gives it 60 seconds for the metadata store to boot, and only afterwards tries (and retries if needed) to declare the exchange.
1 parent 3d8945f commit 55f724a

File tree

3 files changed

+43
-3
lines changed

3 files changed

+43
-3
lines changed

deps/rabbit/src/rabbit.erl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1079,6 +1079,7 @@ stop(State) ->
10791079
[] -> rabbit_prelaunch:set_stop_reason(normal);
10801080
_ -> rabbit_prelaunch:set_stop_reason(State)
10811081
end,
1082+
rabbit_db:clear_init_finished(),
10821083
rabbit_boot_state:set(stopped),
10831084
ok.
10841085

deps/rabbit/src/rabbit_db.erl

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,17 @@
1414

1515
-include_lib("rabbit_common/include/logging.hrl").
1616

17+
-define(PT_KEY_INIT_FINISHED, {?MODULE, node(), initialisation_finished}).
18+
1719
-export([init/0,
1820
reset/0,
1921
force_reset/0,
2022
force_load_on_next_boot/0,
2123
is_virgin_node/0, is_virgin_node/1,
2224
dir/0,
23-
ensure_dir_exists/0]).
25+
ensure_dir_exists/0,
26+
is_init_finished/0,
27+
clear_init_finished/0]).
2428

2529
%% Exported to be used by various rabbit_db_* modules
2630
-export([
@@ -62,6 +66,7 @@ init() ->
6266
"DB: initialization successeful",
6367
#{domain => ?RMQLOG_DOMAIN_DB}),
6468

69+
init_finished(),
6570
post_init(IsVirgin),
6671

6772
ok;
@@ -102,6 +107,24 @@ init_using_khepri() ->
102107
#{domain => ?RMQLOG_DOMAIN_DB})
103108
end.
104109

110+
init_finished() ->
111+
%% Used during initialisation by rabbit_logger_exchange_h.erl
112+
%% If an exchange logger is configured, it needs to declare the
113+
%% exchange. For this, it requires the metadata store to be
114+
%% initialised. The initialisation happens on a rabbit boot step,
115+
%% after the second phase of the prelaunch where the logger is
116+
%% configured.
117+
%% Using this persistent term the logger exchange can delay
118+
%% declaring the exchange until the metadata store is ready.
119+
persistent_term:put(?PT_KEY_INIT_FINISHED, true).
120+
121+
is_init_finished() ->
122+
persistent_term:get(?PT_KEY_INIT_FINISHED, false).
123+
124+
clear_init_finished() ->
125+
_ = persistent_term:erase(?PT_KEY_INIT_FINISHED),
126+
ok.
127+
105128
-spec reset() -> Ret when
106129
Ret :: ok.
107130
%% @doc Resets the database and the node.

deps/rabbit/src/rabbit_logger_exchange_h.erl

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,11 +124,27 @@ start_setup_proc(#{config := InternalConfig} = Config) ->
124124
{ok, DefaultVHost} = application:get_env(rabbit, default_vhost),
125125
Exchange = rabbit_misc:r(DefaultVHost, exchange, ?LOG_EXCH_NAME),
126126
InternalConfig1 = InternalConfig#{exchange => Exchange},
127-
128-
Pid = spawn(fun() -> setup_proc(Config#{config => InternalConfig1}) end),
127+
Pid = spawn(fun() ->
128+
wait_for_initial_pass(60),
129+
setup_proc(Config#{config => InternalConfig1})
130+
end),
129131
InternalConfig2 = InternalConfig1#{setup_proc => Pid},
130132
Config#{config => InternalConfig2}.
131133

134+
%% Declaring an exchange requires the metadata store to be ready
135+
%% which happens on a boot step after the second phase of the prelaunch.
136+
%% This function waits for the store initialisation.
137+
wait_for_initial_pass(0) ->
138+
ok;
139+
wait_for_initial_pass(N) ->
140+
case rabbit_db:is_init_finished() of
141+
false ->
142+
timer:sleep(1000),
143+
wait_for_initial_pass(N - 1);
144+
true ->
145+
ok
146+
end.
147+
132148
setup_proc(
133149
#{config := #{exchange := #resource{name = Name,
134150
virtual_host = VHost}}} = Config) ->

0 commit comments

Comments
 (0)