Skip to content

Commit a7ebb26

Browse files
michaelklishinmergify[bot]
michaelklishin
authored and
mergify[bot]
committed
By @Ayanda-D: new CLI health check that detects QQs without an elected reachable leader #13433 (#13487)
* Implement rabbitmq-queues leader_health_check command for quorum queues (cherry picked from commit c26edbe) * Tests for rabbitmq-queues leader_health_check command (cherry picked from commit 6cc03b0) * Ensure calling ParentPID in leader health check execution and reuse and extend formatting API, with amqqueue:to_printable/2 (cherry picked from commit 76d66a1) * Extend core leader health check tests and update badrpc error handling in cli tests (cherry picked from commit 857e2a7) * Refactor leader_health_check command validators and ignore vhost arg (cherry picked from commit 6cf9339) * Update leader_health_check_command description and banner (cherry picked from commit 96b8bce) * Improve output formatting for healthy leaders and support silent mode in rabbitmq-queues leader_health_check command (cherry picked from commit 239a69b) * Support global flag to run leader health check for all queues in all vhosts on local node (cherry picked from commit 48ba3e1) * Return immediately for leader health checks on empty vhosts (cherry picked from commit 7873737) * Rename leader health check timeout refs (cherry picked from commit b7dec89) * Update banner message for global leader health check (cherry picked from commit c7da4d5) * QQ leader-health-check: check_process_limit_safety before spawning leader checks (cherry picked from commit 1736845) * Log leader health check result in broker logs (if any leaderless queues) (cherry picked from commit 1084179) * Ensure check_passed result for leader health internal calls) (cherry picked from commit 68739a6) * Extend CLI format output to process check_passed payload (cherry picked from commit 5f5e992) * Format leader healthcheck result log and function exports (cherry picked from commit ebffd7d) * Change leader_health_check command scope from queues to diagnostics (cherry picked from commit 663fc98) * Update (c) line year (cherry picked from commit df82f12) * Rename command to check_for_quorum_queues_without_an_elected_leader and use across_all_vhosts option for global checks (cherry picked from commit b2acbae) * Use rabbit_db_queue for qq leader health check lookups and introduce rabbit_db_queue:get_all_by_type_and_vhost/2. Update leader health check timeout to 5s and process limit threshold to 20% of node's process_limit. (cherry picked from commit 7a8e166) * Update tests: quorum_queue_SUITE and rabbit_db_queue_SUITE (cherry picked from commit 9bdb81f) * Fix typo (cli test module) (cherry picked from commit 6158568) * Small refactor - simpler final leader health check result return on function head match (cherry picked from commit ea07938) * Clear dialyzer warning & fix type spec (cherry picked from commit a45aa81) * Ignore result without strict match to avoid diayzer warning (cherry picked from commit bb43c0b) * 'rabbitmq-diagnostics check_for_quorum_queues_without_an_elected_leader' documentation edits (cherry picked from commit 845230b) * 'rabbitmq-diagnostics check_for_quorum_queues_without_an_elected_leader' output copywriting (cherry picked from commit 235f43b) * diagnostics check_for_quorum_queues_without_an_elected_leader: behave like a health check w.r.t. error reporting (cherry picked from commit db73767) * check_for_quorum_queues_without_an_elected_leader: handle --quiet and --silent plus simplify function heads. References #13433. (cherry picked from commit 7b39231) --------- Co-authored-by: Ayanda Dube <[email protected]> (cherry picked from commit 09f1ab4) (cherry picked from commit e1d7481) # Conflicts: # deps/rabbit/test/quorum_queue_SUITE.erl
1 parent a7ed4cf commit a7ebb26

File tree

8 files changed

+437
-0
lines changed

8 files changed

+437
-0
lines changed

‎deps/rabbit/src/amqqueue.erl

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,12 @@
6666
pattern_match_on_type/1,
6767
pattern_match_on_durable/1,
6868
pattern_match_on_type_and_durable/2,
69+
pattern_match_on_type_and_vhost/2,
6970
reset_decorators/1,
7071
set_immutable/1,
7172
qnode/1,
7273
to_printable/1,
74+
to_printable/2,
7375
macros/0]).
7476

7577
-define(record_version, amqqueue_v2).
@@ -531,6 +533,12 @@ pattern_match_on_durable(IsDurable) ->
531533
pattern_match_on_type_and_durable(Type, IsDurable) ->
532534
#amqqueue{type = Type, durable = IsDurable, _ = '_'}.
533535

536+
-spec pattern_match_on_type_and_vhost(atom(), binary()) ->
537+
amqqueue_pattern().
538+
539+
pattern_match_on_type_and_vhost(Type, VHost) ->
540+
#amqqueue{type = Type, vhost = VHost, _ = '_'}.
541+
534542
-spec reset_decorators(amqqueue()) -> amqqueue().
535543

536544
reset_decorators(#amqqueue{} = Queue) ->
@@ -564,6 +572,14 @@ to_printable(#amqqueue{name = QName = #resource{name = Name},
564572
<<"virtual_host">> => VHost,
565573
<<"type">> => Type}.
566574

575+
-spec to_printable(rabbit_types:r(queue), atom() | binary()) -> #{binary() => any()}.
576+
to_printable(QName = #resource{name = Name, virtual_host = VHost}, Type) ->
577+
_ = rabbit_queue_type:discover(Type),
578+
#{<<"readable_name">> => rabbit_data_coercion:to_binary(rabbit_misc:rs(QName)),
579+
<<"name">> => Name,
580+
<<"virtual_host">> => VHost,
581+
<<"type">> => Type}.
582+
567583
% private
568584

569585
macros() ->

‎deps/rabbit/src/rabbit_db_queue.erl

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
get_all/0,
2222
get_all/1,
2323
get_all_by_type/1,
24+
get_all_by_type_and_vhost/2,
2425
get_all_by_type_and_node/3,
2526
list/0,
2627
count/0,
@@ -829,6 +830,28 @@ get_all_by_type(Type) ->
829830
khepri => fun() -> get_all_by_pattern_in_khepri(Pattern) end
830831
}).
831832

833+
%% -------------------------------------------------------------------
834+
%% get_all_by_type_and_vhost().
835+
%% -------------------------------------------------------------------
836+
837+
-spec get_all_by_type_and_vhost(Type, VHost) -> [Queue] when
838+
Type :: atom(),
839+
VHost :: binary(),
840+
Queue :: amqqueue:amqqueue().
841+
842+
%% @doc Gets all queues belonging to the given type and vhost
843+
%%
844+
%% @returns a list of queue records.
845+
%%
846+
%% @private
847+
848+
get_all_by_type_and_vhost(Type, VHost) ->
849+
Pattern = amqqueue:pattern_match_on_type_and_vhost(Type, VHost),
850+
rabbit_khepri:handle_fallback(
851+
#{mnesia => fun() -> get_all_by_pattern_in_mnesia(Pattern) end,
852+
khepri => fun() -> get_all_by_pattern_in_khepri(Pattern) end
853+
}).
854+
832855
get_all_by_pattern_in_mnesia(Pattern) ->
833856
rabbit_db:list_in_mnesia(?MNESIA_TABLE, Pattern).
834857

‎deps/rabbit/src/rabbit_quorum_queue.erl

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@
8282
file_handle_other_reservation/0,
8383
file_handle_release_reservation/0]).
8484

85+
-export([leader_health_check/2,
86+
run_leader_health_check/4]).
87+
8588
-ifdef(TEST).
8689
-export([filter_promotable/2,
8790
ra_machine_config/1]).
@@ -144,6 +147,8 @@
144147
-define(SNAPSHOT_INTERVAL, 8192). %% the ra default is 4096
145148
% -define(UNLIMITED_PREFETCH_COUNT, 2000). %% something large for ra
146149
-define(MIN_CHECKPOINT_INTERVAL, 8192). %% the ra default is 16384
150+
-define(LEADER_HEALTH_CHECK_TIMEOUT, 5_000).
151+
-define(GLOBAL_LEADER_HEALTH_CHECK_TIMEOUT, 60_000).
147152

148153
%%----------- QQ policies ---------------------------------------------------
149154

@@ -2120,3 +2125,75 @@ file_handle_other_reservation() ->
21202125
file_handle_release_reservation() ->
21212126
ok.
21222127

2128+
leader_health_check(QueueNameOrRegEx, VHost) ->
2129+
%% Set a process limit threshold to 20% of ErlangVM process limit, beyond which
2130+
%% we cannot spawn any new processes for executing QQ leader health checks.
2131+
ProcessLimitThreshold = round(0.2 * erlang:system_info(process_limit)),
2132+
2133+
leader_health_check(QueueNameOrRegEx, VHost, ProcessLimitThreshold).
2134+
2135+
leader_health_check(QueueNameOrRegEx, VHost, ProcessLimitThreshold) ->
2136+
Qs =
2137+
case VHost of
2138+
across_all_vhosts ->
2139+
rabbit_db_queue:get_all_by_type(?MODULE);
2140+
VHost when is_binary(VHost) ->
2141+
rabbit_db_queue:get_all_by_type_and_vhost(?MODULE, VHost)
2142+
end,
2143+
check_process_limit_safety(length(Qs), ProcessLimitThreshold),
2144+
ParentPID = self(),
2145+
HealthCheckRef = make_ref(),
2146+
HealthCheckPids =
2147+
lists:flatten(
2148+
[begin
2149+
{resource, _VHostN, queue, QueueName} = QResource = amqqueue:get_name(Q),
2150+
case re:run(QueueName, QueueNameOrRegEx, [{capture, none}]) of
2151+
match ->
2152+
{ClusterName, _} = rabbit_amqqueue:pid_of(Q),
2153+
_Pid = spawn(fun() -> run_leader_health_check(ClusterName, QResource, HealthCheckRef, ParentPID) end);
2154+
_ ->
2155+
[]
2156+
end
2157+
end || Q <- Qs, amqqueue:get_type(Q) == ?MODULE]),
2158+
Result = wait_for_leader_health_checks(HealthCheckRef, length(HealthCheckPids), []),
2159+
_ = spawn(fun() -> maybe_log_leader_health_check_result(Result) end),
2160+
Result.
2161+
2162+
run_leader_health_check(ClusterName, QResource, HealthCheckRef, From) ->
2163+
Leader = ra_leaderboard:lookup_leader(ClusterName),
2164+
2165+
%% Ignoring result here is required to clear a diayzer warning.
2166+
_ =
2167+
case ra_server_proc:ping(Leader, ?LEADER_HEALTH_CHECK_TIMEOUT) of
2168+
{pong,leader} ->
2169+
From ! {ok, HealthCheckRef, QResource};
2170+
_ ->
2171+
From ! {error, HealthCheckRef, QResource}
2172+
end,
2173+
ok.
2174+
2175+
wait_for_leader_health_checks(_Ref, 0, UnhealthyAcc) -> UnhealthyAcc;
2176+
wait_for_leader_health_checks(Ref, N, UnhealthyAcc) ->
2177+
receive
2178+
{ok, Ref, _QResource} ->
2179+
wait_for_leader_health_checks(Ref, N - 1, UnhealthyAcc);
2180+
{error, Ref, QResource} ->
2181+
wait_for_leader_health_checks(Ref, N - 1, [amqqueue:to_printable(QResource, ?MODULE) | UnhealthyAcc])
2182+
after
2183+
?GLOBAL_LEADER_HEALTH_CHECK_TIMEOUT ->
2184+
UnhealthyAcc
2185+
end.
2186+
2187+
check_process_limit_safety(QCount, ProcessLimitThreshold) ->
2188+
case (erlang:system_info(process_count) + QCount) >= ProcessLimitThreshold of
2189+
true ->
2190+
rabbit_log:warning("Leader health check not permitted, process limit threshold will be exceeded."),
2191+
throw({error, leader_health_check_process_limit_exceeded});
2192+
false ->
2193+
ok
2194+
end.
2195+
2196+
maybe_log_leader_health_check_result([]) -> ok;
2197+
maybe_log_leader_health_check_result(Result) ->
2198+
Qs = lists:map(fun(R) -> catch maps:get(<<"readable_name">>, R) end, Result),
2199+
rabbit_log:warning("Leader health check result (unhealthy leaders detected): ~tp", [Qs]).

‎deps/rabbit/test/quorum_queue_SUITE.erl

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,13 @@ all_tests() ->
188188
priority_queue_fifo,
189189
priority_queue_2_1_ratio,
190190
requeue_multiple_true,
191+
<<<<<<< HEAD
191192
requeue_multiple_false
193+
=======
194+
requeue_multiple_false,
195+
subscribe_from_each,
196+
leader_health_check
197+
>>>>>>> e1d748131 (By @Ayanda-D: new CLI health check that detects QQs without an elected reachable leader #13433 (#13487))
192198
].
193199

194200
memory_tests() ->
@@ -4106,6 +4112,129 @@ amqpl_headers(Config) ->
41064112
ok = amqp_channel:cast(Ch, #'basic.ack'{delivery_tag = DeliveryTag,
41074113
multiple = true}).
41084114

4115+
leader_health_check(Config) ->
4116+
VHost1 = <<"vhost1">>,
4117+
VHost2 = <<"vhost2">>,
4118+
4119+
set_up_vhost(Config, VHost1),
4120+
set_up_vhost(Config, VHost2),
4121+
4122+
%% check empty vhost
4123+
?assertEqual([],
4124+
rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4125+
[<<".*">>, VHost1])),
4126+
?assertEqual([],
4127+
rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4128+
[<<".*">>, across_all_vhosts])),
4129+
4130+
Conn1 = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 0, VHost1),
4131+
{ok, Ch1} = amqp_connection:open_channel(Conn1),
4132+
4133+
Conn2 = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 0, VHost2),
4134+
{ok, Ch2} = amqp_connection:open_channel(Conn2),
4135+
4136+
Qs1 = [<<"Q.1">>, <<"Q.2">>, <<"Q.3">>],
4137+
Qs2 = [<<"Q.4">>, <<"Q.5">>, <<"Q.6">>],
4138+
4139+
%% in vhost1
4140+
[?assertEqual({'queue.declare_ok', Q, 0, 0},
4141+
declare(Ch1, Q, [{<<"x-queue-type">>, longstr, <<"quorum">>}]))
4142+
|| Q <- Qs1],
4143+
4144+
%% in vhost2
4145+
[?assertEqual({'queue.declare_ok', Q, 0, 0},
4146+
declare(Ch2, Q, [{<<"x-queue-type">>, longstr, <<"quorum">>}]))
4147+
|| Q <- Qs2],
4148+
4149+
%% test sucessful health checks in vhost1, vhost2, across_all_vhosts
4150+
?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4151+
[<<".*">>, VHost1])),
4152+
?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4153+
[<<"Q.*">>, VHost1])),
4154+
[?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4155+
[Q, VHost1])) || Q <- Qs1],
4156+
4157+
?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4158+
[<<".*">>, VHost2])),
4159+
?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4160+
[<<"Q.*">>, VHost2])),
4161+
[?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4162+
[Q, VHost2])) || Q <- Qs2],
4163+
4164+
?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4165+
[<<".*">>, across_all_vhosts])),
4166+
?assertEqual([], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4167+
[<<"Q.*">>, across_all_vhosts])),
4168+
4169+
%% clear leaderboard
4170+
Qs = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_amqqueue, list, []),
4171+
4172+
[{_Q1_ClusterName, _Q1Res},
4173+
{_Q2_ClusterName, _Q2Res},
4174+
{_Q3_ClusterName, _Q3Res},
4175+
{_Q4_ClusterName, _Q4Res},
4176+
{_Q5_ClusterName, _Q5Res},
4177+
{_Q6_ClusterName, _Q6Res}] = QQ_Clusters =
4178+
lists:usort(
4179+
[begin
4180+
{ClusterName, _} = amqqueue:get_pid(Q),
4181+
{ClusterName, amqqueue:get_name(Q)}
4182+
end
4183+
|| Q <- Qs, amqqueue:get_type(Q) == rabbit_quorum_queue]),
4184+
4185+
[Q1Data, Q2Data, Q3Data, Q4Data, Q5Data, Q6Data] = QQ_Data =
4186+
[begin
4187+
rabbit_ct_broker_helpers:rpc(Config, 0, ra_leaderboard, clear, [Q_ClusterName]),
4188+
_QData = amqqueue:to_printable(Q_Res, rabbit_quorum_queue)
4189+
end
4190+
|| {Q_ClusterName, Q_Res} <- QQ_Clusters],
4191+
4192+
%% test failed health checks in vhost1, vhost2, across_all_vhosts
4193+
?assertEqual([Q1Data], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4194+
[<<"Q.1">>, VHost1])),
4195+
?assertEqual([Q2Data], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4196+
[<<"Q.2">>, VHost1])),
4197+
?assertEqual([Q3Data], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4198+
[<<"Q.3">>, VHost1])),
4199+
?assertEqual([Q1Data, Q2Data, Q3Data],
4200+
lists:usort(rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4201+
[<<".*">>, VHost1]))),
4202+
?assertEqual([Q1Data, Q2Data, Q3Data],
4203+
lists:usort(rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4204+
[<<"Q.*">>, VHost1]))),
4205+
4206+
?assertEqual([Q4Data], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4207+
[<<"Q.4">>, VHost2])),
4208+
?assertEqual([Q5Data], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4209+
[<<"Q.5">>, VHost2])),
4210+
?assertEqual([Q6Data], rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4211+
[<<"Q.6">>, VHost2])),
4212+
?assertEqual([Q4Data, Q5Data, Q6Data],
4213+
lists:usort(rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4214+
[<<".*">>, VHost2]))),
4215+
?assertEqual([Q4Data, Q5Data, Q6Data],
4216+
lists:usort(rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4217+
[<<"Q.*">>, VHost2]))),
4218+
4219+
?assertEqual(QQ_Data,
4220+
lists:usort(rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4221+
[<<"Q.*">>, across_all_vhosts]))),
4222+
?assertEqual(QQ_Data,
4223+
lists:usort(rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_quorum_queue, leader_health_check,
4224+
[<<"Q.*">>, across_all_vhosts]))),
4225+
4226+
%% cleanup
4227+
[?assertMatch(#'queue.delete_ok'{},
4228+
amqp_channel:call(Ch1, #'queue.delete'{queue = Q}))
4229+
|| Q <- Qs1],
4230+
[?assertMatch(#'queue.delete_ok'{},
4231+
amqp_channel:call(Ch1, #'queue.delete'{queue = Q}))
4232+
|| Q <- Qs2],
4233+
4234+
amqp_connection:close(Conn1),
4235+
amqp_connection:close(Conn2).
4236+
4237+
41094238
leader_locator_client_local(Config) ->
41104239
[Server1 | _] = Servers = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
41114240
Q = ?config(queue_name, Config),
@@ -4426,6 +4555,11 @@ declare_passive(Ch, Q, Args) ->
44264555
auto_delete = false,
44274556
passive = true,
44284557
arguments = Args}).
4558+
4559+
set_up_vhost(Config, VHost) ->
4560+
rabbit_ct_broker_helpers:add_vhost(Config, VHost),
4561+
rabbit_ct_broker_helpers:set_full_permissions(Config, <<"guest">>, VHost).
4562+
44294563
assert_queue_type(Server, Q, Expected) ->
44304564
assert_queue_type(Server, <<"/">>, Q, Expected).
44314565

‎deps/rabbit/test/rabbit_db_queue_SUITE.erl

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ all_tests() ->
3535
get_all,
3636
get_all_by_vhost,
3737
get_all_by_type,
38+
get_all_by_type_and_vhost,
3839
get_all_by_type_and_node,
3940
list,
4041
count,
@@ -198,6 +199,30 @@ get_all_by_type1(_Config) ->
198199
?assertEqual([Q4], rabbit_db_queue:get_all_by_type(rabbit_stream_queue)),
199200
passed.
200201

202+
get_all_by_type_and_vhost(Config) ->
203+
passed = rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, get_all_by_type_and_vhost1, [Config]).
204+
205+
get_all_by_type_and_vhost1(_Config) ->
206+
VHost1 = <<"carrots">>,
207+
VHost2 = <<"cabage">>,
208+
QName = rabbit_misc:r(VHost1, queue, <<"test-queue">>),
209+
QName2 = rabbit_misc:r(VHost2, queue, <<"test-queue2">>),
210+
QName3 = rabbit_misc:r(VHost2, queue, <<"test-queue3">>),
211+
QName4 = rabbit_misc:r(VHost1, queue, <<"test-queue4">>),
212+
Q = new_queue(QName, rabbit_classic_queue),
213+
Q2 = new_queue(QName2, rabbit_quorum_queue),
214+
Q3 = new_queue(QName3, rabbit_quorum_queue),
215+
Q4 = new_queue(QName4, rabbit_stream_queue),
216+
Quorum = lists:sort([Q2, Q3]),
217+
?assertEqual([], rabbit_db_queue:get_all_by_type_and_vhost(rabbit_classic_queue, VHost1)),
218+
?assertEqual([], lists:sort(rabbit_db_queue:get_all_by_type_and_vhost(rabbit_quorum_queue, VHost2))),
219+
?assertEqual([], rabbit_db_queue:get_all_by_type_and_vhost(rabbit_stream_queue, VHost1)),
220+
set_list([Q, Q2, Q3, Q4]),
221+
?assertEqual([Q], rabbit_db_queue:get_all_by_type_and_vhost(rabbit_classic_queue, VHost1)),
222+
?assertEqual(Quorum, lists:sort(rabbit_db_queue:get_all_by_type_and_vhost(rabbit_quorum_queue, VHost2))),
223+
?assertEqual([Q4], rabbit_db_queue:get_all_by_type_and_vhost(rabbit_stream_queue, VHost1)),
224+
passed.
225+
201226
get_all_by_type_and_node(Config) ->
202227
passed = rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, get_all_by_type_and_node1, [Config]).
203228

‎deps/rabbitmq_cli/lib/rabbitmq/cli/core/output.ex

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ defmodule RabbitMQ.CLI.Core.Output do
1818
:ok
1919
end
2020

21+
def format_output({:ok, :check_passed, output}, formatter, options) do
22+
{:ok, formatter.format_output(output, options)}
23+
end
24+
2125
def format_output({:ok, output}, formatter, options) do
2226
{:ok, formatter.format_output(output, options)}
2327
end

0 commit comments

Comments
 (0)