Skip to content

Commit 07196e2

Browse files
dcorbachomichaelklishin
dcorbacho
authored and
michaelklishin
committed
Reduce the number of metrics served by GET /api/queues
Introduce GET /api/queues/detailed endpoint Just removed garbage_collection, idle_since and any 'null' value /api/queues with 10k classic queues returns 7.4MB of data /api/queues/detailed with 10k classic queues returns 11MB of data This sits behind a new feature flag, required to collect data from all nodes: detailed_queues_endpoint
1 parent 55bd90c commit 07196e2

15 files changed

+247
-31
lines changed

‎deps/rabbitmq_management/priv/www/api/index.html

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,13 +485,22 @@ <h2>Reference</h2>
485485
<td></td>
486486
<td></td>
487487
<td class="path">/api/queues</td>
488-
<td>A list of all queues. Use <a href="#pagination">pagination parameters</a> to filter queues.
488+
<td>A list of all queues returning a reduced set of fields. Use <a href="#pagination">pagination parameters</a> to filter queues.
489489
The parameter <code>enable_queue_totals=true</code> can be used in combination with the
490490
<code>disable_stats=true</code> parameter to return a reduced set of fields and significantly
491491
reduce the amount of data returned by this endpoint. That in turn can significantly reduce
492492
CPU and bandwidth footprint of such requests.
493493
</td>
494494
</tr>
495+
<tr>
496+
<td>X</td>
497+
<td></td>
498+
<td></td>
499+
<td></td>
500+
<td class="path">/api/queues/detailed</td>
501+
<td>A list of all queues containing all available information about the queues. Use <a href="#pagination">pagination parameters</a> to filter queues.
502+
</td>
503+
</tr>
495504
<tr>
496505
<td>X</td>
497506
<td></td>

‎deps/rabbitmq_management/src/rabbit_mgmt_app.erl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@
2626
[{description, "Imports definition file at management.load_definitions"},
2727
{mfa, {rabbit_mgmt_load_definitions, boot, []}}]}).
2828

29+
-rabbit_feature_flag(
30+
{detailed_queues_endpoint,
31+
#{desc => "Add a detailed queues HTTP API endpoint. Reduce number of metrics in the default endpoint.",
32+
stability => stable,
33+
depends_on => [feature_flags_v2]
34+
}}).
35+
2936
start(_Type, _StartArgs) ->
3037
case rabbit_mgmt_agent_config:is_metrics_collector_enabled() of
3138
true ->

‎deps/rabbitmq_management/src/rabbit_mgmt_db.erl

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,13 +145,20 @@ augment_exchanges(Xs, Ranges, _) ->
145145

146146
%% we can only cache if no ranges are requested.
147147
%% The mgmt ui doesn't use ranges for queue listings
148-
-spec augment_queues([proplists:proplist()], ranges(), basic | full) -> any().
149-
augment_queues(Qs, ?NO_RANGES = Ranges, basic) ->
148+
-spec augment_queues([proplists:proplist()], ranges(), basic | detailed | full) -> any().
149+
augment_queues(Qs, ?NO_RANGES = Ranges, basic) ->
150+
submit_cached(queues,
151+
fun(Interval, Queues) ->
152+
list_basic_queue_stats(Ranges, Queues, Interval)
153+
end, Qs, max(60000, length(Qs) * 2));
154+
augment_queues(Qs, ?NO_RANGES = Ranges, detailed) ->
150155
submit_cached(queues,
151156
fun(Interval, Queues) ->
152157
list_queue_stats(Ranges, Queues, Interval)
153158
end, Qs, max(60000, length(Qs) * 2));
154-
augment_queues(Qs, Ranges, basic) ->
159+
augment_queues(Qs, Ranges, basic) ->
160+
submit(fun(Interval) -> list_basic_queue_stats(Ranges, Qs, Interval) end);
161+
augment_queues(Qs, Ranges, detailed) ->
155162
submit(fun(Interval) -> list_queue_stats(Ranges, Qs, Interval) end);
156163
augment_queues(Qs, Ranges, _) ->
157164
submit(fun(Interval) -> detail_queue_stats(Ranges, Qs, Interval) end).
@@ -349,8 +356,16 @@ consumers_stats(VHost) ->
349356
-spec list_queue_stats(ranges(), [proplists:proplist()], integer()) ->
350357
[proplists:proplist()].
351358
list_queue_stats(Ranges, Objs, Interval) ->
359+
list_queue_stats(Ranges, Objs, Interval, all_list_queue_data).
360+
361+
-spec list_basic_queue_stats(ranges(), [proplists:proplist()], integer()) ->
362+
[proplists:proplist()].
363+
list_basic_queue_stats(Ranges, Objs, Interval) ->
364+
list_queue_stats(Ranges, Objs, Interval, all_list_basic_queue_data).
365+
366+
list_queue_stats(Ranges, Objs, Interval, Fun) ->
352367
Ids = [id_lookup(queue_stats, Obj) || Obj <- Objs],
353-
DataLookup = get_data_from_nodes({rabbit_mgmt_data, all_list_queue_data, [Ids, Ranges]}),
368+
DataLookup = get_data_from_nodes({rabbit_mgmt_data, Fun, [Ids, Ranges]}),
354369
adjust_hibernated_memory_use(
355370
[begin
356371
Id = id_lookup(queue_stats, Obj),

‎deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ dispatcher() ->
134134
{"/exchanges/:vhost/:exchange/bindings/source", rabbit_mgmt_wm_bindings, [exchange_source]},
135135
{"/exchanges/:vhost/:exchange/bindings/destination", rabbit_mgmt_wm_bindings, [exchange_destination]},
136136
{"/queues", rabbit_mgmt_wm_queues, []},
137+
{"/queues/detailed", rabbit_mgmt_wm_queues, [detailed]},
137138
{"/queues/:vhost", rabbit_mgmt_wm_queues, []},
138139
{"/queues/:vhost/:queue", rabbit_mgmt_wm_queue, []},
139140
{"/queues/:vhost/:destination/bindings", rabbit_mgmt_wm_bindings, [queue]},

‎deps/rabbitmq_management/src/rabbit_mgmt_wm_queues.erl

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,36 +23,41 @@
2323

2424
%%--------------------------------------------------------------------
2525

26-
init(Req, _State) ->
27-
{cowboy_rest, rabbit_mgmt_headers:set_common_permission_headers(Req, ?MODULE), #context{}}.
26+
init(Req, State) ->
27+
Mode = case State of
28+
[] -> basic;
29+
[detailed] -> detailed
30+
end,
31+
{cowboy_rest, rabbit_mgmt_headers:set_common_permission_headers(Req, ?MODULE), {Mode, #context{}}}.
2832

2933
variances(Req, Context) ->
3034
{[<<"accept-encoding">>, <<"origin">>], Req, Context}.
3135

3236
content_types_provided(ReqData, Context) ->
3337
{rabbit_mgmt_util:responder_map(to_json), ReqData, Context}.
3438

35-
resource_exists(ReqData, Context) ->
39+
resource_exists(ReqData, {Mode, Context}) ->
3640
{case queues0(ReqData) of
3741
vhost_not_found -> false;
3842
_ -> true
39-
end, ReqData, Context}.
43+
end, ReqData, {Mode, Context}}.
4044

41-
to_json(ReqData, Context) ->
45+
to_json(ReqData, {Mode, Context}) ->
4246
try
4347
Basic = basic_vhost_filtered(ReqData, Context),
4448
Data = rabbit_mgmt_util:augment_resources(Basic, ?DEFAULT_SORT,
4549
?BASIC_COLUMNS, ReqData,
46-
Context, fun augment/2),
47-
rabbit_mgmt_util:reply(Data, ReqData, Context)
50+
Context, augment(Mode)),
51+
rabbit_mgmt_util:reply(Data, ReqData, {Mode, Context})
4852
catch
4953
{error, invalid_range_parameters, Reason} ->
5054
rabbit_mgmt_util:bad_request(iolist_to_binary(Reason), ReqData,
51-
Context)
55+
{Mode, Context})
5256
end.
5357

54-
is_authorized(ReqData, Context) ->
55-
rabbit_mgmt_util:is_authorized_vhost(ReqData, Context).
58+
is_authorized(ReqData, {Mode, Context}) ->
59+
{Res, RD2, C2} = rabbit_mgmt_util:is_authorized_vhost(ReqData, Context),
60+
{Res, RD2, {Mode, C2}}.
5661

5762
%%--------------------------------------------------------------------
5863
%% Exported functions
@@ -77,18 +82,31 @@ basic(ReqData) ->
7782
end.
7883

7984
augmented(ReqData, Context) ->
80-
augment(rabbit_mgmt_util:filter_vhost(basic(ReqData), ReqData, Context), ReqData).
85+
Fun = augment(basic),
86+
Fun(rabbit_mgmt_util:filter_vhost(basic(ReqData), ReqData, Context), ReqData).
8187

8288
%%--------------------------------------------------------------------
8389
%% Private helpers
8490

85-
augment(Basic, ReqData) ->
86-
case rabbit_mgmt_util:disable_stats(ReqData) of
87-
false ->
88-
rabbit_mgmt_db:augment_queues(Basic, rabbit_mgmt_util:range_ceil(ReqData),
89-
basic);
90-
true ->
91-
Basic
91+
augment(Mode) ->
92+
fun(Basic, ReqData) ->
93+
case rabbit_mgmt_util:disable_stats(ReqData) of
94+
false ->
95+
%% The reduced endpoint needs to sit behind a feature flag, as it calls a different data aggregation function that is used against all cluster nodes.
96+
%% Data can be collected locally even if other nodes in the cluster do not, it's just a local ETS table. But it can't be queried until all nodes enable the FF.
97+
IsEnabled = rabbit_feature_flags:is_enabled(detailed_queues_endpoint),
98+
Stats = case {IsEnabled, Mode, rabbit_mgmt_util:columns(ReqData)} of
99+
{false, _, _} -> detailed;
100+
{_, detailed, _} -> detailed;
101+
{_, _, all} -> basic;
102+
_ -> detailed
103+
end,
104+
rabbit_log:warning("AUGMENT QUEUES is_enabled ~p stats ~p", [IsEnabled, Stats]),
105+
rabbit_mgmt_db:augment_queues(Basic, rabbit_mgmt_util:range_ceil(ReqData),
106+
Stats);
107+
true ->
108+
Basic
109+
end
92110
end.
93111

94112
basic_vhost_filtered(ReqData, Context) ->

‎deps/rabbitmq_management/test/clustering_SUITE.erl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,7 @@ disable_plugin(Config) ->
837837

838838
clear_all_table_data() ->
839839
[ets:delete_all_objects(T) || {T, _} <- ?CORE_TABLES],
840-
[ets:delete_all_objects(T) || {T, _} <- ?TABLES],
840+
rabbit_mgmt_storage:reset(),
841841
[gen_server:call(P, purge_cache)
842842
|| {_, P, _, _} <- supervisor:which_children(rabbit_mgmt_db_cache_sup)],
843843
send_to_all_collectors(purge_old_stats).

‎deps/rabbitmq_management/test/clustering_prop_SUITE.erl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ force_all() ->
170170

171171
clear_all_table_data() ->
172172
[ets:delete_all_objects(T) || {T, _} <- ?CORE_TABLES],
173-
[ets:delete_all_objects(T) || {T, _} <- ?TABLES],
173+
rabbit_mgmt_storage:reset(),
174174
[gen_server:call(P, purge_cache)
175175
|| {_, P, _, _} <- supervisor:which_children(rabbit_mgmt_db_cache_sup)].
176176

‎deps/rabbitmq_management/test/rabbit_mgmt_http_SUITE.erl

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ all_tests() -> [
114114
connections_channels_pagination_test,
115115
exchanges_pagination_test,
116116
exchanges_pagination_permissions_test,
117+
queues_detailed_test,
117118
queue_pagination_test,
118119
queue_pagination_columns_test,
119120
queues_pagination_permissions_test,
@@ -224,6 +225,13 @@ init_per_testcase(Testcase = disabled_qq_replica_opers_test, Config) ->
224225
rabbit_ct_broker_helpers:rpc_all(Config,
225226
application, set_env, [rabbitmq_management, restrictions, Restrictions]),
226227
rabbit_ct_helpers:testcase_started(Config, Testcase);
228+
init_per_testcase(queues_detailed_test, Config) ->
229+
IsEnabled = rabbit_ct_broker_helpers:is_feature_flag_enabled(
230+
Config, detailed_queues_endpoint),
231+
case IsEnabled of
232+
true -> Config;
233+
false -> {skip, "The detailed queues endpoint is not available."}
234+
end;
227235
init_per_testcase(Testcase, Config) ->
228236
rabbit_ct_broker_helpers:close_all_connections(Config, 0, <<"rabbit_mgmt_SUITE:init_per_testcase">>),
229237
rabbit_ct_helpers:testcase_started(Config, Testcase).
@@ -2238,8 +2246,18 @@ queue_pagination_test(Config) ->
22382246
#{name => <<"test2_reg">>, vhost => <<"/">>},
22392247
#{name => <<"reg_test3">>, vhost =><<"vh1">>}
22402248
], maps:get(items, FirstPage)),
2241-
2242-
2249+
%% The reduced API version just has the most useful fields.
2250+
%% garbage_collection is not one of them
2251+
IsEnabled = rabbit_ct_broker_helpers:is_feature_flag_enabled(
2252+
Config, detailed_queues_endpoint),
2253+
case IsEnabled of
2254+
true ->
2255+
[?assertNot(maps:is_key(garbage_collection, Item)) ||
2256+
Item <- maps:get(items, FirstPage)];
2257+
false ->
2258+
[?assert(maps:is_key(garbage_collection, Item)) ||
2259+
Item <- maps:get(items, FirstPage)]
2260+
end,
22432261
ReverseSortedByName = http_get(Config,
22442262
"/queues?page=2&page_size=2&sort=name&sort_reverse=true",
22452263
?OK),
@@ -2338,6 +2356,17 @@ queue_pagination_columns_test(Config) ->
23382356
vhost => <<"vh1">>}
23392357
], maps:get(items, ColumnsNameVhost)),
23402358

2359+
?awaitMatch(
2360+
true,
2361+
begin
2362+
ColumnsGarbageCollection = http_get(Config, "/queues?columns=name,garbage_collection&page=2&page_size=2", ?OK),
2363+
%% The reduced API version just has the most useful fields,
2364+
%% but we can still query any info item using `columns`
2365+
lists:all(fun(Item) ->
2366+
maps:is_key(garbage_collection, Item)
2367+
end,
2368+
maps:get(items, ColumnsGarbageCollection))
2369+
end, 30000),
23412370

23422371
http_delete(Config, "/queues/%2F/queue_a", {group, '2xx'}),
23432372
http_delete(Config, "/queues/vh1/queue_b", {group, '2xx'}),
@@ -2346,6 +2375,44 @@ queue_pagination_columns_test(Config) ->
23462375
http_delete(Config, "/vhosts/vh1", {group, '2xx'}),
23472376
passed.
23482377

2378+
queues_detailed_test(Config) ->
2379+
QArgs = #{},
2380+
http_put(Config, "/queues/%2F/queue_a", QArgs, {group, '2xx'}),
2381+
http_put(Config, "/queues/%2F/queue_c", QArgs, {group, '2xx'}),
2382+
2383+
?awaitMatch(
2384+
true,
2385+
begin
2386+
Detailed = http_get(Config, "/queues/detailed", ?OK),
2387+
lists:all(fun(Item) ->
2388+
maps:is_key(garbage_collection, Item)
2389+
end, Detailed)
2390+
end, 30000),
2391+
2392+
Detailed = http_get(Config, "/queues/detailed", ?OK),
2393+
?assertNot(lists:any(fun(Item) ->
2394+
maps:is_key(backing_queue_status, Item)
2395+
end, Detailed)),
2396+
%% It's null
2397+
?assert(lists:any(fun(Item) ->
2398+
maps:is_key(single_active_consumer_tag, Item)
2399+
end, Detailed)),
2400+
2401+
Reduced = http_get(Config, "/queues", ?OK),
2402+
?assertNot(lists:any(fun(Item) ->
2403+
maps:is_key(garbage_collection, Item)
2404+
end, Reduced)),
2405+
?assertNot(lists:any(fun(Item) ->
2406+
maps:is_key(backing_queue_status, Item)
2407+
end, Reduced)),
2408+
?assertNot(lists:any(fun(Item) ->
2409+
maps:is_key(single_active_consumer_tag, Item)
2410+
end, Reduced)),
2411+
2412+
http_delete(Config, "/queues/%2F/queue_a", {group, '2xx'}),
2413+
http_delete(Config, "/queues/%2F/queue_c", {group, '2xx'}),
2414+
passed.
2415+
23492416
queues_pagination_permissions_test(Config) ->
23502417
http_put(Config, "/users/non-admin", [{password, <<"non-admin">>},
23512418
{tags, <<"management">>}], {group, '2xx'}),

‎deps/rabbitmq_management_agent/include/rabbit_mgmt_metrics.hrl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
{exchange_stats_publish_in, set},
3636
{consumer_stats, set},
3737
{queue_stats, set},
38+
{queue_basic_stats, set},
3839
{queue_msg_stats, set},
3940
{vhost_msg_stats, set},
4041
{queue_process_stats, set},

‎deps/rabbitmq_management_agent/src/rabbit_mgmt_data.erl

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
-export([overview_data/4,
1919
consumer_data/2,
2020
all_list_queue_data/3,
21+
all_list_basic_queue_data/3,
2122
all_detail_queue_data/3,
2223
all_exchange_data/3,
2324
all_connection_data/3,
@@ -66,6 +67,12 @@ all_list_queue_data(_Pid, Ids, Ranges) ->
6667
maps:put(Id, Data, Acc)
6768
end, #{}, Ids).
6869

70+
all_list_basic_queue_data(_Pid, Ids, Ranges) ->
71+
lists:foldl(fun (Id, Acc) ->
72+
Data = list_basic_queue_data(Ranges, Id),
73+
maps:put(Id, Data, Acc)
74+
end, #{}, Ids).
75+
6976
all_detail_channel_data(_Pid, Ids, Ranges) ->
7077
lists:foldl(fun (Id, Acc) ->
7178
Data = detail_channel_data(Ranges, Id),
@@ -204,6 +211,11 @@ list_queue_data(Ranges, Id) ->
204211
queue_raw_deliver_stats_data(Ranges, Id) ++
205212
[{queue_stats, lookup_element(queue_stats, Id)}]).
206213

214+
list_basic_queue_data(Ranges, Id) ->
215+
maps:from_list(queue_raw_message_data(Ranges, Id) ++
216+
queue_raw_deliver_stats_data(Ranges, Id) ++
217+
[{queue_stats, lookup_element(queue_basic_stats, Id)}]).
218+
207219
detail_channel_data(Ranges, Id) ->
208220
maps:from_list(channel_raw_message_data(Ranges, Id) ++
209221
channel_raw_detail_stats_data(Ranges, Id) ++

‎deps/rabbitmq_management_agent/src/rabbit_mgmt_format.erl

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
-export([format_nulls/1, escape_html_tags/1]).
1919
-export([print/2, print/1]).
2020

21-
-export([format_queue_stats/1, format_channel_stats/1,
21+
-export([format_queue_stats/1, format_queue_basic_stats/1,
22+
format_channel_stats/1,
2223
format_consumer_arguments/1,
2324
format_connection_created/1,
2425
format_accept_content/1, format_args/1]).
@@ -80,6 +81,40 @@ format_queue_stats({disk_writes, _}) ->
8081
format_queue_stats(Stat) ->
8182
[Stat].
8283

84+
format_queue_basic_stats({_, ''}) ->
85+
[];
86+
format_queue_basic_stats({reductions, _}) ->
87+
[];
88+
format_queue_basic_stats({exclusive_consumer_pid, _}) ->
89+
[];
90+
format_queue_basic_stats({single_active_consumer_pid, _}) ->
91+
[];
92+
format_queue_basic_stats({slave_pids, Pids}) ->
93+
[{slave_nodes, [node(Pid) || Pid <- Pids]}];
94+
format_queue_basic_stats({leader, Leader}) ->
95+
[{node, Leader}];
96+
format_queue_basic_stats({effective_policy_definition, []}) ->
97+
[{effective_policy_definition, #{}}];
98+
format_queue_basic_stats({synchronised_slave_pids, Pids}) ->
99+
[{synchronised_slave_nodes, [node(Pid) || Pid <- Pids]}];
100+
format_queue_basic_stats({backing_queue_status, Value}) ->
101+
case proplists:get_value(version, Value, undefined) of
102+
undefined -> [];
103+
Version -> [{storage_version, Version}]
104+
end;
105+
format_queue_basic_stats({garbage_collection, _}) ->
106+
[];
107+
format_queue_basic_stats({idle_since, _Value}) ->
108+
[];
109+
format_queue_basic_stats({state, Value}) ->
110+
queue_state(Value);
111+
format_queue_basic_stats({disk_reads, _}) ->
112+
[];
113+
format_queue_basic_stats({disk_writes, _}) ->
114+
[];
115+
format_queue_basic_stats(Stat) ->
116+
[Stat].
117+
83118
format_channel_stats([{idle_since, Value} | Rest]) ->
84119
[{idle_since, now_to_str(Value)} | Rest];
85120
format_channel_stats(Stats) ->

0 commit comments

Comments
 (0)