Skip to content

Commit 041d436

Browse files
mkuratczykmergify[bot]
mkuratczyk
authored and
mergify[bot]
committed
Move memory breakdown metrics to new endpoint
Collecting them on a large system (tens of thousands of processes or more) can be time consuming as we iterate over all processes. By putting them on a separate endpoint, we make that opt-in (cherry picked from commit 618f695)
1 parent ad9e3f7 commit 041d436

File tree

4 files changed

+52
-35
lines changed

4 files changed

+52
-35
lines changed

‎deps/rabbitmq_prometheus/src/collectors/prometheus_rabbitmq_core_metrics_collector.erl

Lines changed: 34 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -86,35 +86,6 @@
8686
{2, ?MILLISECOND, erlang_uptime_seconds, gauge, "Node uptime", uptime}
8787
]},
8888

89-
{node_memory, [
90-
{2, undefined, memory_code_module_bytes, gauge, "Code module memory footprint", code},
91-
{2, undefined, memory_client_connection_reader_bytes, gauge, "Client connection reader processes footprint in bytes", connection_readers},
92-
{2, undefined, memory_client_connection_writer_bytes, gauge, "Client connection writer processes footprint in bytes", connection_writers},
93-
{2, undefined, memory_client_connection_channel_bytes, gauge, "Client connection channel processes footprint in bytes", connection_channels},
94-
{2, undefined, memory_client_connection_other_bytes, gauge, "Client connection other processes footprint in bytes", connection_other},
95-
{2, undefined, memory_classic_queue_erlang_process_bytes, gauge, "Classic queue processes footprint in bytes", queue_procs},
96-
{2, undefined, memory_quorum_queue_erlang_process_bytes, gauge, "Quorum queue processes footprint in bytes", quorum_queue_procs},
97-
{2, undefined, memory_quorum_queue_dlx_erlang_process_bytes, gauge, "Quorum queue DLX worker processes footprint in bytes", quorum_queue_dlx_procs},
98-
{2, undefined, memory_stream_erlang_process_bytes, gauge, "Stream processes footprint in bytes", stream_queue_procs},
99-
{2, undefined, memory_stream_replica_reader_erlang_process_bytes, gauge, "Stream replica reader processes footprint in bytes", stream_queue_replica_reader_procs},
100-
{2, undefined, memory_stream_coordinator_erlang_process_bytes, gauge, "Stream coordinator processes footprint in bytes", stream_queue_coordinator_procs},
101-
{2, undefined, memory_plugin_bytes, gauge, "Total plugin footprint in bytes", plugins},
102-
{2, undefined, memory_modern_metadata_store_bytes, gauge, "Modern metadata store footprint in bytes", metadata_store},
103-
{2, undefined, memory_other_erlang_process_bytes, gauge, "Other processes footprint in bytes", other_proc},
104-
{2, undefined, memory_metrics_bytes, gauge, "Metric table footprint in bytes", metrics},
105-
{2, undefined, memory_management_stats_db_bytes, gauge, "Management stats database footprint in bytes", mgmt_db},
106-
{2, undefined, memory_classic_metadata_store_bytes, gauge, "Classic metadata store footprint in bytes", mnesia},
107-
{2, undefined, memory_quorum_queue_ets_table_bytes, gauge, "Quorum queue ETS tables footprint in bytes", quorum_ets},
108-
{2, undefined, memory_modern_metadata_store_ets_table_bytes, gauge, "Modern metadata store ETS tables footprint in bytes", metadata_store_ets},
109-
{2, undefined, memory_other_ets_table_bytes, gauge, "Other ETS tables footprint in bytes", other_ets},
110-
{2, undefined, memory_binary_heap_bytes, gauge, "Binary heap size in bytes", binary},
111-
{2, undefined, memory_message_index_bytes, gauge, "Message index footprint in bytes", msg_index},
112-
{2, undefined, memory_atom_table_bytes, gauge, "Atom table size in bytes", atom},
113-
{2, undefined, memory_other_system_bytes, gauge, "Other runtime footprint in bytes", other_system},
114-
{2, undefined, memory_runtime_allocated_unused_bytes, gauge, "Runtime allocated but unused blocks size in bytes", allocated_unused},
115-
{2, undefined, memory_runtime_reserved_unallocated_bytes, gauge, "Runtime reserved but unallocated blocks size in bytes", reserved_unallocated}
116-
]},
117-
11889
{node_persister_metrics, [
11990
{2, undefined, io_read_ops_total, counter, "Total number of I/O read operations", io_read_count},
12091
{2, undefined, io_read_bytes_total, counter, "Total number of I/O bytes read", io_read_bytes},
@@ -277,6 +248,36 @@
277248
]}
278249
]).
279250

251+
-define(METRICS_MEMORY_BREAKDOWN, [
252+
{node_memory, [
253+
{2, undefined, memory_code_module_bytes, gauge, "Code module memory footprint", code},
254+
{2, undefined, memory_client_connection_reader_bytes, gauge, "Client connection reader processes footprint in bytes", connection_readers},
255+
{2, undefined, memory_client_connection_writer_bytes, gauge, "Client connection writer processes footprint in bytes", connection_writers},
256+
{2, undefined, memory_client_connection_channel_bytes, gauge, "Client connection channel processes footprint in bytes", connection_channels},
257+
{2, undefined, memory_client_connection_other_bytes, gauge, "Client connection other processes footprint in bytes", connection_other},
258+
{2, undefined, memory_classic_queue_erlang_process_bytes, gauge, "Classic queue processes footprint in bytes", queue_procs},
259+
{2, undefined, memory_quorum_queue_erlang_process_bytes, gauge, "Quorum queue processes footprint in bytes", quorum_queue_procs},
260+
{2, undefined, memory_quorum_queue_dlx_erlang_process_bytes, gauge, "Quorum queue DLX worker processes footprint in bytes", quorum_queue_dlx_procs},
261+
{2, undefined, memory_stream_erlang_process_bytes, gauge, "Stream processes footprint in bytes", stream_queue_procs},
262+
{2, undefined, memory_stream_replica_reader_erlang_process_bytes, gauge, "Stream replica reader processes footprint in bytes", stream_queue_replica_reader_procs},
263+
{2, undefined, memory_stream_coordinator_erlang_process_bytes, gauge, "Stream coordinator processes footprint in bytes", stream_queue_coordinator_procs},
264+
{2, undefined, memory_plugin_bytes, gauge, "Total plugin footprint in bytes", plugins},
265+
{2, undefined, memory_modern_metadata_store_bytes, gauge, "Modern metadata store footprint in bytes", metadata_store},
266+
{2, undefined, memory_other_erlang_process_bytes, gauge, "Other processes footprint in bytes", other_proc},
267+
{2, undefined, memory_metrics_bytes, gauge, "Metric table footprint in bytes", metrics},
268+
{2, undefined, memory_management_stats_db_bytes, gauge, "Management stats database footprint in bytes", mgmt_db},
269+
{2, undefined, memory_classic_metadata_store_bytes, gauge, "Classic metadata store footprint in bytes", mnesia},
270+
{2, undefined, memory_quorum_queue_ets_table_bytes, gauge, "Quorum queue ETS tables footprint in bytes", quorum_ets},
271+
{2, undefined, memory_modern_metadata_store_ets_table_bytes, gauge, "Modern metadata store ETS tables footprint in bytes", metadata_store_ets},
272+
{2, undefined, memory_other_ets_table_bytes, gauge, "Other ETS tables footprint in bytes", other_ets},
273+
{2, undefined, memory_binary_heap_bytes, gauge, "Binary heap size in bytes", binary},
274+
{2, undefined, memory_message_index_bytes, gauge, "Message index footprint in bytes", msg_index},
275+
{2, undefined, memory_atom_table_bytes, gauge, "Atom table size in bytes", atom},
276+
{2, undefined, memory_other_system_bytes, gauge, "Other runtime footprint in bytes", other_system},
277+
{2, undefined, memory_runtime_allocated_unused_bytes, gauge, "Runtime allocated but unused blocks size in bytes", allocated_unused},
278+
{2, undefined, memory_runtime_reserved_unallocated_bytes, gauge, "Runtime reserved but unallocated blocks size in bytes", reserved_unallocated}
279+
]}]).
280+
280281
-define(TOTALS, [
281282
%% ordering differs from metrics above, refer to list comprehension
282283
{connection_created, connections, gauge, "Connections currently open"},
@@ -305,6 +306,10 @@ collect_mf('per-object', Callback) ->
305306
totals(Callback),
306307
emit_identity_info(Callback),
307308
ok;
309+
collect_mf('memory-breakdown', Callback) ->
310+
collect(false, ?METRIC_NAME_PREFIX, false, ?METRICS_MEMORY_BREAKDOWN, Callback),
311+
emit_identity_info(Callback),
312+
ok;
308313
collect_mf(_Registry, Callback) ->
309314
PerObjectMetrics = application:get_env(rabbitmq_prometheus, return_per_object_metrics, false),
310315
collect(PerObjectMetrics, ?METRIC_NAME_PREFIX, false, ?METRICS_RAW, Callback),

‎deps/rabbitmq_prometheus/src/rabbit_prometheus_dispatcher.erl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ build_dispatcher() ->
3333
prometheus_registry:register_collectors('detailed', [
3434
prometheus_rabbitmq_core_metrics_collector
3535
]),
36+
prometheus_registry:register_collectors('memory-breakdown', [
37+
prometheus_rabbitmq_core_metrics_collector
38+
]),
3639
rabbit_prometheus_handler:setup(),
3740
cowboy_router:compile([{'_', dispatcher()}]).
3841

‎deps/rabbitmq_prometheus/src/rabbit_prometheus_handler.erl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ is_authorized(ReqData, Context) ->
4646
setup() ->
4747
setup_metrics(telemetry_registry()),
4848
setup_metrics('per-object'),
49+
setup_metrics('memory-breakdown'),
4950
setup_metrics('detailed').
5051

5152
setup_metrics(Registry) ->

‎deps/rabbitmq_prometheus/test/rabbit_prometheus_http_SUITE.erl

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ all() ->
2525
{group, commercial},
2626
{group, detailed_metrics},
2727
{group, special_chars},
28-
{group, authentication}
28+
{group, authentication},
29+
{group, memory_breakdown_endpoint_metrics}
2930
].
3031

3132
groups() ->
@@ -49,6 +50,9 @@ groups() ->
4950
endpoint_per_object_metrics,
5051
specific_erlang_metrics_present_test
5152
]},
53+
{memory_breakdown_endpoint_metrics, [], [
54+
memory_breakdown_metrics_test
55+
]},
5256
{commercial, [], [
5357
build_info_product_test
5458
]},
@@ -247,7 +251,9 @@ init_per_group(special_chars, Config0) ->
247251
init_per_group(authentication, Config) ->
248252
Config1 = rabbit_ct_helpers:merge_app_env(
249253
Config, {rabbitmq_prometheus, [{authentication, [{enabled, true}]}]}),
250-
init_per_group(authentication, Config1, []).
254+
init_per_group(authentication, Config1, []);
255+
init_per_group(memory_breakdown_endpoint_metrics, Config) ->
256+
init_per_group(memory_breakdown_endpoint_metrics, Config, []).
251257

252258

253259

@@ -387,10 +393,6 @@ aggregated_metrics_test(Config) ->
387393
?assertEqual(match, re:run(Body, "^rabbitmq_queue_consumers ", [{capture, none}, multiline])),
388394
?assertEqual(match, re:run(Body, "TYPE rabbitmq_auth_attempts_total", [{capture, none}, multiline])),
389395
?assertEqual(nomatch, re:run(Body, "TYPE rabbitmq_auth_attempts_detailed_total", [{capture, none}, multiline])),
390-
%% Memory breakdown
391-
?assertEqual(match, re:run(Body, "^rabbitmq_memory_quorum_queue_erlang_process_bytes ", [{capture, none}, multiline])),
392-
?assertEqual(match, re:run(Body, "^rabbitmq_memory_classic_queue_erlang_process_bytes ", [{capture, none}, multiline])),
393-
?assertEqual(match, re:run(Body, "^rabbitmq_memory_binary_heap_bytes ", [{capture, none}, multiline])),
394396
%% Check the first metric value in each ETS table that requires converting
395397
?assertEqual(match, re:run(Body, "^rabbitmq_erlang_uptime_seconds ", [{capture, none}, multiline])),
396398
?assertEqual(match, re:run(Body, "^rabbitmq_io_read_time_seconds_total ", [{capture, none}, multiline])),
@@ -437,6 +439,12 @@ per_object_metrics_test(Config, Path) ->
437439
%% Check the first TOTALS metric value
438440
?assertEqual(match, re:run(Body, "^rabbitmq_connections ", [{capture, none}, multiline])).
439441

442+
memory_breakdown_metrics_test(Config) ->
443+
{_Headers, Body} = http_get_with_pal(Config, "/metrics/memory-breakdown", [], 200),
444+
?assertEqual(match, re:run(Body, "^rabbitmq_memory_quorum_queue_erlang_process_bytes ", [{capture, none}, multiline])),
445+
?assertEqual(match, re:run(Body, "^rabbitmq_memory_classic_queue_erlang_process_bytes ", [{capture, none}, multiline])),
446+
?assertEqual(match, re:run(Body, "^rabbitmq_memory_binary_heap_bytes ", [{capture, none}, multiline])).
447+
440448
build_info_test(Config) ->
441449
{_Headers, Body} = http_get_with_pal(Config, [], 200),
442450
?assertEqual(match, re:run(Body, "^rabbitmq_build_info{", [{capture, none}, multiline])),

0 commit comments

Comments
 (0)