Skip to content

Commit 82e25af

Browse files
luosmkuratczyk
luos
and
mkuratczyk
authored
Grafana: make sure dashboards do not break when detailed metrics are used (#5945)
* Fix broken dashboards if detailed metrics are used If detailed metrics are pulled into the same prometheus, then we get an error in Grafana: execution: many-to-many matching not allowed: matching labels must be unique on one side This is because both endpoints provide `rabbit_identity_info` which is not unique to the endpoint. * add detailed metric scraper to prometheus config --------- Co-authored-by: Michal Kuratczyk <[email protected]>
1 parent 10fb936 commit 82e25af

File tree

8 files changed

+165
-147
lines changed

8 files changed

+165
-147
lines changed

‎deps/rabbitmq_prometheus/docker/grafana/dashboards/Erlang-Distribution.json

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@
134134
"pluginVersion": "8.3.4",
135135
"targets": [
136136
{
137-
"expr": "count(erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) OR vector(0)",
137+
"expr": "count(erlang_vm_dist_node_state * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) OR vector(0)",
138138
"format": "time_series",
139139
"interval": "",
140140
"intervalFactor": 1,
@@ -214,7 +214,7 @@
214214
"pluginVersion": "8.3.4",
215215
"targets": [
216216
{
217-
"expr": "count(erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} == 3) OR vector(0)",
217+
"expr": "count(erlang_vm_dist_node_state * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} == 3) OR vector(0)",
218218
"format": "time_series",
219219
"interval": "",
220220
"intervalFactor": 1,
@@ -294,7 +294,7 @@
294294
"pluginVersion": "8.3.4",
295295
"targets": [
296296
{
297-
"expr": "count(erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} == 1) OR vector(0)",
297+
"expr": "count(erlang_vm_dist_node_state * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} == 1) OR vector(0)",
298298
"format": "time_series",
299299
"interval": "",
300300
"intervalFactor": 1,
@@ -374,7 +374,7 @@
374374
"pluginVersion": "8.3.4",
375375
"targets": [
376376
{
377-
"expr": "count(erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} == 2) OR vector(0)",
377+
"expr": "count(erlang_vm_dist_node_state * on(instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} == 2) OR vector(0)",
378378
"format": "time_series",
379379
"interval": "",
380380
"intervalFactor": 1,
@@ -466,7 +466,7 @@
466466
"displayAliasType": "Warning / Critical",
467467
"displayType": "Regular",
468468
"displayValueWithAlias": "Never",
469-
"expr": "erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
469+
"expr": "erlang_vm_dist_node_state * on(instance, job) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
470470
"format": "time_series",
471471
"intervalFactor": 1,
472472
"legendFormat": " {{rabbitmq_node}} -> {{peer}}",
@@ -738,7 +738,7 @@
738738
"pluginVersion": "8.3.4",
739739
"targets": [
740740
{
741-
"expr": "erlang_vm_dist_node_queue_size_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
741+
"expr": "erlang_vm_dist_node_queue_size_bytes * on(instance, job) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
742742
"format": "time_series",
743743
"interval": "",
744744
"intervalFactor": 1,
@@ -1002,7 +1002,7 @@
10021002
"pluginVersion": "8.3.4",
10031003
"targets": [
10041004
{
1005-
"expr": "rate(erlang_vm_dist_send_bytes[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
1005+
"expr": "rate(erlang_vm_dist_send_bytes[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
10061006
"format": "time_series",
10071007
"intervalFactor": 1,
10081008
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -1248,7 +1248,7 @@
12481248
"pluginVersion": "8.3.4",
12491249
"targets": [
12501250
{
1251-
"expr": "rate(erlang_vm_dist_recv_bytes[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
1251+
"expr": "rate(erlang_vm_dist_recv_bytes[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
12521252
"format": "time_series",
12531253
"intervalFactor": 1,
12541254
"legendFormat": "{{rabbitmq_node}} <- {{peer}}",
@@ -1494,7 +1494,7 @@
14941494
"pluginVersion": "8.3.4",
14951495
"targets": [
14961496
{
1497-
"expr": "rate(erlang_vm_dist_send_cnt[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
1497+
"expr": "rate(erlang_vm_dist_send_cnt[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
14981498
"format": "time_series",
14991499
"intervalFactor": 1,
15001500
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -1740,7 +1740,7 @@
17401740
"pluginVersion": "8.3.4",
17411741
"targets": [
17421742
{
1743-
"expr": "rate(erlang_vm_dist_recv_cnt[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
1743+
"expr": "rate(erlang_vm_dist_recv_cnt[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
17441744
"format": "time_series",
17451745
"intervalFactor": 1,
17461746
"legendFormat": "{{rabbitmq_node}} <- {{peer}}",
@@ -1986,7 +1986,7 @@
19861986
"pluginVersion": "8.3.4",
19871987
"targets": [
19881988
{
1989-
"expr": "(rate(erlang_vm_dist_send_bytes[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) / \n(rate(erlang_vm_dist_send_cnt[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})",
1989+
"expr": "(rate(erlang_vm_dist_send_bytes[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) / \n(rate(erlang_vm_dist_send_cnt[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})",
19901990
"format": "time_series",
19911991
"intervalFactor": 1,
19921992
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -2232,7 +2232,7 @@
22322232
"pluginVersion": "8.3.4",
22332233
"targets": [
22342234
{
2235-
"expr": "(rate(erlang_vm_dist_recv_bytes[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) / \n(rate(erlang_vm_dist_recv_cnt[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})",
2235+
"expr": "(rate(erlang_vm_dist_recv_bytes[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) / \n(rate(erlang_vm_dist_recv_cnt[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})",
22362236
"format": "time_series",
22372237
"intervalFactor": 1,
22382238
"legendFormat": "{{rabbitmq_node}} <- {{peer}}",
@@ -2495,7 +2495,7 @@
24952495
"pluginVersion": "8.3.4",
24962496
"targets": [
24972497
{
2498-
"expr": "erlang_vm_dist_port_memory_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
2498+
"expr": "erlang_vm_dist_port_memory_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
24992499
"format": "time_series",
25002500
"intervalFactor": 1,
25012501
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -2606,7 +2606,7 @@
26062606
"pluginVersion": "8.3.4",
26072607
"targets": [
26082608
{
2609-
"expr": "erlang_vm_dist_port_queue_size_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
2609+
"expr": "erlang_vm_dist_port_queue_size_bytes * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
26102610
"format": "time_series",
26112611
"intervalFactor": 1,
26122612
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -2711,7 +2711,7 @@
27112711
"displayAliasType": "Warning / Critical",
27122712
"displayType": "Regular",
27132713
"displayValueWithAlias": "Never",
2714-
"expr": "erlang_vm_dist_proc_status{type=\"$erlang_vm_dist_proc_type\"} * on(instance) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} ",
2714+
"expr": "erlang_vm_dist_proc_status{type=\"$erlang_vm_dist_proc_type\"} * on(instance, job) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"} ",
27152715
"format": "time_series",
27162716
"intervalFactor": 1,
27172717
"legendFormat": " {{rabbitmq_node}} -> {{peer}}",
@@ -2848,7 +2848,7 @@
28482848
"pluginVersion": "8.3.4",
28492849
"targets": [
28502850
{
2851-
"expr": "erlang_vm_dist_proc_message_queue_len{type=\"$erlang_vm_dist_proc_type\"} * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
2851+
"expr": "erlang_vm_dist_proc_message_queue_len{type=\"$erlang_vm_dist_proc_type\"} * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
28522852
"format": "time_series",
28532853
"intervalFactor": 1,
28542854
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -3094,7 +3094,7 @@
30943094
"pluginVersion": "8.3.4",
30953095
"targets": [
30963096
{
3097-
"expr": "erlang_vm_dist_proc_memory_bytes{type=\"$erlang_vm_dist_proc_type\"} * on(instance) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
3097+
"expr": "erlang_vm_dist_proc_memory_bytes{type=\"$erlang_vm_dist_proc_type\"} * on(instance, job) group_left(rabbitmq_node, rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
30983098
"format": "time_series",
30993099
"intervalFactor": 1,
31003100
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -3340,7 +3340,7 @@
33403340
"pluginVersion": "8.3.4",
33413341
"targets": [
33423342
{
3343-
"expr": "rate(erlang_vm_dist_proc_reductions{type=\"$erlang_vm_dist_proc_type\"}[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
3343+
"expr": "rate(erlang_vm_dist_proc_reductions{type=\"$erlang_vm_dist_proc_type\"}[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
33443344
"format": "time_series",
33453345
"intervalFactor": 1,
33463346
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -3487,4 +3487,4 @@
34873487
"uid": "d-SFCCmZz",
34883488
"version": 20220805,
34893489
"weekStart": ""
3490-
}
3490+
}

‎deps/rabbitmq_prometheus/docker/grafana/dashboards/Erlang-Distributions-Compare.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@
160160
"pluginVersion": "8.3.4",
161161
"targets": [
162162
{
163-
"expr": "rate(erlang_vm_dist_send_bytes[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=~\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
163+
"expr": "rate(erlang_vm_dist_send_bytes[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=~\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
164164
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
165165
"refId": "A"
166166
}
@@ -415,7 +415,7 @@
415415
"pluginVersion": "8.3.4",
416416
"targets": [
417417
{
418-
"expr": "rate(erlang_vm_dist_send_bytes[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=~\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
418+
"expr": "rate(erlang_vm_dist_send_bytes[60s]) * on(instance, job) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=~\"$rabbitmq_cluster\", namespace=\"$namespace\"}",
419419
"format": "time_series",
420420
"intervalFactor": 1,
421421
"legendFormat": "{{rabbitmq_node}} -> {{peer}}",
@@ -1955,4 +1955,4 @@
19551955
"uid": "C0jeDstZk",
19561956
"version": 20220805,
19571957
"weekStart": ""
1958-
}
1958+
}

0 commit comments

Comments
 (0)