Add alert when fluentd queue buffers grow too large

Author: priteau

etc/kayobe/kolla/config/prometheus/fluentd.rules

@@ -0,0 +1,13 @@
+{% raw %}
+groups:
+- name: Fluentd
+  rules:
+  - alert: FluentdBufferTooLarge
+    expr: (fluentd_output_status_buffer_total_bytes / 1024^2) > 128
+    for: 15m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Fluentd at {{ $labels.instance }} reports large queue buffers"
+      description: "Fluentd queue buffers on {{ $labels.instance }} are using {{ $value }} MiB."
+{% endraw %}

releasenotes/notes/fluentd-buffer-too-large-alert-682893410855387f.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds a new Prometheus alert ``FluentdBufferTooLarge`` which is raised when
+    the total size of queue buffers grows above 128 MiB.