update alertmanager comments

Author: sjpb

ansible/roles/alertmanager/README.md

@@ -21,7 +21,8 @@ Alertmanager is enabled by default on the `control` node in the
 In general usage may only require:
 - Adding the `control` node into the `alertmanager` group in `environments/site/groups`
   if upgrading an existing environment.
-- Enabling the Slack integration (see below).
+- Enabling the Slack integration (see section below).
+- Possibly setting `alertmanager_web_external_url`.
 
 ## Role variables
 

ansible/roles/alertmanager/defaults/main.yml

@@ -15,13 +15,11 @@ alertmanager_web_listen_addresses:
   - ":{{ alertmanager_port }}"
 alertmanager_web_external_url: "http://localhost:{{ alertmanager_port}}/" # TODO: is this right??
 
-alertmanager_data_retention: '120h' # --data.retention # How long to keep data for
-alertmanager_data_maintenance_interval: '15m' # --data.maintenance-interval: Interval between garbage collection and snapshotting to disk of the silences and the notification logs
+alertmanager_data_retention: '120h'
+alertmanager_data_maintenance_interval: '15m'
 alertmanager_config_flags: {} # other command-line parameters as shown by `man alertmanager`
-# TODO: data retention?
 alertmanager_config_template: alertmanager.yml.j2
 
-
 # everything below here is interpolated into alertmanager_config_default:
 
 # Uncomment below and add Slack bot app creds for Slack integration
@@ -43,7 +41,3 @@ alertmanager_config_default:
   receivers: "{{ alertmanager_receivers }}"
 
 alertmanager_config_extra: {} # top-level only
-
-
-# TODO: routes??
-# TODO: see PR with additional alerts

ansible/roles/alertmanager/tasks/configure.yml

@@ -9,8 +9,6 @@
     - "{{ alertmanager_config_file | dirname }}"
     - "{{ alertmanager_storage_path }}"
 
-# TODO: selinux?
-
 - name: Create alertmanager service file with immutable options
   template:
     src: alertmanager.service.j2
@@ -20,7 +18,7 @@
     mode: u=rw,go=r
   register: _alertmanager_service
   notify: Restart alertmanager
-  # TODO: how do we cope with the binary changing?
+
 
 - name: Template alertmanager config
   ansible.builtin.template:

environments/common/files/prometheus/rules/slurm.rules

@@ -14,3 +14,6 @@ groups:
       description: '{{ $value }} Slurm nodes are in fail status'
       summary: 'At least one Slurm node is failed.'
     expr: "slurm_nodes_fail > 0\n"
+
+# TODO: alert on slurm_scheduler_dbd_queue_size - see vpenso exporter, man sdiag, and MaxDBDMsgs
+# but node its dynamic