add basic auth with default user for alertmanager

Author: sjpb

ansible/roles/alertmanager/README.md

@@ -45,8 +45,10 @@ General variables:
 The following variables are equivalent to similarly-named arguments to the
 `alertmanager` binary. See `man alertmanager` for more info:
 
-- `alertmanager_config_file`: String, path alertmanager config file will be
-  written to. Parent directory will be created if necessary. 
+- `alertmanager_config_file`: String, path the main alertmanager config file
+  will be written to. Parent directory will be created if necessary. 
+- `alertmanager_web_config_file`: String, path alertmanager web config file
+  will be written to. Parent directory will be created if necessary. 
 - `alertmanager_storage_path`: String, base path for data storage.
 - `alertmanager_web_listen_addresses`: List of strings, defining addresses to listeen on.
 - `alertmanager_web_external_url`: String, the URL under which Alertmanager is
@@ -59,7 +61,7 @@ The following variables are equivalent to similarly-named arguments to the
   alertmanager commandline as `--{{ key }}={{ value }}`.
 - `alertmanager_default_receivers`:
 
-The following variables are templated into the [alertmanager configuration](https://prometheus.io/docs/alerting/latest/configuration/):
+The following variables are templated into the alertmanager [main configuration](https://prometheus.io/docs/alerting/latest/configuration/):
 - `alertmanager_config_template`: String, path to configuration template. The default
   is to template in `alertmanager_config_default` and `alertmanager_config_extra`.
 - `alertmanager_config_default`: Mapping with default configuration for the
@@ -85,3 +87,9 @@ The following variables are templated into the [alertmanager configuration](http
         - weekdays: ['monday:friday']
     ```
   Note that `route` and `receivers` keys should not be added here.
+
+The following variables are templated into the alertmanager [web configuration](https://prometheus.io/docs/alerting/latest/https/):
+- `alertmanager_web_config_default`: Mapping with default configuration for
+  `basic_auth_users` providing the default web user.
+- `alertmanager_alertmanager_web_config_extra`: Mapping with additional web
+  configuration. Keys in this become top-level keys in the web configuration.

ansible/roles/alertmanager/defaults/main.yml

@@ -8,19 +8,26 @@ alertmanager_enabled: true
 alertmanager_system_user: alertmanager
 alertmanager_system_group: "{{ alertmanager_system_user }}"
 alertmanager_config_file: /etc/alertmanager/alertmanager.yml
+alertmanager_web_config_file: /etc/alertmanager/alertmanager-web.yml
 alertmanager_storage_path: /var/lib/alertmanager
 
 alertmanager_port: '9093'
 alertmanager_web_listen_addresses:
   - ":{{ alertmanager_port }}"
-alertmanager_web_external_url: "http://{{ hostvars[groups['alertmanager'].0].ansible_host }}:{{ alertmanager_port}}/"
+alertmanager_web_external_url: '' # defined in environments/common/inventory/group_vars/all/alertmanager.yml for visibility
 
 alertmanager_data_retention: '120h'
 alertmanager_data_maintenance_interval: '15m'
 alertmanager_config_flags: {} # other command-line parameters as shown by `man alertmanager`
 alertmanager_config_template: alertmanager.yml.j2
+alertmanager_web_config_template: alertmanager-web.yml.j2
 
-# everything below here is interpolated into alertmanager_config_default:
+alertmanager_web_config_default:
+  basic_auth_users:
+    alertmanager: "{{ vault_alertmanager_admin_password | password_hash('bcrypt', '1234567890123456789012', ident='2b') }}"
+alertmanager_alertmanager_web_config_extra: {} # top-level only
+
+# Variables below are interpolated into alertmanager_config_default:
 
 # Uncomment below and add Slack bot app creds for Slack integration
 # alertmanager_slack_integration:

ansible/roles/alertmanager/tasks/configure.yml

@@ -7,6 +7,7 @@
     mode: u=rwX,go=rX
   loop:
     - "{{ alertmanager_config_file | dirname }}"
+    - "{{ alertmanager_web_config_file | dirname }}"
     - "{{ alertmanager_storage_path }}"
 
 - name: Create alertmanager service file with immutable options
@@ -19,7 +20,6 @@
   register: _alertmanager_service
   notify: Restart alertmanager
 
-
 - name: Template alertmanager config
   ansible.builtin.template:
     src: "{{ alertmanager_config_template }}"
@@ -29,6 +29,15 @@
     mode: u=rw,go=
   notify: Restart alertmanager
 
+- name: Template alertmanager web config
+  ansible.builtin.template:
+    src: "{{ alertmanager_web_config_template }}"
+    dest: "{{ alertmanager_web_config_file }}"
+    owner: "{{ alertmanager_system_user }}"
+    group: "{{ alertmanager_system_group }}"
+    mode: u=rw,go=
+  notify: Restart alertmanager
+
 - meta: flush_handlers
 
 - name: Ensure alertmanager service state

ansible/roles/alertmanager/templates/alertmanager.service.j2

@@ -24,6 +24,7 @@ ExecStart={{ alertmanager_binary_dir }}/alertmanager \
   --web.listen-address={{ address }} \
 {% endfor %}
   --web.external-url={{ alertmanager_web_external_url }} \
+  --web.config.file={{ alertmanager_web_config_file }} \
 {% for flag, flag_value in alertmanager_config_flags.items() %}
   --{{ flag }}={{ flag_value }} \
 {% endfor %}

ansible/roles/passwords/defaults/main.yml

@@ -11,6 +11,7 @@ slurm_appliance_secrets:
   vault_k3s_node_password: "{{ vault_k3s_node_password | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
   vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
   vault_demo_user_password: "{{ vault_demo_user_password | default(lookup('password', '/dev/null')) }}"
+  vault_alertmanager_admin_password: "{{ vault_alertmanager_admin_password | default(lookup('password', '/dev/null')) }}"
 
 secrets_openhpc_mungekey_default:
   content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}"

docs/alerting.md

@@ -9,30 +9,65 @@ describe the overall alerting process:
   sending out notifications via methods such as email, on-call notification
   systems, and chat platforms.
 
-By default, both a `prometheus` server and an `alertmanager` server are
-deployed on the control node for new environments:
+The general Prometheus configuration is described in
+[monitoring-and-logging.md](./monitoring-and-logging.md#defaults-3) - note that
+section specifies some role variables which commonly need modification.
+
+The alertmanager server is defined by the [ansible/roles/alertmanager](../ansible/roles/alertmanager/README.md),
+and all the configuration options and defaults are defined there. The defaults
+are fully functional, except that a [receiver](https://prometheus.io/docs/alerting/latest/configuration/#receiver)
+must be configured to generate notifications.
+
+## Enabling alertmanager
+
+1. Ensure both the `prometheus` and `alertmanager` servers are deployed on the
+control node  - for new environments the `cookiecutter` tool will have done
+this:
+
+    ```ini
+    # environments/site/groups:
+    [prometheus:children]
+    control
+
+    [alertmanager:children]
+    control
+    ```
+
+2. If the appliance was deployed before the alertmanager functionality was included,
+generate a password for the alertmanager UI user:
 
-```ini
-# environments/site/groups:
-[prometheus:children]
-control
+    ```shell
+    ansible-playbook ansible/adhoc/generate-passwords.yml
+    ```
 
-[alertmanager:children]
-control
+3. Configure a receiver to generate notifications from alerts. Currently a Slack
+integration is provided (see below) but alternative receivers could be defined
+via overriding role defaults.
+ 
+4. If desired, any other [role defaults](../ansible/roles/alertmanager/README.md)
+may be overriden in e.g. `environments/site/inventory/group_vars/all/alertmanager.yml`.
+
+5. Run the `monitoring.yml` playbook (if the cluster is already up) to configure
+both alertmanager and prometheus:
+
+    ```shell
+    ansible-playbook ansible/monitoring.yml
+    ```
+
+## Access
+
+There is a web interface provided by the alertmanager server. The default
+address can be seen using:
+
+```shell
+ansible localhost -m debug -a var=alertmanager_web_external_url
 ```
 
-The general Prometheus configuration is described in
-[monitoring-and-logging.md](./monitoring-and-logging.md#defaults-3) - note this
-section specifies some role variables which commonly need modification.
+The user is `alertmanager` and the autogenerated password can be seen using:
 
-The alertmanager server is defined by the [ansible/roles/alertmanager](../ansible/roles/alertmanager/README.md),
-and all the configuration options and defaults are defined there. By default
-it will be fully functional but:
-- `alertmanager_web_external_url` is likely to require modification.
-- A [receiver](https://prometheus.io/docs/alerting/latest/configuration/#receiver)
-  must be defined to actually provide notifications. Currently a Slack receiver
-  integration is provided (see below) but alternative receivers
-  could be defined using the provided role variables.
+```shell
+ansible localhost -m debug -a var=vault_alertmanager_admin_password
+```
 
 ## Slack receiver
 
@@ -72,10 +107,11 @@ of alerts via Slack.
 ## Alerting Rules
 
 These are part of [Prometheus configuration](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)
-which is defined appliance at
+which is defined for the appliance at
 [environments/common/inventory/group_vars/all/prometheus.yml](../environments/common/inventory/group_vars/all/prometheus.yml).
 
-Two `cloudalchemy.prometheus` role variables are relevant:
+Two [cloudalchemy.prometheus](https://github.com/cloudalchemy/ansible-prometheus)
+role variables are relevant:
 - `prometheus_alert_rules_files`: Paths to check for files providing rules.
   Note these are copied to Prometheus config directly, so jinja expressions for
   Prometheus do not need escaping.

environments/common/inventory/group_vars/all/alertmanager.yml

@@ -12,3 +12,5 @@ alertmanager_slack_receiver: # defined here as needs prometheus address
       text: "{{ '{{' }} .GroupLabels.alertname {{ '}}' }} : {{ '{{' }}  .CommonAnnotations.description {{ '}}' }}"
       title_link: "{{ prometheus_web_external_url }}/alerts?receiver=slack-receiver"
       send_resolved: true
+
+alertmanager_web_external_url: "http://{{ hostvars[groups['alertmanager'].0].ansible_host }}:{{ alertmanager_port}}/"