Add Blackbox monitoring for backend endpoints

etc/kayobe/kolla/inventory/group_vars/prometheus-blackbox-exporter

@@ -132,6 +132,45 @@ prometheus_blackbox_exporter_endpoints_default:
     enabled: "{{ enable_rabbitmq | bool }}"
   - endpoints: "{% set redis_endpoints = [] %}{% for host in groups.get('redis', []) %}{{ redis_endpoints.append('redis_' + host.replace('-', '') + ':tcp_connect:' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['redis_port']) }}{% endfor %}{{ redis_endpoints }}"
     enabled: "{{ enable_redis | bool }}"
+  # Backend endpoints
+  - endpoints: "{% set barbican_endpoints = [] %}{% for host in groups.get('barbican-api', []) %}{{ barbican_endpoints.append('barbican_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['barbican_api_port']) }}{% endfor %}{{ barbican_endpoints }}"
+    enabled: "{{ enable_barbican | bool }}"
+  - endpoints: "{% set blazar_endpoints = [] %}{% for host in groups.get('blazar-api', []) %}{{ blazar_endpoints.append('blazar_backend_' + host.replace('-', '') + ':os_endpoint:' +  'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['blazar_api_port']) }}{% endfor %}{{ blazar_endpoints }}"
+    enabled: "{{ enable_blazar | bool }}"
+  - endpoints: "{% set cinder_endpoints = [] %}{% for host in groups.get('cinder-api', []) %}{{ cinder_endpoints.append('cinder_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['cinder_api_port']) }}{% endfor %}{{ cinder_endpoints }}"
+    enabled: "{{ enable_cinder | bool }}"
+  - endpoints: "{% set designate_endpoints = [] %}{% for host in groups.get('designate-api', []) %}{{ designate_endpoints.append('designate_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['designate_api_port']) }}{% endfor %}{{ designate_endpoints }}"
+    enabled: "{{ enable_designate | bool }}"
+  - endpoints: "{% set glance_endpoints = [] %}{% for host in groups.get('glance-api', []) %}{{ glance_endpoints.append('glance_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if glance_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['glance_api_port']) }}{% endfor %}{{ glance_endpoints }}"
+    enabled: "{{ enable_glance | bool }}"
+  - endpoints: "{% set gnocchi_endpoints = [] %}{% for host in groups.get('gnocchi-api', []) %}{{ gnocchi_endpoints.append('gnocchi_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['gnocchi_api_port']) }}{% endfor %}{{ gnocchi_endpoints }}"
+    enabled: "{{ enable_gnocchi | bool }}"
+  - endpoints: "{% set heat_endpoints = [] %}{% for host in groups.get('heat-api', []) %}{{ heat_endpoints.append('heat_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['heat_api_port']) }}{% endfor %}{{ heat_endpoints }}"
+    enabled: "{{ enable_heat | bool }}"
+  - endpoints: "{% set heat_cfn_endpoints = [] %}{% for host in groups.get('heat-api-cfn', []) %}{{ heat_cfn_endpoints.append('heat_cfn_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['heat_api_cfn_port']) }}{% endfor %}{{ heat_cfn_endpoints }}"
+    enabled: "{{ enable_heat | bool }}"
+  - endpoints: "{% set horizon_endpoints = [] %}{% for host in groups.get('horizon', []) %}{{ horizon_endpoints.append('horizon_backend_' + host.replace('-', '') + ':http_2xx:' + ('https' if horizon_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['horizon_listen_port']) }}{% endfor %}{{ horizon_endpoints }}"
+    enabled: "{{ enable_horizon | bool }}"
+  - endpoints: "{% set keystone_endpoints = [] %}{% for host in groups.get('keystone-api', []) %}{{ keystone_endpoints.append('keystone_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['keystone_internal_port']) }}{% endfor %}{{ keystone_endpoints }}"
+    enabled: "{{ enable_keystone | bool }}"
+  - endpoints: "{% set magnum_endpoints = [] %}{% for host in groups.get('magnum-api', []) %}{{ magnum_endpoints.append('magnum_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['magnum_api_port']) }}{% endfor %}{{ magnum_endpoints }}"
+    enabled: "{{ enable_magnum | bool }}"
+  - endpoints: "{% set manila_endpoints = [] %}{% for host in groups.get('manila-api', []) %}{{ manila_endpoints.append('manila_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['manila_api_port']) }}{% endfor %}{{ manila_endpoints }}"
+    enabled: "{{ enable_manila | bool }}"
+  - endpoints: "{% set neutron_endpoints = [] %}{% for host in groups.get('neutron-server', []) %}{{ neutron_endpoints.append('neutron_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if neutron_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['neutron_server_port']) }}{% endfor %}{{ neutron_endpoints }}"
+    enabled: "{{ enable_neutron | bool }}"
+  - endpoints: "{% set nova_endpoints = [] %}{% for host in groups.get('nova-api', []) %}{{ nova_endpoints.append('nova_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['nova_api_port']) }}{% endfor %}{{ nova_endpoints }}"
+    enabled: "{{ enable_nova | bool }}"
+  - endpoints: "{% set octavia_endpoints = [] %}{% for host in groups.get('octavia-api', []) %}{{ octavia_endpoints.append('octavia_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['octavia_api_port']) }}{% endfor %}{{ octavia_endpoints }}"
+    enabled: "{{ enable_octavia | bool }}"
+  - endpoints: "{% set placement_endpoints = [] %}{% for host in groups.get('placement-api', []) %}{{ placement_endpoints.append('placement_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['placement_api_port']) }}{% endfor %}{{ placement_endpoints }}"
+    enabled: "{{ enable_placement | bool }}"
+  - endpoints: "{% set ironic_endpoints = [] %}{% for host in groups.get('ironic-api', []) %}{{ ironic_endpoints.append('ironic_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['ironic_api_port']) }}{% endfor %}{{ ironic_endpoints }}"
+    enabled: "{{ enable_ironic | bool }}"
+  - endpoints: "{% set ironic_inspector_endpoints = [] %}{% for host in groups.get('ironic-inspector', []) %}{{ ironic_inspector_endpoints.append('ironic_inspector_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['ironic_inspector_port']) }}{% endfor %}{{ ironic_inspector_endpoints }}"
+    enabled: "{{ enable_ironic | bool }}"
+  - endpoints: "{% set swift_endpoints = [] %}{% for host in groups.get('swift-api', []) %}{{ swift_endpoints.append('swift_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['swift_api_port']) }}{% endfor %}{{ swift_endpoints }}"
+    enabled: "{{ enable_swift | bool }}"
 
 # Ensure service endpoints are defined
 blazar_internal_base_endpoint: "{{ blazar_internal_fqdn | kolla_url(internal_protocol, blazar_api_port) }}"

releasenotes/notes/blackbox-backend-4415919e10a1aa4e.yaml

@@ -0,0 +1,14 @@
+---
+features:
+  - |
+    Added Blackbox monitoring for backend endpoints by default. Note that this
+    configuration will only work if the Blackbox exporters have access to the
+    backend endpoints.
+issues:
+  - |
+    Backend Blackbox monitoring will not work if the exporter does not have
+    access to the backend OpenStack endpoints. This usually happens when
+    separate monitoring nodes are deployed. In this case, move the Blackbox
+    exporter to the Haproxy group, remove the endpoints from
+    `etc/kayobe/kolla/inventory/group_vars/prometheus-blackbox-exporter`, or
+    silence the alerts permanently.