Add rule for Pod (un)readiness and alarm to Slack

alertmanager/alertmanager.yaml

@@ -0,0 +1,33 @@
+global:
+  resolve_timeout: 5m
+  slack_api_url: ''
+route:
+  group_by: ['alertname', 'cluster', 'service', 'deployment', 'namespace']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 1h
+  receiver: default-receiver
+  routes:
+  - match:
+      alertname: DeadMansSwitch
+    receiver: 'null'
+receivers:
+- name: 'default-receiver'
+  slack_configs:
+  - channel: '#alerts'
+    title: >-
+        [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
+        {{ range .Alerts }}
+        {{ .Annotations.summary }}
+        {{ end }}
+    text: >-
+        {{ range .Alerts }}
+           *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
+          *Description:* {{ .Annotations.description }}
+          *Graph:* <{{ .GeneratorURL }}|:chart_with_upwards_trend:> *Runbook:* <{{ .Annotations.runbook }}|:spiral_note_pad:>
+          *Details:*
+          {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
+          {{ end }}
+        {{ end }}
+    send_resolved: true
+- name: 'null'

alertmanager/update.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+set -x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ -z "${NAMESPACE}" ]; then
+    NAMESPACE=monitoring
+fi
+
+kctl() {
+    kubectl --namespace "$NAMESPACE" "$@"
+}
+
+SECRET=alertmanager-main
+
+kctl create secret generic $SECRET --from-file $DIR/alertmanager.yaml --dry-run -o=yaml \
+  | kctl replace secret $SECRET -f -

custom-prometheus/config.sh

@@ -14,6 +14,4 @@ kctl() {
 SECRET=prometheus-custom
 
 kctl create secret generic $SECRET --from-file $DIR/config/prometheus.yaml --dry-run -o=yaml \
-  | kctl replace secret $SECRET -f - \
-  && kctl scale --replicas=0 statefulset prometheus-custom \
-  && kctl scale --replicas=1 statefulset prometheus-custom
+  | kctl replace secret $SECRET -f -

k8s-rules-apply.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+set -x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ -z "${NAMESPACE}" ]; then
+    NAMESPACE=monitoring
+fi
+
+kctl() {
+    kubectl --namespace "$NAMESPACE" "$@"
+}
+
+SECRET=prometheus-custom
+
+$DIR/k8s-rules-generate.sh | kctl replace -f -

generate-prometheus-rules.sh → k8s-rules-generate.sh

@@ -4,7 +4,7 @@ cd kube-prometheus
 . hack/scripts/generate-rules-configmap.sh
 cd ..
 
-for f in rules/*.rules.yaml
+for f in k8s-yolean-rules/*.y*ml
 do
   echo "  $(basename $f): |+"
   cat $f | sed "s/^/    /g"

k8s-yolean-rules/pod-unavailable.yml

@@ -0,0 +1,14 @@
+groups:
+- name: ./yolean.availability.rules
+  rules:
+  - alert: AvailabilityCheckFailed
+    expr: "(kube_deployment_status_replicas_available / kube_deployment_spec_replicas) * 100 < 100"
+    # NOTE This seems to be optional, thus missing should equal 0 ?
+    # for: 5s
+    # NOTE:
+    # Both labels and annotations support templates:
+    # https://prometheus.io/docs/alerting/rules/#templating
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ $value }}% of {{ $labels.deployment }}[{{ $labels.namespace }}] available"