[obs] re-enable regular not active alerts (#18341)

* [obs] Add back critical regular not active alerts

Related to ENG-15

Now that we have related data, we should resume triggering alerts if the data condition occurs.

* [obs] Fix runbook_url for GitpodImageBuildDurationAnomaly

Was getting 404

* [obs] Fix GitpodWorkspaceTooManyRegularNotActiveMk2 given https://www.gitpodstatus.com/incidents/bsrqgmsxw1gr

* [obs] share why regular not active is excluded from Dedicated

* [obs] consolidate runbook for regular not active alerts

Author: kylos101

operations/observability/mixins/workspace/rules/central/image-builder.yaml

@@ -18,7 +18,7 @@ spec:
           severity: critical
           dedicated: included
         annotations:
-          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodImageBuildDurationAnomaly.md
+          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodImagebuildDurationAnomaly.md
           summary: image-builder duration is unusually high in cluster {{ $labels.cluster }}
           description: Users are waiting too long for image builds
         expr: |

operations/observability/mixins/workspace/rules/satellite/workspaces.yaml

@@ -21,26 +21,30 @@ spec:
     rules:
     - alert: GitpodWorkspaceTooManyRegularNotActiveMk2
       labels:
-        severity: warning
+        severity: critical
+        # TODO: uncomment after recording rule import is working in Grafana Cloud
+        # dedicated: included
       for: 10m
       annotations:
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceTooManyRegularNotActive.md
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceRegularNotActive.md
         summary: too many running but inactive workspaces
         description: too many running but inactive workspaces.
       expr: |
-        gitpod_workspace_regular_not_active_percentage_mk2 > 0.08
+        sum(gitpod_workspace_regular_not_active_percentage_mk2) by(cluster) > 0.08
         AND
         sum(gitpod_ws_manager_mk2_workspace_activity_total) by(cluster) > 25
 
     - alert: GitpodWorkspacesNotStartingMk2
       labels:
-        severity: warning
+        severity: critical
+        # TODO: uncomment after recording rule import is working in Grafana Cloud
+        # dedicated: included
       for: 10m
       annotations:
-        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceNotStarting.md
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceRegularNotActive.md
         summary: workspaces are not starting.
         description: inactive regular workspaces exists but workspaces are not being started.
       expr: |
-        avg_over_time(gitpod_workspace_regular_not_active_percentage_mk2[1m]) > 0
+        sum by(cluster) (avg_over_time(gitpod_workspace_regular_not_active_percentage_mk2[1m]) > 0)
         AND
-        rate(gitpod_ws_manager_mk2_workspace_startup_seconds_sum{type="Regular"}[1m]) == 0
+        sum by(cluster) (rate(gitpod_ws_manager_mk2_workspace_startup_seconds_sum{type="Regular"}[1m])) == 0