Surface API server warnings when applying plan steps.

Responses from the API server may include RFC2616 14.46 warnings (for
example, due to deprecation). Since the catalog-operator creates and
updates cluster resources on behalf of users, these warnings should be
surfaced to users.

When at least one warning is returned during the creation or update of
a step resource, catalog-operator now creates an Event with reason
"AppliedWithWarnings", involvedObject and fieldPath set to the plan
and step that generated the warning, respectively, and its message
containing the warning text.

In addition, a new metric, "installplan_warnings_total", counts the
number of steps that have been applied with warnings.

Signed-off-by: Ben Luddy <[email protected]>

Author: exdx

deploy/chart/templates/0000_90_olm_01-prometheus-rule.yaml

@@ -9,23 +9,30 @@ metadata:
     role: alert-rules
 spec:
   groups:
-    - name: olm.csv_abnormal.rules
-      rules:
-        - alert: CsvAbnormalFailedOver2Min
-          expr: csv_abnormal{phase=~"^Failed$"}
-          for: 2m
-          labels:
-            severity: warning
-            namespace: "{{ "{{ $labels.namespace }}" }}"
-          annotations:
-            message: Failed to install Operator {{ printf "{{ $labels.name }}"  }} version {{ printf "{{ $labels.version }}"  }}. Reason-{{ printf "{{ $labels.reason }}" }}
-
-        - alert: CsvAbnormalOver30Min
-          expr: csv_abnormal{phase=~"(^Replacing$|^Pending$|^Deleting$|^Unknown$)"}
-          for: 30m
-          labels:
-            severity: warning
-            namespace: "{{ "{{ $labels.namespace }}" }}"
-          annotations:
-            message: Failed to install Operator {{ printf "{{ $labels.name }}"  }} version {{ printf "{{ $labels.version }}"  }}. Phase-{{ printf "{{ $labels.phase }}" }} Reason-{{ printf "{{ $labels.reason }}" }}
+  - name: olm.csv_abnormal.rules
+    rules:
+      - alert: CsvAbnormalFailedOver2Min
+        expr: csv_abnormal{phase=~"^Failed$"}
+        for: 2m
+        labels:
+          severity: warning
+          namespace: "{{ "{{ $labels.namespace }}" }}"
+        annotations:
+          message: Failed to install Operator {{ printf "{{ $labels.name }}"  }} version {{ printf "{{ $labels.version }}"  }}. Reason-{{ printf "{{ $labels.reason }}" }}
+      - alert: CsvAbnormalOver30Min
+        expr: csv_abnormal{phase=~"(^Replacing$|^Pending$|^Deleting$|^Unknown$)"}
+        for: 30m
+        labels:
+          severity: warning
+          namespace: "{{ "{{ $labels.namespace }}" }}"
+        annotations:
+          message: Failed to install Operator {{ printf "{{ $labels.name }}"  }} version {{ printf "{{ $labels.version }}"  }}. Phase-{{ printf "{{ $labels.phase }}" }} Reason-{{ printf "{{ $labels.reason }}" }}
+  - name: olm.installplan.rules
+    rules:
+    - alert: InstallPlanStepAppliedWithWarnings
+      expr: sum(sum_over_time(installplan_warnings_total[5m])) > 0
+      labels:
+        severity: warning
+      annotations:
+        message: The API server returned a warning during installation or upgrade of an operator. An Event with reason "AppliedWithWarnings" has been created with complete details, including a reference to the InstallPlan step that generated the warning.
 {{ end }}

pkg/controller/operators/catalog/client_factory_stub_test.go

@@ -0,0 +1,33 @@
+package catalog
+
+import (
+	"k8s.io/client-go/dynamic"
+
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/clients"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
+)
+
+type stubClientFactory struct {
+	operatorClient   operatorclient.ClientInterface
+	kubernetesClient versioned.Interface
+	dynamicClient    dynamic.Interface
+}
+
+var _ clients.Factory = &stubClientFactory{}
+
+func (f *stubClientFactory) WithConfigTransformer(clients.ConfigTransformer) clients.Factory {
+	return f
+}
+
+func (f *stubClientFactory) NewOperatorClient() (operatorclient.ClientInterface, error) {
+	return f.operatorClient, nil
+}
+
+func (f *stubClientFactory) NewKubernetesClient() (versioned.Interface, error) {
+	return f.kubernetesClient, nil
+}
+
+func (f *stubClientFactory) NewDynamicClient() (dynamic.Interface, error) {
+	return f.dynamicClient, nil
+}