unit test for "OLM pod restarts"

Signed-off-by: Josef Karasek <[email protected]>

Author: Josef Karasek

cmd/olm/main.go

@@ -272,7 +272,7 @@ func main() {
 }
 
 func ensureCSVMetric(logger *logrus.Logger, c *versioned.Clientset) error {
-	logger.Debug("Emitting CSV metric")
+	logger.Debug("emitting CSV metric")
 	listOpts := metav1.ListOptions{}
 	csvs, err := c.OperatorsV1alpha1().ClusterServiceVersions(metav1.NamespaceAll).List(context.TODO(), listOpts)
 	if err != nil {

test/e2e/metrics_e2e_test.go

@@ -15,6 +15,7 @@ import (
 	. "github.com/onsi/gomega"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -112,6 +113,41 @@ var _ = Describe("Metrics are generated for OLM managed resources", func() {
 				})
 			})
 		})
+
+		When("the OLM pod restars", func() {
+			var (
+				cleanupCSV cleanupFunc
+				csv        v1alpha1.ClusterServiceVersion
+			)
+			BeforeEach(func() {
+				packageName := genName("csv-test-")
+				packageStable := fmt.Sprintf("%s-stable", packageName)
+				csv = newCSV(packageStable, testNamespace, "", semver.MustParse("0.1.0"), nil, nil, nil)
+
+				var err error
+				_, err = createCSV(c, crc, csv, testNamespace, false, false)
+				Expect(err).ToNot(HaveOccurred())
+
+				_, err = fetchCSV(crc, csv.Name, testNamespace, csvSucceededChecker)
+				Expect(err).ToNot(HaveOccurred())
+			})
+			AfterEach(func() {
+				if cleanupCSV != nil {
+					cleanupCSV()
+				}
+			})
+			It("csv metric is preserved", func() {
+				Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"), "8081")).To(
+					ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))),
+				)
+
+				restartDeploymentWithLabel(c, "app=olm-operator")
+
+				Expect(getMetricsFromPod(c, getPodWithLabel(c, "app=olm-operator"), "8081")).To(
+					ContainElement(LikeMetric(WithFamily("csv_succeeded"), WithName(csv.Name), WithValue(1))),
+				)
+			})
+		})
 	})
 
 	Context("Metrics emitted by objects during operator installation", func() {
@@ -392,6 +428,51 @@ func getPodWithLabel(client operatorclient.ClientInterface, label string) *corev
 	return &podList.Items[0]
 }
 
+func getDeploymentWithLabel(client operatorclient.ClientInterface, label string) *appsv1.Deployment {
+	listOptions := metav1.ListOptions{LabelSelector: label}
+	var deploymentList *appsv1.DeploymentList
+	EventuallyWithOffset(1, func() (numDeps int, err error) {
+		deploymentList, err = client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).List(context.TODO(), listOptions)
+		if deploymentList != nil {
+			numDeps = len(deploymentList.Items)
+		}
+
+		return
+	}).Should(Equal(1), "expected exactly one Deployment")
+
+	return &deploymentList.Items[0]
+}
+
+func restartDeploymentWithLabel(client operatorclient.ClientInterface, l string) {
+	d := getDeploymentWithLabel(client, l)
+	z := int32(0)
+	oldZ := *d.Spec.Replicas
+	d.Spec.Replicas = &z
+	_, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Update(context.TODO(), d, metav1.UpdateOptions{})
+	Expect(err).ToNot(HaveOccurred())
+
+	EventuallyWithOffset(1, func() (replicas int32, err error) {
+		deployment, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Get(context.TODO(), d.Name, metav1.GetOptions{})
+		if deployment != nil {
+			replicas = deployment.Status.Replicas
+		}
+		return
+	}).Should(Equal(int32(0)), "expected exactly 0 Deployments")
+
+	updated := getDeploymentWithLabel(client, l)
+	updated.Spec.Replicas = &oldZ
+	_, err = client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Update(context.TODO(), updated, metav1.UpdateOptions{})
+	Expect(err).ToNot(HaveOccurred())
+
+	EventuallyWithOffset(1, func() (replicas int32, err error) {
+		deployment, err := client.KubernetesInterface().AppsV1().Deployments(operatorNamespace).Get(context.TODO(), d.Name, metav1.GetOptions{})
+		if deployment != nil {
+			replicas = deployment.Status.Replicas
+		}
+		return
+	}).Should(Equal(oldZ), "expected exactly 1 Deployment")
+}
+
 func getMetricsFromPod(client operatorclient.ClientInterface, pod *corev1.Pod, port string) []Metric {
 	ctx.Ctx().Logf("querying pod %s/%s\n", pod.GetNamespace(), pod.GetName())