✨ added reconciles_total metric

Author: droot

pkg/controller/controller.go

@@ -78,15 +78,15 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error)
 
 	// Create controller with dependencies set
 	c := &controller.Controller{
-		Do:                      options.Reconciler,
-		Cache:                   mgr.GetCache(),
-		Config:                  mgr.GetConfig(),
-		Scheme:                  mgr.GetScheme(),
-		Client:                  mgr.GetClient(),
-		Recorder:                mgr.GetRecorder(name),
-		Queue:                   workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
+		Do:       options.Reconciler,
+		Cache:    mgr.GetCache(),
+		Config:   mgr.GetConfig(),
+		Scheme:   mgr.GetScheme(),
+		Client:   mgr.GetClient(),
+		Recorder: mgr.GetRecorder(name),
+		Queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
 		MaxConcurrentReconciles: options.MaxConcurrentReconciles,
-		Name:                    name,
+		Name: name,
 	}
 
 	// Add the controller as a Manager components

pkg/internal/controller/controller.go

@@ -214,13 +214,15 @@ func (c *Controller) processNextWorkItem() bool {
 		c.Queue.AddRateLimited(req)
 		log.Error(err, "Reconciler error", "controller", c.Name, "request", req)
 		ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
-
+		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "error").Inc()
 		return false
 	} else if result.RequeueAfter > 0 {
 		c.Queue.AddAfter(req, result.RequeueAfter)
+		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue_after").Inc()
 		return true
 	} else if result.Requeue {
 		c.Queue.AddRateLimited(req)
+		ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue").Inc()
 		return true
 	}
 
@@ -231,6 +233,7 @@ func (c *Controller) processNextWorkItem() bool {
 	// TODO(directxman12): What does 1 mean?  Do we want level constants?  Do we want levels at all?
 	log.V(1).Info("Successfully Reconciled", "controller", c.Name, "request", req)
 
+	ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "success").Inc()
 	// Return true, don't take a break
 	return true
 }

pkg/internal/controller/controller_test.go

@@ -63,9 +63,9 @@ var _ = Describe("controller", func() {
 		informers = &informertest.FakeInformers{}
 		ctrl = &Controller{
 			MaxConcurrentReconciles: 1,
-			Do:                      fakeReconcile,
-			Queue:                   queue,
-			Cache:                   informers,
+			Do:    fakeReconcile,
+			Queue: queue,
+			Cache: informers,
 		}
 		ctrl.InjectFunc(func(interface{}) error { return nil })
 	})
@@ -410,12 +410,12 @@ var _ = Describe("controller", func() {
 
 		Context("should update prometheus metrics", func() {
 			It("should requeue a Request if there is an error and continue processing items", func(done Done) {
-				var queueLength, reconcileErrs dto.Metric
+				var queueLength, reconcileErrs, reconcileTotal dto.Metric
 				ctrlmetrics.QueueLength.Reset()
 				Expect(func() error {
 					ctrlmetrics.QueueLength.WithLabelValues(ctrl.Name).Write(&queueLength)
 					if queueLength.GetGauge().GetValue() != 0.0 {
-						return fmt.Errorf("metrics not reset")
+						return fmt.Errorf("metric queue length not reset")
 					}
 					return nil
 				}()).Should(Succeed())
@@ -424,7 +424,24 @@ var _ = Describe("controller", func() {
 				Expect(func() error {
 					ctrlmetrics.ReconcileErrors.WithLabelValues(ctrl.Name).Write(&reconcileErrs)
 					if reconcileErrs.GetCounter().GetValue() != 0.0 {
-						return fmt.Errorf("metrics not reset")
+						return fmt.Errorf("metric reconcile errors not reset")
+					}
+					return nil
+				}()).Should(Succeed())
+
+				ctrlmetrics.ReconcileTotal.Reset()
+				Expect(func() error {
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "success").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 0.0 {
+						return fmt.Errorf("metric reconcile total with label success not reset")
+					}
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "error").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 0.0 {
+						return fmt.Errorf("metric reconcile total with label error not reset")
+					}
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 0.0 {
+						return fmt.Errorf("metric reconcile total with label retry not reset")
 					}
 					return nil
 				}()).Should(Succeed())
@@ -444,7 +461,7 @@ var _ = Describe("controller", func() {
 				Eventually(func() error {
 					ctrlmetrics.QueueLength.WithLabelValues(ctrl.Name).Write(&queueLength)
 					if queueLength.GetGauge().GetValue() != 1.0 {
-						return fmt.Errorf("metrics not updated")
+						return fmt.Errorf("metric queue length not updated")
 					}
 					return nil
 				}, 2.0).Should(Succeed())
@@ -455,6 +472,13 @@ var _ = Describe("controller", func() {
 					}
 					return nil
 				}, 2.0).Should(Succeed())
+				Eventually(func() error {
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "error").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 1.0 {
+						return fmt.Errorf("metric reconcile total not updated")
+					}
+					return nil
+				}, 2.0).Should(Succeed())
 
 				By("Invoking Reconciler a second time without error")
 				fakeReconcile.Err = nil
@@ -464,6 +488,23 @@ var _ = Describe("controller", func() {
 				Eventually(ctrl.Queue.Len).Should(Equal(0))
 				Eventually(func() int { return ctrl.Queue.NumRequeues(request) }).Should(Equal(0))
 
+				By("Examining ReconcileTotal post second reconcile")
+				Eventually(func() error {
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "error").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 1.0 {
+						return fmt.Errorf("metric reconcile total with label error not updated")
+					}
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "success").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 1.0 {
+						return fmt.Errorf("metric reconcile total with label success not updated")
+					}
+					ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry").Write(&reconcileTotal)
+					if reconcileTotal.GetCounter().GetValue() != 0.0 {
+						return fmt.Errorf("metric reconcile total with label error was updated")
+					}
+					return nil
+				}, 2.0).Should(Succeed())
+
 				close(done)
 			}, 2.0)
 

pkg/internal/controller/metrics/metrics.go

@@ -29,6 +29,15 @@ var (
 		Help: "Length of reconcile queue per controller",
 	}, []string{"controller"})
 
+	// ReconcileTotal is a prometheus counter metrics which holds the total
+	// number of reconciles per controller. It has two labels. controller label refers
+	// to the controller name and result label refers to the reconcile result i.e
+	// success, error, requeue, requeue_after.
+	ReconcileTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "controller_runtime_reconcile_total",
+		Help: "Total number of reconciles per controller",
+	}, []string{"controller", "result"})
+
 	// ReconcileErrors is a prometheus counter metrics which holds the total
 	// number of errors from the Reconciler
 	ReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
@@ -47,6 +56,7 @@ var (
 func init() {
 	metrics.Registry.MustRegister(
 		QueueLength,
+		ReconcileTotal,
 		ReconcileErrors,
 		ReconcileTime,
 		// expose process metrics like CPU, Memory, file descriptor usage etc.