Reestablish watch for the certificate paths

Author: m-messiah

pkg/certwatcher/certwatcher.go

@@ -27,6 +27,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
 	logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
 )
@@ -120,8 +121,31 @@ func (cw *CertWatcher) Start(ctx context.Context) error {
 	return cw.watcher.Close()
 }
 
+func (cw *CertWatcher) ensureAllFilesAreWatched() {
+	watchList := sets.New(cw.watcher.WatchList()...)
+	difference := sets.New(cw.certPath, cw.keyPath).Difference(watchList)
+	if difference.Len() == 0 {
+		return
+	}
+
+	for _, missingWatchPath := range difference.UnsortedList() {
+		log.V(1).Info("re-adding missing watch", "path", missingWatchPath)
+		if err := cw.watcher.Add(missingWatchPath); err != nil {
+			log.Error(err, "failed to add watch", "path", missingWatchPath)
+			return
+		}
+	}
+
+	log.V(1).Info("all files are watched again", "list", cw.watcher.WatchList())
+
+	if err := cw.ReadCertificate(); err != nil {
+		log.Error(err, "error re-reading certificate")
+	}
+}
+
 // Watch reads events from the watcher's channel and reacts to changes.
 func (cw *CertWatcher) Watch() {
+	watcherHealthTimer := time.NewTicker(time.Second)
 	for {
 		select {
 		case event, ok := <-cw.watcher.Events:
@@ -139,6 +163,8 @@ func (cw *CertWatcher) Watch() {
 			}
 
 			log.Error(err, "certificate watch error")
+		case <-watcherHealthTimer.C:
+			cw.ensureAllFilesAreWatched()
 		}
 	}
 }

pkg/certwatcher/certwatcher_test.go

@@ -34,6 +34,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
 )
@@ -113,7 +114,7 @@ var _ = Describe("CertWatcher", func() {
 			Eventually(func() bool {
 				secondcert, _ := watcher.GetCertificate(nil)
 				first := firstcert.PrivateKey.(*rsa.PrivateKey)
-				return first.Equal(secondcert.PrivateKey)
+				return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber
 			}).ShouldNot(BeTrue())
 
 			ctxCancel()
@@ -143,14 +144,41 @@ var _ = Describe("CertWatcher", func() {
 			Eventually(func() bool {
 				secondcert, _ := watcher.GetCertificate(nil)
 				first := firstcert.PrivateKey.(*rsa.PrivateKey)
-				return first.Equal(secondcert.PrivateKey)
+				return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber
 			}).ShouldNot(BeTrue())
 
 			ctxCancel()
 			Eventually(doneCh, "4s").Should(BeClosed())
 			Expect(called.Load()).To(BeNumerically(">=", 1))
 		})
 
+		It("should reload currentCert after move out", func() {
+			doneCh := startWatcher()
+			called := atomic.Int64{}
+			watcher.RegisterCallback(func(crt tls.Certificate) {
+				called.Add(1)
+				Expect(crt.Certificate).ToNot(BeEmpty())
+			})
+
+			firstcert, _ := watcher.GetCertificate(nil)
+
+			Expect(os.Rename(certPath, certPath+".old")).To(Succeed())
+			Expect(os.Rename(keyPath, keyPath+".old")).To(Succeed())
+
+			err := writeCerts(certPath, keyPath, "192.168.0.3")
+			Expect(err).ToNot(HaveOccurred())
+
+			Eventually(func() bool {
+				secondcert, _ := watcher.GetCertificate(nil)
+				first := firstcert.PrivateKey.(*rsa.PrivateKey)
+				return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber
+			}, "10s", "1s").ShouldNot(BeTrue())
+
+			ctxCancel()
+			Eventually(doneCh, "4s").Should(BeClosed())
+			Expect(called.Load()).To(BeNumerically(">=", 1))
+		})
+
 		Context("prometheus metric read_certificate_total", func() {
 			var readCertificateTotalBefore float64
 			var readCertificateErrorsBefore float64