Improve Leader Election Hand Off

This commit introduces a couple of changes to the package server manager
to improve hand off between running pods during an upgrade or
redeployment.

1. The package server manager will now voluntarily release its lease on
manager exit which will speed up voluntary leader transition as the new
leader shouldn't have to wait the LeaseDuration time before taking
over.

I should note that enabling this setting expects that the binary will
immediately exit upon release.

2. The package server manager deployment has had its .strategy.type
field updated from Rolling to Recreate, which will prevent the new pod
from attempting to acquire the lease before the current leader has
shutdown and released its lease.

Signed-off-by: Alexander Greene <[email protected]>

Author: awgreene

cmd/package-server-manager/main.go

@@ -80,17 +80,18 @@ func run(cmd *cobra.Command, args []string) error {
 
 	packageserverCSVFields := fields.Set{"metadata.name": name}
 	mgr, err := ctrl.NewManager(restConfig, manager.Options{
-		Scheme:                  setupScheme(),
-		Namespace:               namespace,
-		MetricsBindAddress:      metricsAddr,
-		LeaderElection:          !disableLeaderElection,
-		LeaderElectionNamespace: namespace,
-		LeaderElectionID:        leaderElectionConfigmapName,
-		LeaseDuration:           &le.LeaseDuration.Duration,
-		RenewDeadline:           &le.RenewDeadline.Duration,
-		RetryPeriod:             &le.RetryPeriod.Duration,
-		HealthProbeBindAddress:  healthCheckAddr,
-		PprofBindAddress:        pprofAddr,
+		Scheme:                        setupScheme(),
+		Namespace:                     namespace,
+		MetricsBindAddress:            metricsAddr,
+		LeaderElection:                !disableLeaderElection,
+		LeaderElectionNamespace:       namespace,
+		LeaderElectionID:              leaderElectionConfigmapName,
+		LeaseDuration:                 &le.LeaseDuration.Duration,
+		RenewDeadline:                 &le.RenewDeadline.Duration,
+		RetryPeriod:                   &le.RetryPeriod.Duration,
+		HealthProbeBindAddress:        healthCheckAddr,
+		PprofBindAddress:              pprofAddr,
+		LeaderElectionReleaseOnCancel: true,
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
 				&olmv1alpha1.ClusterServiceVersion{}: {

manifests/0000_50_olm_06-psm-operator.deployment.ibm-cloud-managed.yaml

@@ -9,7 +9,7 @@ metadata:
     include.release.openshift.io/ibm-cloud-managed: "true"
 spec:
   strategy:
-    type: RollingUpdate
+    type: Recreate
   replicas: 1
   selector:
     matchLabels:

manifests/0000_50_olm_06-psm-operator.deployment.yaml

@@ -9,7 +9,7 @@ metadata:
     include.release.openshift.io/self-managed-high-availability: "true"
 spec:
   strategy:
-    type: RollingUpdate
+    type: Recreate
   replicas: 1
   selector:
     matchLabels:

scripts/generate_crds_manifests.sh

@@ -123,7 +123,7 @@ metadata:
   annotations:
 spec:
   strategy:
-    type: RollingUpdate
+    type: Recreate
   replicas: 1
   selector:
     matchLabels: