leader election bugfix: Delete evicted leader pods (#2210)

* leader election bugfix: Delete evicted leader pods

Before this patch, when the leader pod is hard evicted but not deleted
the leader lock configmap is not garbage collected and subsequent
operator pods can never become leader. With this patch, an operator
attempting to become the leader is able to delete evicted leader pods
triggering garbage collection and allowing leader election to continue.

Sometimes, evicted operator pods will remain, even with this patch.
This occurs when the leader operator pod is evicted and a new operator
pod is created on the same node. In this case, the new pod will also be
evicted. When an operator pod is created on a non-failing node, leader
election will delete only the evicted leader pod, leaving any evicted
operator pods that were not the leader.

To replicate the evicted state, I used a `kind` cluster with 2 worker
nodes with altered kubelet configuration and a memory-hog version of the
memcached operator.
See the [altered operator docs](https://github.com/asmacdo/go-memcahced-operator/blob/explosive-operator/README.md)

Author: asmacdo

pkg/leader/leader.go

@@ -109,7 +109,35 @@ func Become(ctx context.Context, lockName string) error {
 			log.Info("Became the leader.")
 			return nil
 		case apierrors.IsAlreadyExists(err):
-			log.Info("Not the leader. Waiting.")
+			existingOwners := existing.GetOwnerReferences()
+			switch {
+			case len(existingOwners) != 1:
+				log.Info("Leader lock configmap must have exactly one owner reference.", "ConfigMap", existing)
+			case existingOwners[0].Kind != "Pod":
+				log.Info("Leader lock configmap owner reference must be a pod.", "OwnerReference", existingOwners[0])
+			default:
+				leaderPod := &corev1.Pod{}
+				key = crclient.ObjectKey{Namespace: ns, Name: existingOwners[0].Name}
+				err = client.Get(ctx, key, leaderPod)
+				switch {
+				case apierrors.IsNotFound(err):
+					log.Info("Leader pod has been deleted, waiting for garbage collection do remove the lock.")
+				case err != nil:
+					return err
+				case isPodEvicted(*leaderPod) && leaderPod.GetDeletionTimestamp() == nil:
+					log.Info("Operator pod with leader lock has been evicted.", "leader", leaderPod.Name)
+					log.Info("Deleting evicted leader.")
+					// Pod may not delete immediately, continue with backoff
+					err := client.Delete(ctx, leaderPod)
+					if err != nil {
+						log.Error(err, "Leader pod could not be deleted.")
+					}
+
+				default:
+					log.Info("Not the leader. Waiting.")
+				}
+			}
+
 			select {
 			case <-time.After(wait.Jitter(backoff, .2)):
 				if backoff < maxBackoffInterval {
@@ -143,3 +171,9 @@ func myOwnerRef(ctx context.Context, client crclient.Client, ns string) (*metav1
 	}
 	return owner, nil
 }
+
+func isPodEvicted(pod corev1.Pod) bool {
+	podFailed := pod.Status.Phase == corev1.PodFailed
+	podEvicted := pod.Status.Reason == "Evicted"
+	return podFailed && podEvicted
+}