leader election bugfix: Delete evicted leader pods

Before this patch, when the leader pod is evicted but not deleted, the
leader lock configmap is not garbage collected and subsequent operaters
can never become leader. With this patch, an operator attempting to
become the leader is able to delete the evicted operator pod, triggering
garbage collection and allowing leader election to continue.

Sometimes, evicted operator pods will remain, even with this patch.
This occurs when the leader operator pod is evicted, a new operator pod
is created on the same node. In this case, the new pod will also be
evicted. When an operator pod is created on a non-failing node, leader
election will delete only the evicted leader pod, leaving any evicted
operator pods that were not the leader.

To replicate the evicted state, I used a `kind` cluster with 2 worker
nodes with altered kubelet configuration:
  `memory.available`: Set about 8Gi less than the host machine `avail
  Mem` from `top`.
  `evictionPressureTransitionPeriod: 5s`: Allows kubelet to evict
  earlier.
With these settings in place (and kubelet restarted), a memory-explosion
function is triggered, which results in the eviction of all pods on that
node.

operator-framework#1305
Fixes operator-framework#1305
operator-framework#1874
Fixes operator-framework#1874

Makefile

@@ -67,7 +67,7 @@ generate: gen-cli-doc gen-test-framework  ## Run all generate targets
 .PHONY: install release_builds release
 
 install: ## Build & install the Operator SDK CLI binary
-	$(Q)go install \
+	$(Q)go1.13.4 install \
 		-gcflags "all=-trimpath=${GOPATH}" \
 		-asmflags "all=-trimpath=${GOPATH}" \
 		-ldflags " \

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/ghodss/yaml v1.0.1-0.20180820084758-c7ce16629ff4
+	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-logr/logr v0.1.0
 	github.com/go-logr/zapr v0.1.1
 	github.com/gobuffalo/packr v1.30.1 // indirect
@@ -31,6 +32,7 @@ require (
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.1.2
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190605231540-b8a4faf68e36
 	github.com/operator-framework/operator-registry v1.1.1
 	github.com/pborman/uuid v1.2.0
@@ -49,7 +51,9 @@ require (
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	go.uber.org/zap v1.10.0
 	golang.org/x/tools v0.0.0-20191018212557-ed542cd5b28a
+	google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e // indirect
 	gopkg.in/gorp.v1 v1.7.2 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0
 	k8s.io/apiextensions-apiserver v0.0.0

pkg/leader/leader.go

@@ -109,7 +109,32 @@ func Become(ctx context.Context, lockName string) error {
 			log.Info("Became the leader.")
 			return nil
 		case apierrors.IsAlreadyExists(err):
-			log.Info("Not the leader. Waiting.")
+			log.Info("Checking status of leader.")
+			existing := &corev1.ConfigMap{}
+			key := crclient.ObjectKey{Namespace: ns, Name: lockName}
+			err := client.Get(ctx, key, existing)
+			if err != nil {
+				return err
+			}
+			leaderPod := &corev1.Pod{}
+			key = crclient.ObjectKey{Namespace: ns, Name: existing.ObjectMeta.OwnerReferences[0].Name}
+			err = client.Get(ctx, key, leaderPod)
+			if err != nil {
+				return err
+			}
+
+			// TODO Should this be configurable?
+			if leaderPod.Status.Phase == corev1.PodFailed && leaderPod.Status.Reason == "Evicted" {
+				err := client.Delete(ctx, leaderPod)
+				if err == nil {
+					log.Info("Evicted leader pod has been deleted.")
+				} else {
+					log.Info("Evicted leader pod could not be deleted.")
+					return err
+				}
+			} else {
+				log.Info("Not the leader. Waiting.")
+			}
 			select {
 			case <-time.After(wait.Jitter(backoff, .2)):
 				if backoff < maxBackoffInterval {