diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go
index 705ef28b15..ea18bba158 100644
--- a/test/e2e/e2e_leaderelection_test.go
+++ b/test/e2e/e2e_leaderelection_test.go
@@ -27,6 +27,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	clientset "k8s.io/client-go/kubernetes"
@@ -85,6 +86,9 @@ func TestLeaderElection(t *testing.T) {
 	s1.Client = clientSet
 	s1.DeschedulingInterval = 5 * time.Second
 	s1.LeaderElection.LeaderElect = true
+	s1.LeaderElection.RetryPeriod = metav1.Duration{
+		Duration: time.Second,
+	}
 	s1.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG")
 	s1.PolicyConfigFile = "./policy_leaderelection_a.yaml"
 
@@ -95,9 +99,20 @@ func TestLeaderElection(t *testing.T) {
 	s2.Client = clientSet
 	s2.DeschedulingInterval = 5 * time.Second
 	s2.LeaderElection.LeaderElect = true
+	s2.LeaderElection.RetryPeriod = metav1.Duration{
+		Duration: time.Second,
+	}
 	s2.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG")
 	s2.PolicyConfigFile = "./policy_leaderelection_b.yaml"
 
+	// Delete the descheduler lease
+	err = clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		t.Fatalf("Unable to remove kube-system/descheduler lease: %v", err)
+	} else {
+		t.Logf("Removed kube-system/descheduler lease")
+	}
+
 	t.Log("starting deschedulers")
 
 	go func() {
@@ -141,7 +156,11 @@ func TestLeaderElection(t *testing.T) {
 			t.Logf("Only the pods in %s namespace are evicted. Pods before: %s, Pods after %s", ns2, podListBOrg, podListB)
 		}
 	} else {
-		t.Fatalf("Pods are evicted in both namespaces. For %s namespace Pods before: %s, Pods after %s. And, for %s namespace Pods before: %s, Pods after: %s", ns1, podListAOrg, podListA, ns2, podListBOrg, podListB)
+		if left && right {
+			t.Fatalf("No pods evicted. Probably none of the deschedulers were running.")
+		} else {
+			t.Fatalf("Pods are evicted in both namespaces.\n\tFor %s namespace\n\tPods before: %s,\n\tPods after %s.\n\tAnd, for %s namespace\n\tPods before: %s,\n\tPods after: %s", ns1, podListAOrg, podListA, ns2, podListBOrg, podListB)
+		}
 	}
 }