diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go
index b99fc7cf1eb3..7dd5ae64fa87 100644
--- a/pkg/daemons/control/server.go
+++ b/pkg/daemons/control/server.go
@@ -68,8 +68,7 @@ func Server(ctx context.Context, cfg *config.Control) error {
 				if err := secretsencrypt.Register(ctx,
 					controllerName,
 					cfg,
-					cfg.Runtime.Core.Core().V1().Node(),
-					cfg.Runtime.Core.Core().V1().Secret()); err != nil {
+					cfg.Runtime.Core.Core().V1().Node()); err != nil {
 					logrus.Errorf("Failed to register %s controller: %v", controllerName, err)
 				}
 			}
diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go
index 03976d7f9e02..ac820fdb798f 100644
--- a/pkg/secretsencrypt/controller.go
+++ b/pkg/secretsencrypt/controller.go
@@ -38,7 +38,7 @@ type handler struct {
 	ctx           context.Context
 	controlConfig *config.Control
 	nodes         coreclient.NodeController
-	secrets       coreclient.SecretController
+	k8s           *kubernetes.Clientset
 	recorder      record.EventRecorder
 }
 
@@ -47,12 +47,14 @@ func Register(
 	controllerName string,
 	controlConfig *config.Control,
 	nodes coreclient.NodeController,
-	secrets coreclient.SecretController,
 ) error {
 	restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor)
 	if err != nil {
 		return err
 	}
+	// For secrets we need a much higher QPS than what wrangler provides, so we create a new clientset
+	restConfig.QPS = 200
+	restConfig.Burst = 200
 	k8s, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		return err
@@ -62,7 +64,7 @@ func Register(
 		ctx:           ctx,
 		controlConfig: controlConfig,
 		nodes:         nodes,
-		secrets:       secrets,
+		k8s:           k8s,
 		recorder:      util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault),
 	}
 
@@ -217,7 +219,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (
 
 func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error {
 	secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
-		return h.secrets.List(metav1.NamespaceAll, opts)
+		return h.k8s.CoreV1().Secrets(metav1.NamespaceAll).List(h.ctx, opts)
 	}))
 	secretPager.PageSize = secretListPageSize
 
@@ -227,10 +229,10 @@ func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error {
 		if !ok {
 			return errors.New("failed to convert object to Secret")
 		}
-		if _, err := h.secrets.Update(secret); err != nil && !apierrors.IsConflict(err) {
+		if _, err := h.k8s.CoreV1().Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) {
 			return fmt.Errorf("failed to update secret: %v", err)
 		}
-		if i != 0 && i%10 == 0 {
+		if i != 0 && i%50 == 0 {
 			h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
 		}
 		i++