support switching resource injector webhook to Fail

Using MatchConditions in the mutating webhook allow us
to only call the webhook for pods with annotation  `k8s.v1.cni.cncf.io/networks`

Adding this feature under a featureGate as the MatchConditions was introduced in
newer k8s version (beta on 1.28)

Signed-off-by: Sebastian Sch <sebassch@gmail.com>

bindata/manifests/webhook/003-webhook.yaml

@@ -15,7 +15,14 @@ webhooks:
   - name: network-resources-injector-config.k8s.io
     sideEffects: None
     admissionReviewVersions: ["v1", "v1beta1"]
+    {{- if .resourceInjectorMatchCondition}}
+    failurePolicy: Fail
+    matchConditions:
+      - name: 'include-networks-annotation'
+        expression: '"k8s.v1.cni.cncf.io/networks" in object.metadata.annotations'
+    {{- else }}
     failurePolicy: Ignore
+    {{- end}}
     clientConfig:
       service:
         name: network-resources-injector-service

controllers/sriovoperatorconfig_controller.go

@@ -247,6 +247,12 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(ctx context.Context, dc
 			data.Data["ExternalControlPlane"] = external
 		}
 
+		// check for ResourceInjectorMatchConditionFeatureGate feature gate
+		data.Data[consts.ResourceInjectorMatchConditionFeatureGate] = false
+		if resourceInjector, ok := dc.Spec.FeatureGates[consts.ResourceInjectorMatchConditionFeatureGate]; ok {
+			data.Data[consts.ResourceInjectorMatchConditionFeatureGate] = resourceInjector
+		}
+
 		objs, err := render.RenderDir(path, &data)
 		if err != nil {
 			logger.Error(err, "Fail to render webhook manifests")

controllers/sriovoperatorconfig_controller_test.go

@@ -7,7 +7,11 @@ import (
 
 	admv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo/v2"
@@ -93,6 +97,9 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 			cancel()
 			wg.Wait()
 		})
+
+		err = k8sClient.Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "default"}})
+		Expect(err).ToNot(HaveOccurred())
 	})
 
 	Context("When is up", func() {
@@ -103,8 +110,8 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 			config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{
 				EnableInjector:        true,
 				EnableOperatorWebhook: true,
-				// ConfigDaemonNodeSelector: map[string]string{},
-				LogLevel: 2,
+				LogLevel:              2,
+				FeatureGates:          map[string]bool{},
 			}
 			err = k8sClient.Update(ctx, config)
 			Expect(err).NotTo(HaveOccurred())
@@ -284,5 +291,44 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 				return strings.Join(daemonSet.Spec.Template.Spec.Containers[0].Args, " ")
 			}, util.APITimeout*10, util.RetryInterval).Should(ContainSubstring("disable-plugins=mellanox"))
 		})
+
+		It("should render the resourceInjectorMatchCondition in the mutation if feature flag is enabled and block only pods with the networks annotation", func() {
+			By("set the feature flag")
+			config := &sriovnetworkv1.SriovOperatorConfig{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
+
+			config.Spec.FeatureGates = map[string]bool{}
+			config.Spec.FeatureGates[consts.ResourceInjectorMatchConditionFeatureGate] = true
+			err := k8sClient.Update(ctx, config)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("checking the webhook have all the needed configuration")
+			mutateCfg := &admv1.MutatingWebhookConfiguration{}
+			err = wait.PollUntilContextTimeout(ctx, util.RetryInterval, util.APITimeout, true, func(ctx context.Context) (done bool, err error) {
+				err = k8sClient.Get(ctx, types.NamespacedName{Name: "network-resources-injector-config", Namespace: testNamespace}, mutateCfg)
+				if err != nil {
+					if errors.IsNotFound(err) {
+						return false, nil
+					}
+					return false, err
+				}
+				if len(mutateCfg.Webhooks) != 1 {
+					return false, nil
+				}
+				if *mutateCfg.Webhooks[0].FailurePolicy != admv1.Fail {
+					return false, nil
+				}
+				if len(mutateCfg.Webhooks[0].MatchConditions) != 1 {
+					return false, nil
+				}
+
+				if mutateCfg.Webhooks[0].MatchConditions[0].Name != "include-networks-annotation" {
+					return false, nil
+				}
+
+				return true, nil
+			})
+			Expect(err).ToNot(HaveOccurred())
+		})
 	})
 })

pkg/consts/constants.go

@@ -113,7 +113,13 @@ const (
 	KernelArgIntelIommu = "intel_iommu=on"
 	KernelArgIommuPt    = "iommu=pt"
 
+	// Feature gates
+	// ParallelNicConfigFeatureGate: allow to configure nics in parallel
 	ParallelNicConfigFeatureGate = "parallelNicConfig"
+
+	// ResourceInjectorMatchConditionFeatureGate: switch injector to fail policy and add mactch condition
+	// this will make the mutating webhook to be called only when a pod has 'k8s.v1.cni.cncf.io/networks' annotation
+	ResourceInjectorMatchConditionFeatureGate = "resourceInjectorMatchCondition"
 )
 
 const (