Deploy RDMA CNI as a part of SR-IOV Operator

Signed-off-by: Ivan Kolodiazhnyi <ikolodiazhny@nvidia.com>
k8snetworkplumbingwg · Mar 26, 2024 · af30c8a · af30c8a
1 parent f9a5002
commit af30c8a
Show file tree

Hide file tree

Showing 9 changed files with 38 additions and 9 deletions.
diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml
@@ -70,6 +70,23 @@ spec:
             - name: os-release
               mountPath: /host/etc/os-release
               readOnly: true
+        {{- if ne .RDMACNIImage "" }}
+        - name: rdma-cni
+          image: {{.RDMACNIImage}}
+          command:
+            - /bin/sh
+            - -c
+            - cp /usr/bin/rdma /host/opt/cni/bin/
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 10Mi
+          volumeMounts:
+            - name: cnibin
+              mountPath: /host/opt/cni/bin
+      {{- end }}
       {{- if .UsedSystemdMode}}
         - name: sriov-service-copy
           image: {{.Image}}

diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go
@@ -162,6 +162,7 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context,
 	data.Data["Namespace"] = vars.Namespace
 	data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE")
 	data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE")
+	data.Data["RDMACNIImage"] = os.Getenv("RDMA_CNI_IMAGE")
 	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
 	data.Data["ClusterType"] = vars.ClusterType
 	data.Data["DevMode"] = os.Getenv("DEV_MODE")

diff --git a/controllers/suite_test.go b/controllers/suite_test.go
@@ -109,6 +109,8 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	err = os.Setenv("SRIOV_INFINIBAND_CNI_IMAGE", "mock-image")
 	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("RDMA_CNI_IMAGE", "mock-image")
+	Expect(err).NotTo(HaveOccurred())
 	err = os.Setenv("SRIOV_DEVICE_PLUGIN_IMAGE", "mock-image")
 	Expect(err).NotTo(HaveOccurred())
 	err = os.Setenv("NETWORK_RESOURCES_INJECTOR_IMAGE", "mock-image")

diff --git a/deploy/operator.yaml b/deploy/operator.yaml
@@ -54,6 +54,8 @@ spec:
               value: $SRIOV_CNI_IMAGE
             - name: SRIOV_INFINIBAND_CNI_IMAGE
               value: $SRIOV_INFINIBAND_CNI_IMAGE
+            - name: RDMA_CNI_IMAGE
+              value: $RDMA_CNI_IMAGE
             - name: SRIOV_DEVICE_PLUGIN_IMAGE
               value: $SRIOV_DEVICE_PLUGIN_IMAGE
             - name: NETWORK_RESOURCES_INJECTOR_IMAGE

diff --git a/deployment/sriov-network-operator/README.md b/deployment/sriov-network-operator/README.md
@@ -118,12 +118,13 @@ This section contains general parameters that apply to both the operator and dae
 
 ### Images parameters
 
-| Name | description |
-| ---- | ----------- |
-| `images.operator` | Operator controller image |
-| `images.sriovConfigDaemon` | Daemon node agent image |
-| `images.sriovCni` | SR-IOV CNI image |
-| `images.ibSriovCni` | InfiniBand SR-IOV CNI image |
-| `images.sriovDevicePlugin` | SR-IOV device plugin image |
-| `images.resourcesInjector` | Resources Injector image |
-| `images.webhook` | Operator Webhook image |
+| Name                       | description                 |
+|----------------------------|-----------------------------|
+| `images.operator`          | Operator controller image   |
+| `images.sriovConfigDaemon` | Daemon node agent image     |
+| `images.sriovCni`          | SR-IOV CNI image            |
+| `images.ibSriovCni`        | InfiniBand SR-IOV CNI image |
+| `images.rdmaCni`           | RDMA CNI image              |
+| `images.sriovDevicePlugin` | SR-IOV device plugin image  |
+| `images.resourcesInjector` | Resources Injector image    |
+| `images.webhook`           | Operator Webhook image      |
diff --git a/deployment/sriov-network-operator/templates/operator.yaml b/deployment/sriov-network-operator/templates/operator.yaml
@@ -56,6 +56,8 @@ spec:
               value: {{ .Values.images.sriovCni }}
             - name: SRIOV_INFINIBAND_CNI_IMAGE
               value: {{ .Values.images.ibSriovCni }}
+            - name: RDMA_CNI_IMAGE
+              value: {{ .Values.images.rdmaCni }}
             - name: SRIOV_DEVICE_PLUGIN_IMAGE
               value: {{ .Values.images.sriovDevicePlugin }}
             - name: NETWORK_RESOURCES_INJECTOR_IMAGE

diff --git a/deployment/sriov-network-operator/values.yaml b/deployment/sriov-network-operator/values.yaml
@@ -95,6 +95,7 @@ images:
   sriovConfigDaemon: ghcr.io/k8snetworkplumbingwg/sriov-network-operator-config-daemon
   sriovCni: ghcr.io/k8snetworkplumbingwg/sriov-cni
   ibSriovCni: ghcr.io/k8snetworkplumbingwg/ib-sriov-cni
+  rdmaCni: ghcr.io/k8snetworkplumbingwg/rdma-cni
   sriovDevicePlugin: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin
   resourcesInjector: ghcr.io/k8snetworkplumbingwg/network-resources-injector
   webhook: ghcr.io/k8snetworkplumbingwg/sriov-network-operator-webhook

diff --git a/hack/env.sh b/hack/env.sh
@@ -1,6 +1,7 @@
 if [ -z $SKIP_VAR_SET ]; then
         export SRIOV_CNI_IMAGE=${SRIOV_CNI_IMAGE:-ghcr.io/k8snetworkplumbingwg/sriov-cni}
         export SRIOV_INFINIBAND_CNI_IMAGE=${SRIOV_INFINIBAND_CNI_IMAGE:-ghcr.io/k8snetworkplumbingwg/ib-sriov-cni}
+        export RDMA_CNI_IMAGE=${RDMA_CNI_IMAGE-ghcr.io/k8snetworkplumbingwg/rdma-cni}
         export SRIOV_DEVICE_PLUGIN_IMAGE=${SRIOV_DEVICE_PLUGIN_IMAGE:-ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin}
         export NETWORK_RESOURCES_INJECTOR_IMAGE=${NETWORK_RESOURCES_INJECTOR_IMAGE:-ghcr.io/k8snetworkplumbingwg/network-resources-injector}
         export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE=${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE:-ghcr.io/k8snetworkplumbingwg/sriov-network-operator-config-daemon}
@@ -9,6 +10,7 @@ if [ -z $SKIP_VAR_SET ]; then
 else
         [ -z $SRIOV_CNI_IMAGE ] && echo "SRIOV_CNI_IMAGE is empty but SKIP_VAR_SET is set" && exit 1
         [ -z $SRIOV_INFINIBAND_CNI_IMAGE ] && echo "SRIOV_INFINIBAND_CNI_IMAGE is empty but SKIP_VAR_SET is set" && exit 1
+        [ -z ${$RDMA_CNI_IMAGE+set} ] && echo "RDMA_CNI_IMAGE is empty but SKIP_VAR_SET is set" && exit 1
         [ -z $SRIOV_DEVICE_PLUGIN_IMAGE ] && echo "SRIOV_DEVICE_PLUGIN_IMAGE is empty but SKIP_VAR_SET is set" && exit 1
         [ -z $NETWORK_RESOURCES_INJECTOR_IMAGE ] && echo "NETWORK_RESOURCES_INJECTOR_IMAGE is empty but SKIP_VAR_SET is set" && exit 1
         [ -z $SRIOV_NETWORK_CONFIG_DAEMON_IMAGE ] && echo "SRIOV_NETWORK_CONFIG_DAEMON_IMAGE is empty but SKIP_VAR_SET is set" && exit 1

diff --git a/hack/run-e2e-test.sh b/hack/run-e2e-test.sh
@@ -7,6 +7,7 @@ KUBECONFIG=${KUBECONFIG:-/root/env/ign/auth/kubeconfig}
 
 echo ${SRIOV_CNI_IMAGE}
 echo ${SRIOV_INFINIBAND_CNI_IMAGE}
+echo ${RDMA_CNI_IMAGE}
 echo ${SRIOV_DEVICE_PLUGIN_IMAGE}
 echo ${NETWORK_RESOURCES_INJECTOR_IMAGE}
 echo ${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}