vhost-vdpa support

SriovNetworkNodePolicy API: VDPA device type can take now the following values: "virtio" and "vhost" Vhost/vdpa: Same behaviour of virtio/vdpa apart for the drivers that are loaded in the kernel (vhost-vdpa instead of virtio-vdpa) Signed-off-by: Leonardo Milleri <lmilleri@redhat.com>
k8snetworkplumbingwg · Aug 4, 2023 · f3ebe10 · f3ebe10
1 parent d9902ca
commit f3ebe10
Show file tree

Hide file tree

Showing 13 changed files with 322 additions and 29 deletions.
diff --git a/api/v1/helper_test.go b/api/v1/helper_test.go
@@ -94,14 +94,37 @@ func newNodePolicy() *v1.SriovNetworkNodePolicy {
 	}
 }
 
-func newVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
+func newVirtioVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
 	return &v1.SriovNetworkNodePolicy{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "p1",
 		},
 		Spec: v1.SriovNetworkNodePolicySpec{
 			DeviceType: consts.DeviceTypeNetDevice,
 			VdpaType:   consts.VdpaTypeVirtio,
+			NicSelector: v1.SriovNetworkNicSelector{
+				PfNames:     []string{"ens803f1#2-3"},
+				RootDevices: []string{"0000:86:00.1"},
+				Vendor:      "8086",
+			},
+			NodeSelector: map[string]string{
+				"feature.node.kubernetes.io/network-sriov.capable": "true",
+			},
+			NumVfs:       4,
+			Priority:     99,
+			ResourceName: "virtiovdpa",
+		},
+	}
+}
+
+func newVhostVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
+	return &v1.SriovNetworkNodePolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "p1",
+		},
+		Spec: v1.SriovNetworkNodePolicySpec{
+			DeviceType: consts.DeviceTypeNetDevice,
+			VdpaType:   consts.VdpaTypeVhost,
 			NicSelector: v1.SriovNetworkNicSelector{
 				PfNames:     []string{"ens803f1"},
 				RootDevices: []string{"0000:86:00.1"},
@@ -112,7 +135,7 @@ func newVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
 			},
 			NumVfs:       2,
 			Priority:     99,
-			ResourceName: "p1res",
+			ResourceName: "vhostvdpa",
 		},
 	}
 }
@@ -437,6 +460,55 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
 				},
 			},
 		},
+		{
+			// vdpa policy with same priority (both virtio and vhost), VfRange's do not overlap so all is merged
+			tname: "one vdpa policy present same pf same priority partitioning",
+			currentState: func() *v1.SriovNetworkNodeState {
+				st := newNodeState()
+				st.Spec.Interfaces = []v1.Interface{
+					{
+						Name:       "ens803f1",
+						NumVfs:     4,
+						PciAddress: "0000:86:00.1",
+						VfGroups: []v1.VfGroup{
+							{
+								DeviceType:   consts.DeviceTypeNetDevice,
+								VdpaType:     consts.VdpaTypeVhost,
+								ResourceName: "vhostvdpa",
+								VfRange:      "0-1",
+								PolicyName:   "p2",
+							},
+						},
+					},
+				}
+				return st
+			}(),
+			policy: newVirtioVdpaNodePolicy(),
+			equalP: true,
+			expectedInterfaces: []v1.Interface{
+				{
+					Name:       "ens803f1",
+					NumVfs:     4,
+					PciAddress: "0000:86:00.1",
+					VfGroups: []v1.VfGroup{
+						{
+							DeviceType:   consts.DeviceTypeNetDevice,
+							VdpaType:     consts.VdpaTypeVirtio,
+							ResourceName: "virtiovdpa",
+							VfRange:      "2-3",
+							PolicyName:   "p1",
+						},
+						{
+							DeviceType:   consts.DeviceTypeNetDevice,
+							VdpaType:     consts.VdpaTypeVhost,
+							ResourceName: "vhostvdpa",
+							VfRange:      "0-1",
+							PolicyName:   "p2",
+						},
+					},
+				},
+			},
+		},
 		{
 			// policy with same priority that overwrites the 2 present groups in
 			// SriovNetworkNodeState because they overlap VfRange
@@ -637,7 +709,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
 	}
 }
 
-func TestVdpaNodePolicyApply(t *testing.T) {
+func TestVirtioVdpaNodePolicyApply(t *testing.T) {
 	testtable := []struct {
 		tname              string
 		currentState       *v1.SriovNetworkNodeState
@@ -647,20 +719,67 @@ func TestVdpaNodePolicyApply(t *testing.T) {
 		expectedErr        bool
 	}{
 		{
-			tname:        "vdpa configuration",
+			tname:        "virtio/vdpa configuration",
 			currentState: newNodeState(),
-			policy:       newVdpaNodePolicy(),
+			policy:       newVirtioVdpaNodePolicy(),
 			equalP:       false,
 			expectedInterfaces: []v1.Interface{
 				{
 					Name:       "ens803f1",
-					NumVfs:     2,
+					NumVfs:     4,
 					PciAddress: "0000:86:00.1",
 					VfGroups: []v1.VfGroup{
 						{
 							DeviceType:   consts.DeviceTypeNetDevice,
 							VdpaType:     consts.VdpaTypeVirtio,
-							ResourceName: "p1res",
+							ResourceName: "virtiovdpa",
+							VfRange:      "2-3",
+							PolicyName:   "p1",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tc := range testtable {
+		t.Run(tc.tname, func(t *testing.T) {
+			err := tc.policy.Apply(tc.currentState, tc.equalP)
+			if tc.expectedErr && err == nil {
+				t.Errorf("Apply expecting error.")
+			} else if !tc.expectedErr && err != nil {
+				t.Errorf("Apply error:\n%s", err)
+			}
+			if diff := cmp.Diff(tc.expectedInterfaces, tc.currentState.Spec.Interfaces); diff != "" {
+				t.Errorf("SriovNetworkNodeState spec diff (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestVhostVdpaNodePolicyApply(t *testing.T) {
+	testtable := []struct {
+		tname              string
+		currentState       *v1.SriovNetworkNodeState
+		policy             *v1.SriovNetworkNodePolicy
+		expectedInterfaces v1.Interfaces
+		equalP             bool
+		expectedErr        bool
+	}{
+		{
+			tname:        "vhost/vdpa configuration",
+			currentState: newNodeState(),
+			policy:       newVhostVdpaNodePolicy(),
+			equalP:       false,
+			expectedInterfaces: []v1.Interface{
+				{
+					Name:       "ens803f1",
+					NumVfs:     2,
+					PciAddress: "0000:86:00.1",
+					VfGroups: []v1.VfGroup{
+						{
+							DeviceType:   consts.DeviceTypeNetDevice,
+							VdpaType:     consts.VdpaTypeVhost,
+							ResourceName: "vhostvdpa",
 							VfRange:      "0-1",
 							PolicyName:   "p1",
 						},

diff --git a/api/v1/sriovnetworknodepolicy_types.go b/api/v1/sriovnetworknodepolicy_types.go
@@ -54,8 +54,8 @@ type SriovNetworkNodePolicySpec struct {
 	// +kubebuilder:validation:Enum=legacy;switchdev
 	// NIC Device Mode. Allowed value "legacy","switchdev".
 	EswitchMode string `json:"eSwitchMode,omitempty"`
-	// +kubebuilder:validation:Enum=virtio
-	// VDPA device type. Allowed value "virtio"
+	// +kubebuilder:validation:Enum=virtio;vhost
+	// VDPA device type. Allowed value "virtio", "vhost"
 	VdpaType string `json:"vdpaType,omitempty"`
 	// Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false.
 	ExcludeTopology bool `json:"excludeTopology,omitempty"`

diff --git a/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml b/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml
@@ -59,8 +59,18 @@ contents:
             extract_min_max_ids
             vdpaType=$(jq -c '.vdpaType' -r <<< $group)
             if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType != "" ]; then
-              vdpa_cmd="vdpa dev add name vdpa:"${VfPciAddr}" mgmtdev pci/"${VfPciAddr}" max_vqp 32"
+              vdpa_name=vdpa:${VfPciAddr}
+              vdpa_cmd="vdpa dev add name "${vdpa_name}" mgmtdev pci/"${VfPciAddr}" max_vqp 32"
               eval $vdpa_cmd
+              # make sure the right driver is bound to the vdpa device
+              current_driver=$(basename "$( readlink -f /sys/bus/vdpa/devices/"${vdpa_name}"/driver )")
+              if [ $current_driver == "vhost_vdpa" ] && [ $vdpaType == "virtio" ]; then
+                echo ${vdpa_name} > /sys/bus/vdpa/drivers/${current_driver}/unbind
+                echo ${vdpa_name} > /sys/bus/vdpa/drivers/virtio_vdpa/bind
+              elif [ $current_driver == "virtio_vdpa" ] && [ $vdpaType == "vhost" ]; then
+                echo ${vdpa_name} > /sys/bus/vdpa/drivers/${current_driver}/unbind
+                echo ${vdpa_name} > /sys/bus/vdpa/drivers/vhost_vdpa/bind
+              fi
             fi
           done
         done

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
@@ -117,9 +117,10 @@ spec:
                 description: SRIOV Network device plugin endpoint resource name
                 type: string
               vdpaType:
-                description: VDPA device type. Allowed value "virtio"
+                description: VDPA device type. Allowed value "virtio", "vhost"
                 enum:
                 - virtio
+                - vhost
                 type: string
             required:
             - nicSelector

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml
@@ -42,8 +42,9 @@ spec:
                 description: NodeSelector selects the nodes to be configured
                 type: object
               configurationMode:
-                description: Flag to enable the sriov-network-config-daemon to use
-                  a systemd mode instead of the regular method
+                description: 'Flag to enable the sriov-network-config-daemon to use
+                  a systemd service to configure SR-IOV devices on boot Default mode:
+                  daemon'
                 enum:
                 - daemon
                 - systemd

diff --git a/controllers/sriovnetworknodepolicy_controller_test.go b/controllers/sriovnetworknodepolicy_controller_test.go
@@ -152,7 +152,7 @@ func TestRenderDevicePluginConfigData(t *testing.T) {
 		expResource dptypes.ResourceConfList
 	}{
 		{
-			tname: "testVdpaVirtio",
+			tname: "testVirtioVdpaVirtio",
 			policy: sriovnetworkv1.SriovNetworkNodePolicy{
 				Spec: v1.SriovNetworkNodePolicySpec{
 					ResourceName: "resourceName",
@@ -170,6 +170,25 @@ func TestRenderDevicePluginConfigData(t *testing.T) {
 					},
 				},
 			},
+		}, {
+			tname: "testVhostVdpaVirtio",
+			policy: sriovnetworkv1.SriovNetworkNodePolicy{
+				Spec: v1.SriovNetworkNodePolicySpec{
+					ResourceName: "resourceName",
+					DeviceType:   consts.DeviceTypeNetDevice,
+					VdpaType:     consts.VdpaTypeVhost,
+				},
+			},
+			expResource: dptypes.ResourceConfList{
+				ResourceList: []dptypes.ResourceConfig{
+					{
+						ResourceName: "resourceName",
+						Selectors: mustMarshallSelector(t, &dptypes.NetDeviceSelectors{
+							VdpaType: dptypes.VdpaType(consts.VdpaTypeVhost),
+						}),
+					},
+				},
+			},
 		},
 		{
 			tname: "testExcludeTopology",

diff --git a/...yment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/...yment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
@@ -117,9 +117,10 @@ spec:
                 description: SRIOV Network device plugin endpoint resource name
                 type: string
               vdpaType:
-                description: VDPA device type. Allowed value "virtio"
+                description: VDPA device type. Allowed value "virtio", "vhost"
                 enum:
                 - virtio
+                - vhost
                 type: string
             required:
             - nicSelector

diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml
@@ -42,8 +42,9 @@ spec:
                 description: NodeSelector selects the nodes to be configured
                 type: object
               configurationMode:
-                description: Flag to enable the sriov-network-config-daemon to use
-                  a systemd mode instead of the regular method
+                description: 'Flag to enable the sriov-network-config-daemon to use
+                  a systemd service to configure SR-IOV devices on boot Default mode:
+                  daemon'
                 enum:
                 - daemon
                 - systemd

diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go
@@ -32,4 +32,5 @@ const (
 	DeviceTypeVfioPci   = "vfio-pci"
 	DeviceTypeNetDevice = "netdevice"
 	VdpaTypeVirtio      = "virtio"
+	VdpaTypeVhost       = "vhost"
 )
diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go
@@ -23,12 +23,14 @@ var PluginName = "generic_plugin"
 const (
 	Vfio = iota
 	VirtioVdpa
+	VhostVdpa
 )
 
 // driver name
 const (
 	vfioPciDriver    = "vfio_pci"
 	virtioVdpaDriver = "virtio_vdpa"
+	vhostVdpaDriver  = "vhost_vdpa"
 )
 
 // function type for determining if a given driver has to be loaded in the kernel
@@ -73,6 +75,13 @@ func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {
 		NeedDriverFunc: needDriverCheckVdpaType,
 		DriverLoaded:   false,
 	}
+	driverStateMap[VhostVdpa] = &DriverState{
+		DriverName:     vhostVdpaDriver,
+		DeviceType:     constants.DeviceTypeNetDevice,
+		VdpaType:       constants.VdpaTypeVhost,
+		NeedDriverFunc: needDriverCheckVdpaType,
+		DriverLoaded:   false,
+	}
 
 	return &GenericPlugin{
 		PluginName:     PluginName,

diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go
@@ -239,6 +239,63 @@ var _ = Describe("Generic plugin", func() {
 			concretePlugin.LoadDriverForTests(networkNodeState)
 			Expect(driverState.DriverLoaded).To(BeTrue())
 		})
+
+		It("should load vhost_vdpa driver", func() {
+			networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{
+				Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{
+					Interfaces: sriovnetworkv1.Interfaces{{
+						PciAddress: "0000:00:00.0",
+						NumVfs:     2,
+						Mtu:        1500,
+						VfGroups: []sriovnetworkv1.VfGroup{{
+							DeviceType:   "netdevice",
+							VdpaType:     "vhost",
+							PolicyName:   "policy-1",
+							ResourceName: "resource-1",
+							VfRange:      "0-1",
+							Mtu:          1500,
+						}}}},
+				},
+				Status: sriovnetworkv1.SriovNetworkNodeStateStatus{
+					Interfaces: sriovnetworkv1.InterfaceExts{{
+						PciAddress:  "0000:00:00.0",
+						NumVfs:      2,
+						TotalVfs:    2,
+						DeviceID:    "1015",
+						Vendor:      "15b3",
+						Name:        "sriovif1",
+						Mtu:         1500,
+						Mac:         "0c:42:a1:55:ee:46",
+						Driver:      "mlx5_core",
+						EswitchMode: "legacy",
+						LinkSpeed:   "25000 Mb/s",
+						LinkType:    "ETH",
+						VFs: []sriovnetworkv1.VirtualFunction{{
+							PciAddress: "0000:00:00.1",
+							DeviceID:   "1016",
+							Vendor:     "15b3",
+							VfID:       0,
+							Driver:     "mlx5_core",
+							Name:       "sriovif1v0",
+							Mtu:        1500,
+							Mac:        "8e:d6:2c:62:87:1b",
+						}, {
+							PciAddress: "0000:00:00.2",
+							DeviceID:   "1016",
+							Vendor:     "15b3",
+							VfID:       0,
+							Driver:     "mlx5_core",
+						}},
+					}},
+				},
+			}
+
+			concretePlugin := genericPlugin.(*generic.GenericPlugin)
+			driverStateMap := concretePlugin.GetDriverStateMap()
+			driverState := driverStateMap[generic.VhostVdpa]
+			concretePlugin.LoadDriverForTests(networkNodeState)
+			Expect(driverState.DriverLoaded).To(BeTrue())
+		})
 	})
 
 })