Replace govdpa lib calls with netlink

This required to support additional configuration options and to remove need to mount /dev/ folder to the container (govdpa requires it) Signed-off-by: Yury Kulazhenkov <ykulazhenkov@nvidia.com>
k8snetworkplumbingwg · Feb 23, 2024 · 7251d38 · 7251d38
1 parent 3d31324
commit 7251d38
Show file tree

Hide file tree

Showing 13 changed files with 234 additions and 296 deletions.
diff --git a/go.mod b/go.mod
@@ -13,7 +13,6 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/jaypipes/ghw v0.9.0
-	github.com/k8snetworkplumbingwg/govdpa v0.1.4
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
 	github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3
 	github.com/onsi/ginkgo/v2 v2.11.0
@@ -92,6 +91,7 @@ require (
 	github.com/jaypipes/pcidb v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k8snetworkplumbingwg/govdpa v0.1.4 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect

diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go
diff --git a/pkg/host/internal/kernel/kernel.go b/pkg/host/internal/kernel/kernel.go
@@ -250,6 +250,15 @@ func (k *kernel) HasDriver(pciAddr string) (bool, string) {
 	return false, ""
 }
 
+// GetDriverByBusAndDevice returns driver for the device or error.
+// returns "", nil if the device has no driver.
+// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
+// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
+func (k *kernel) GetDriverByBusAndDevice(bus, device string) (string, error) {
+	log.Log.V(2).Info("GetDriverByBusAndDevice(): get driver for device", "bus", bus, "device", device)
+	return getDriverByBusAndDevice(bus, device)
+}
+
 func (k *kernel) TryEnableRdma() (bool, error) {
 	log.Log.V(2).Info("tryEnableRdma()")
 	chrootDefinition := utils.GetChrootExtension()

diff --git a/pkg/host/internal/kernel/kernel_test.go b/pkg/host/internal/kernel/kernel_test.go
@@ -195,5 +195,27 @@ var _ = Describe("Kernel", func() {
 				helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0")
 			})
 		})
+		Context("GetDriverByBusAndDevice", func() {
+			It("device has driver", func() {
+				helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{
+					Dirs: []string{
+						"/sys/bus/vdpa/devices/vdpa:0000:d8:00.3"},
+					Symlinks: map[string]string{
+						"/sys/bus/vdpa/devices/vdpa:0000:d8:00.3/driver": "../../../../../bus/vdpa/drivers/vhost_vdpa"},
+				})
+				driver, err := k.GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.3")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(driver).To(Equal("vhost_vdpa"))
+			})
+			It("device has no driver", func() {
+				helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{
+					Dirs: []string{
+						"/sys/bus/vdpa/devices/vdpa:0000:d8:00.3"},
+				})
+				driver, err := k.GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.3")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(driver).To(BeEmpty())
+			})
+		})
 	})
 })
diff --git a/pkg/host/internal/lib/govdpa/govdpa.go b/pkg/host/internal/lib/govdpa/govdpa.go
diff --git a/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go b/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go
diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go
diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go
@@ -41,6 +41,15 @@ type NetlinkLib interface {
 	// Equivalent to: `devlink dev eswitch set $dev mode switchdev`
 	// Equivalent to: `devlink dev eswitch set $dev mode legacy`
 	DevLinkSetEswitchMode(dev *netlink.DevlinkDevice, newMode string) error
+	// VDPAGetDevByName returns VDPA device selected by name
+	// Equivalent to: `vdpa dev show <name>`
+	VDPAGetDevByName(name string) (*netlink.VDPADev, error)
+	// VDPADelDev removes VDPA device
+	// Equivalent to: `vdpa dev del <name>`
+	VDPADelDev(name string) error
+	// VDPANewDev adds new VDPA device
+	// Equivalent to: `vdpa dev add name <name> mgmtdev <mgmtBus>/mgmtName [params]`
+	VDPANewDev(name, mgmtBus, mgmtName string, params netlink.VDPANewDevParams) error
 }
 
 type libWrapper struct{}
@@ -93,3 +102,21 @@ func (w *libWrapper) DevLinkGetDeviceByName(bus string, device string) (*netlink
 func (w *libWrapper) DevLinkSetEswitchMode(dev *netlink.DevlinkDevice, newMode string) error {
 	return netlink.DevLinkSetEswitchMode(dev, newMode)
 }
+
+// VDPAGetDevByName returns VDPA device selected by name
+// Equivalent to: `vdpa dev show <name>`
+func (w *libWrapper) VDPAGetDevByName(name string) (*netlink.VDPADev, error) {
+	return netlink.VDPAGetDevByName(name)
+}
+
+// VDPADelDev removes VDPA device
+// Equivalent to: `vdpa dev del <name>`
+func (w *libWrapper) VDPADelDev(name string) error {
+	return netlink.VDPADelDev(name)
+}
+
+// VDPANewDev adds new VDPA device
+// Equivalent to: `vdpa dev add name <name> mgmtdev <mgmtBus>/mgmtName [params]`
+func (w *libWrapper) VDPANewDev(name, mgmtBus, mgmtName string, params netlink.VDPANewDevParams) error {
+	return netlink.VDPANewDev(name, mgmtBus, mgmtName, params)
+}