diff --git a/go.mod b/go.mod index c9ca19ffa..51633e802 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/jaypipes/ghw v0.9.0 - github.com/k8snetworkplumbingwg/govdpa v0.1.4 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3 github.com/onsi/ginkgo/v2 v2.11.0 @@ -92,6 +91,7 @@ require ( github.com/jaypipes/pcidb v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/k8snetworkplumbingwg/govdpa v0.1.4 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.17 // indirect diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index 560dcf60d..95053f470 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -313,6 +313,21 @@ func (mr *MockHostHelpersInterfaceMockRecorder) GetCurrentKernelArgs() *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentKernelArgs", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetCurrentKernelArgs)) } +// GetDriverByBusAndDevice mocks base method. +func (m *MockHostHelpersInterface) GetDriverByBusAndDevice(bus, device string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDriverByBusAndDevice", bus, device) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDriverByBusAndDevice indicates an expected call of GetDriverByBusAndDevice. +func (mr *MockHostHelpersInterfaceMockRecorder) GetDriverByBusAndDevice(bus, device interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDriverByBusAndDevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetDriverByBusAndDevice), bus, device) +} + // GetLinkType mocks base method. func (m *MockHostHelpersInterface) GetLinkType(name string) string { m.ctrl.T.Helper() diff --git a/pkg/host/internal/kernel/kernel.go b/pkg/host/internal/kernel/kernel.go index 3bb539374..d007b72bf 100644 --- a/pkg/host/internal/kernel/kernel.go +++ b/pkg/host/internal/kernel/kernel.go @@ -250,6 +250,15 @@ func (k *kernel) HasDriver(pciAddr string) (bool, string) { return false, "" } +// GetDriverByBusAndDevice returns driver for the device or error. +// returns "", nil if the device has no driver. +// bus - the bus path in the sysfs, e.g. "pci" or "vdpa" +// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA +func (k *kernel) GetDriverByBusAndDevice(bus, device string) (string, error) { + log.Log.V(2).Info("GetDriverByBusAndDevice(): get driver for device", "bus", bus, "device", device) + return getDriverByBusAndDevice(bus, device) +} + func (k *kernel) TryEnableRdma() (bool, error) { log.Log.V(2).Info("tryEnableRdma()") chrootDefinition := utils.GetChrootExtension() diff --git a/pkg/host/internal/kernel/kernel_test.go b/pkg/host/internal/kernel/kernel_test.go index b8eef8b07..a76a05807 100644 --- a/pkg/host/internal/kernel/kernel_test.go +++ b/pkg/host/internal/kernel/kernel_test.go @@ -195,5 +195,27 @@ var _ = Describe("Kernel", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0") }) }) + Context("GetDriverByBusAndDevice", func() { + It("device has driver", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/bus/vdpa/devices/vdpa:0000:d8:00.3"}, + Symlinks: map[string]string{ + "/sys/bus/vdpa/devices/vdpa:0000:d8:00.3/driver": "../../../../../bus/vdpa/drivers/vhost_vdpa"}, + }) + driver, err := k.GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.3") + Expect(err).NotTo(HaveOccurred()) + Expect(driver).To(Equal("vhost_vdpa")) + }) + It("device has no driver", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{ + "/sys/bus/vdpa/devices/vdpa:0000:d8:00.3"}, + }) + driver, err := k.GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.3") + Expect(err).NotTo(HaveOccurred()) + Expect(driver).To(BeEmpty()) + }) + }) }) }) diff --git a/pkg/host/internal/lib/govdpa/govdpa.go b/pkg/host/internal/lib/govdpa/govdpa.go deleted file mode 100644 index e85f89db1..000000000 --- a/pkg/host/internal/lib/govdpa/govdpa.go +++ /dev/null @@ -1,40 +0,0 @@ -package govdpa - -import ( - "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" -) - -func New() GoVdpaLib { - return &libWrapper{} -} - -type VdpaDevice interface { - kvdpa.VdpaDevice -} - -//go:generate ../../../../../bin/mockgen -destination mock/mock_govdpa.go -source govdpa.go -type GoVdpaLib interface { - // GetVdpaDevice returns the vdpa device information by a vdpa device name - GetVdpaDevice(vdpaDeviceName string) (VdpaDevice, error) - // AddVdpaDevice adds a new vdpa device to the given management device - AddVdpaDevice(mgmtDeviceName string, vdpaDeviceName string) error - // DeleteVdpaDevice deletes a vdpa device - DeleteVdpaDevice(vdpaDeviceName string) error -} - -type libWrapper struct{} - -// GetVdpaDevice returns the vdpa device information by a vdpa device name -func (w *libWrapper) GetVdpaDevice(name string) (VdpaDevice, error) { - return kvdpa.GetVdpaDevice(name) -} - -// AddVdpaDevice adds a new vdpa device to the given management device -func (w *libWrapper) AddVdpaDevice(mgmtDeviceName string, vdpaDeviceName string) error { - return kvdpa.AddVdpaDevice(mgmtDeviceName, vdpaDeviceName) -} - -// DeleteVdpaDevice deletes a vdpa device -func (w *libWrapper) DeleteVdpaDevice(name string) error { - return kvdpa.DeleteVdpaDevice(name) -} diff --git a/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go b/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go deleted file mode 100644 index cdeeac743..000000000 --- a/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go +++ /dev/null @@ -1,187 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: govdpa.go - -// Package mock_govdpa is a generated GoMock package. -package mock_govdpa - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - kvdpa "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" - govdpa "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa" -) - -// MockVdpaDevice is a mock of VdpaDevice interface. -type MockVdpaDevice struct { - ctrl *gomock.Controller - recorder *MockVdpaDeviceMockRecorder -} - -// MockVdpaDeviceMockRecorder is the mock recorder for MockVdpaDevice. -type MockVdpaDeviceMockRecorder struct { - mock *MockVdpaDevice -} - -// NewMockVdpaDevice creates a new mock instance. -func NewMockVdpaDevice(ctrl *gomock.Controller) *MockVdpaDevice { - mock := &MockVdpaDevice{ctrl: ctrl} - mock.recorder = &MockVdpaDeviceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockVdpaDevice) EXPECT() *MockVdpaDeviceMockRecorder { - return m.recorder -} - -// Driver mocks base method. -func (m *MockVdpaDevice) Driver() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Driver") - ret0, _ := ret[0].(string) - return ret0 -} - -// Driver indicates an expected call of Driver. -func (mr *MockVdpaDeviceMockRecorder) Driver() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Driver", reflect.TypeOf((*MockVdpaDevice)(nil).Driver)) -} - -// MgmtDev mocks base method. -func (m *MockVdpaDevice) MgmtDev() kvdpa.MgmtDev { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MgmtDev") - ret0, _ := ret[0].(kvdpa.MgmtDev) - return ret0 -} - -// MgmtDev indicates an expected call of MgmtDev. -func (mr *MockVdpaDeviceMockRecorder) MgmtDev() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MgmtDev", reflect.TypeOf((*MockVdpaDevice)(nil).MgmtDev)) -} - -// Name mocks base method. -func (m *MockVdpaDevice) Name() string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Name") - ret0, _ := ret[0].(string) - return ret0 -} - -// Name indicates an expected call of Name. -func (mr *MockVdpaDeviceMockRecorder) Name() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockVdpaDevice)(nil).Name)) -} - -// ParentDevicePath mocks base method. -func (m *MockVdpaDevice) ParentDevicePath() (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParentDevicePath") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParentDevicePath indicates an expected call of ParentDevicePath. -func (mr *MockVdpaDeviceMockRecorder) ParentDevicePath() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParentDevicePath", reflect.TypeOf((*MockVdpaDevice)(nil).ParentDevicePath)) -} - -// VhostVdpa mocks base method. -func (m *MockVdpaDevice) VhostVdpa() kvdpa.VhostVdpa { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "VhostVdpa") - ret0, _ := ret[0].(kvdpa.VhostVdpa) - return ret0 -} - -// VhostVdpa indicates an expected call of VhostVdpa. -func (mr *MockVdpaDeviceMockRecorder) VhostVdpa() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VhostVdpa", reflect.TypeOf((*MockVdpaDevice)(nil).VhostVdpa)) -} - -// VirtioNet mocks base method. -func (m *MockVdpaDevice) VirtioNet() kvdpa.VirtioNet { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "VirtioNet") - ret0, _ := ret[0].(kvdpa.VirtioNet) - return ret0 -} - -// VirtioNet indicates an expected call of VirtioNet. -func (mr *MockVdpaDeviceMockRecorder) VirtioNet() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VirtioNet", reflect.TypeOf((*MockVdpaDevice)(nil).VirtioNet)) -} - -// MockGoVdpaLib is a mock of GoVdpaLib interface. -type MockGoVdpaLib struct { - ctrl *gomock.Controller - recorder *MockGoVdpaLibMockRecorder -} - -// MockGoVdpaLibMockRecorder is the mock recorder for MockGoVdpaLib. -type MockGoVdpaLibMockRecorder struct { - mock *MockGoVdpaLib -} - -// NewMockGoVdpaLib creates a new mock instance. -func NewMockGoVdpaLib(ctrl *gomock.Controller) *MockGoVdpaLib { - mock := &MockGoVdpaLib{ctrl: ctrl} - mock.recorder = &MockGoVdpaLibMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockGoVdpaLib) EXPECT() *MockGoVdpaLibMockRecorder { - return m.recorder -} - -// AddVdpaDevice mocks base method. -func (m *MockGoVdpaLib) AddVdpaDevice(mgmtDeviceName, vdpaDeviceName string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddVdpaDevice", mgmtDeviceName, vdpaDeviceName) - ret0, _ := ret[0].(error) - return ret0 -} - -// AddVdpaDevice indicates an expected call of AddVdpaDevice. -func (mr *MockGoVdpaLibMockRecorder) AddVdpaDevice(mgmtDeviceName, vdpaDeviceName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVdpaDevice", reflect.TypeOf((*MockGoVdpaLib)(nil).AddVdpaDevice), mgmtDeviceName, vdpaDeviceName) -} - -// DeleteVdpaDevice mocks base method. -func (m *MockGoVdpaLib) DeleteVdpaDevice(vdpaDeviceName string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteVdpaDevice", vdpaDeviceName) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteVdpaDevice indicates an expected call of DeleteVdpaDevice. -func (mr *MockGoVdpaLibMockRecorder) DeleteVdpaDevice(vdpaDeviceName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVdpaDevice", reflect.TypeOf((*MockGoVdpaLib)(nil).DeleteVdpaDevice), vdpaDeviceName) -} - -// GetVdpaDevice mocks base method. -func (m *MockGoVdpaLib) GetVdpaDevice(vdpaDeviceName string) (govdpa.VdpaDevice, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVdpaDevice", vdpaDeviceName) - ret0, _ := ret[0].(govdpa.VdpaDevice) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetVdpaDevice indicates an expected call of GetVdpaDevice. -func (mr *MockGoVdpaLibMockRecorder) GetVdpaDevice(vdpaDeviceName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVdpaDevice", reflect.TypeOf((*MockGoVdpaLib)(nil).GetVdpaDevice), vdpaDeviceName) -} diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go index c31e78207..c687b7455 100644 --- a/pkg/host/internal/lib/netlink/mock/mock_netlink.go +++ b/pkg/host/internal/lib/netlink/mock/mock_netlink.go @@ -200,3 +200,46 @@ func (mr *MockNetlinkLibMockRecorder) LinkSetVfPortGUID(link, vf, portguid inter mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfPortGUID", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfPortGUID), link, vf, portguid) } + +// VDPADelDev mocks base method. +func (m *MockNetlinkLib) VDPADelDev(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VDPADelDev", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// VDPADelDev indicates an expected call of VDPADelDev. +func (mr *MockNetlinkLibMockRecorder) VDPADelDev(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VDPADelDev", reflect.TypeOf((*MockNetlinkLib)(nil).VDPADelDev), name) +} + +// VDPAGetDevByName mocks base method. +func (m *MockNetlinkLib) VDPAGetDevByName(name string) (*netlink0.VDPADev, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VDPAGetDevByName", name) + ret0, _ := ret[0].(*netlink0.VDPADev) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VDPAGetDevByName indicates an expected call of VDPAGetDevByName. +func (mr *MockNetlinkLibMockRecorder) VDPAGetDevByName(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VDPAGetDevByName", reflect.TypeOf((*MockNetlinkLib)(nil).VDPAGetDevByName), name) +} + +// VDPANewDev mocks base method. +func (m *MockNetlinkLib) VDPANewDev(name, mgmtBus, mgmtName string, params netlink0.VDPANewDevParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VDPANewDev", name, mgmtBus, mgmtName, params) + ret0, _ := ret[0].(error) + return ret0 +} + +// VDPANewDev indicates an expected call of VDPANewDev. +func (mr *MockNetlinkLibMockRecorder) VDPANewDev(name, mgmtBus, mgmtName, params interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VDPANewDev", reflect.TypeOf((*MockNetlinkLib)(nil).VDPANewDev), name, mgmtBus, mgmtName, params) +} diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go index f5a86f61c..782e39998 100644 --- a/pkg/host/internal/lib/netlink/netlink.go +++ b/pkg/host/internal/lib/netlink/netlink.go @@ -41,6 +41,15 @@ type NetlinkLib interface { // Equivalent to: `devlink dev eswitch set $dev mode switchdev` // Equivalent to: `devlink dev eswitch set $dev mode legacy` DevLinkSetEswitchMode(dev *netlink.DevlinkDevice, newMode string) error + // VDPAGetDevByName returns VDPA device selected by name + // Equivalent to: `vdpa dev show ` + VDPAGetDevByName(name string) (*netlink.VDPADev, error) + // VDPADelDev removes VDPA device + // Equivalent to: `vdpa dev del ` + VDPADelDev(name string) error + // VDPANewDev adds new VDPA device + // Equivalent to: `vdpa dev add name mgmtdev /mgmtName [params]` + VDPANewDev(name, mgmtBus, mgmtName string, params netlink.VDPANewDevParams) error } type libWrapper struct{} @@ -93,3 +102,21 @@ func (w *libWrapper) DevLinkGetDeviceByName(bus string, device string) (*netlink func (w *libWrapper) DevLinkSetEswitchMode(dev *netlink.DevlinkDevice, newMode string) error { return netlink.DevLinkSetEswitchMode(dev, newMode) } + +// VDPAGetDevByName returns VDPA device selected by name +// Equivalent to: `vdpa dev show ` +func (w *libWrapper) VDPAGetDevByName(name string) (*netlink.VDPADev, error) { + return netlink.VDPAGetDevByName(name) +} + +// VDPADelDev removes VDPA device +// Equivalent to: `vdpa dev del ` +func (w *libWrapper) VDPADelDev(name string) error { + return netlink.VDPADelDev(name) +} + +// VDPANewDev adds new VDPA device +// Equivalent to: `vdpa dev add name mgmtdev /mgmtName [params]` +func (w *libWrapper) VDPANewDev(name, mgmtBus, mgmtName string, params netlink.VDPANewDevParams) error { + return netlink.VDPANewDev(name, mgmtBus, mgmtName, params) +} diff --git a/pkg/host/internal/vdpa/vdpa.go b/pkg/host/internal/vdpa/vdpa.go index 010098f8b..6024b294f 100644 --- a/pkg/host/internal/vdpa/vdpa.go +++ b/pkg/host/internal/vdpa/vdpa.go @@ -5,51 +5,62 @@ import ( "fmt" "syscall" - "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" + "github.com/vishvananda/netlink" "sigs.k8s.io/controller-runtime/pkg/log" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa" + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" ) +const ( + VhostVdpaDriver = "vhost_vdpa" + VirtioVdpaDriver = "virtio_vdpa" +) + type vdpa struct { - kernel types.KernelInterface - vdpaLib govdpa.GoVdpaLib + kernel types.KernelInterface + netlinkLib netlinkLibPkg.NetlinkLib } -func New(k types.KernelInterface, vdpaLib govdpa.GoVdpaLib) types.VdpaInterface { - return &vdpa{kernel: k, vdpaLib: vdpaLib} +func New(k types.KernelInterface, netlinkLib netlinkLibPkg.NetlinkLib) types.VdpaInterface { + return &vdpa{kernel: k, netlinkLib: netlinkLib} } // CreateVDPADevice creates VDPA device for VF with required type, // pciAddr - PCI address of the VF // vdpaType - type of the VDPA device to create: virtio of vhost func (v *vdpa) CreateVDPADevice(pciAddr, vdpaType string) error { - log.Log.V(2).Info("CreateVDPADevice(): create VDPA device for VF", - "device", pciAddr, "vdpaType", vdpaType) + expectedVDPAName := generateVDPADevName(pciAddr) + funcLog := log.Log.WithValues("device", pciAddr, "vdpaType", vdpaType, "name", expectedVDPAName) + funcLog.V(2).Info("CreateVDPADevice(): create VDPA device for VF") expectedDriver := vdpaTypeToDriver(vdpaType) if expectedDriver == "" { return fmt.Errorf("unknown VDPA device type: %s", vdpaType) } - expectedVDPAName := generateVDPADevName(pciAddr) - _, err := v.vdpaLib.GetVdpaDevice(expectedVDPAName) + _, err := v.netlinkLib.VDPAGetDevByName(expectedVDPAName) if err != nil { if !errors.Is(err, syscall.ENODEV) { - log.Log.Error(err, "CreateVDPADevice(): fail to check if VDPA device exist", - "device", pciAddr, "vdpaDev", expectedVDPAName) + funcLog.Error(err, "CreateVDPADevice(): fail to check if VDPA device exist") return err } - if err := v.vdpaLib.AddVdpaDevice("pci/"+pciAddr, expectedVDPAName); err != nil { - log.Log.Error(err, "CreateVDPADevice(): fail to create VDPA device", - "device", pciAddr, "vdpaDev", expectedVDPAName) - return err + // first try to create VDPA device with MaxVQP parameter set to 32 to exactly match HW offloading use-case with the + // old swtichdev implementation. Create device without MaxVQP parameter if it is not supported. + if err := v.netlinkLib.VDPANewDev(expectedVDPAName, constants.BusPci, pciAddr, netlink.VDPANewDevParams{MaxVQP: 32}); err != nil { + if !errors.Is(err, syscall.ENOTSUP) { + funcLog.Error(err, "CreateVDPADevice(): fail to create VDPA device with MaxVQP parameter") + return err + } + funcLog.V(2).Info("failed to create VDPA device with MaxVQP parameter, try without it") + if err := v.netlinkLib.VDPANewDev(expectedVDPAName, constants.BusPci, pciAddr, netlink.VDPANewDevParams{}); err != nil { + funcLog.Error(err, "CreateVDPADevice(): fail to create VDPA device without MaxVQP parameter") + return err + } } } err = v.kernel.BindDriverByBusAndDevice(constants.BusVdpa, expectedVDPAName, expectedDriver) if err != nil { - log.Log.Error(err, "CreateVDPADevice(): fail to bind VDPA device to the driver", - "device", pciAddr, "vdpaDev", expectedVDPAName, "driver", expectedDriver) + funcLog.Error(err, "CreateVDPADevice(): fail to bind VDPA device to the driver") return err } return nil @@ -58,17 +69,16 @@ func (v *vdpa) CreateVDPADevice(pciAddr, vdpaType string) error { // DeleteVDPADevice removes VDPA device for provided pci address // pciAddr - PCI address of the VF func (v *vdpa) DeleteVDPADevice(pciAddr string) error { - log.Log.V(2).Info("DeleteVDPADevice(): delete VDPA device for VF", - "device", pciAddr) expectedVDPAName := generateVDPADevName(pciAddr) - if err := v.vdpaLib.DeleteVdpaDevice(expectedVDPAName); err != nil { + funcLog := log.Log.WithValues("device", pciAddr, "name", expectedVDPAName) + funcLog.V(2).Info("DeleteVDPADevice(): delete VDPA device for VF") + + if err := v.netlinkLib.VDPADelDev(expectedVDPAName); err != nil { if errors.Is(err, syscall.ENODEV) { - log.Log.V(2).Info("DeleteVDPADevice(): VDPA device not found", - "device", pciAddr, "name", expectedVDPAName) + funcLog.V(2).Info("DeleteVDPADevice(): VDPA device not found") return nil } - log.Log.Error(err, "DeleteVDPADevice(): fail to remove VDPA device", - "device", pciAddr, "name", expectedVDPAName) + funcLog.Error(err, "DeleteVDPADevice(): fail to remove VDPA device") return err } return nil @@ -78,25 +88,30 @@ func (v *vdpa) DeleteVDPADevice(pciAddr string) error { // returns empty string if VDPA device not found or unknown driver is in use // pciAddr - PCI address of the VF func (v *vdpa) DiscoverVDPAType(pciAddr string) string { - expectedVDPADevName := generateVDPADevName(pciAddr) - vdpaDev, err := v.vdpaLib.GetVdpaDevice(expectedVDPADevName) + expectedVDPAName := generateVDPADevName(pciAddr) + funcLog := log.Log.WithValues("device", pciAddr, "name", expectedVDPAName) + funcLog.V(2).Info("DiscoverVDPAType() discover device type") + _, err := v.netlinkLib.VDPAGetDevByName(expectedVDPAName) if err != nil { if errors.Is(err, syscall.ENODEV) { - log.Log.V(2).Info("discoverVDPAType(): VDPA device for VF not found", "device", pciAddr) + funcLog.V(2).Info("DiscoverVDPAType(): VDPA device for VF not found") return "" } - log.Log.Error(err, "getVfInfo(): unable to get VF VDPA devices", "device", pciAddr) + funcLog.Error(err, "DiscoverVDPAType(): unable to get VF VDPA devices") + return "" + } + driverName, err := v.kernel.GetDriverByBusAndDevice(constants.BusVdpa, expectedVDPAName) + if err != nil { + funcLog.Error(err, "DiscoverVDPAType(): unable to get driver info for VF VDPA devices") return "" } - driverName := vdpaDev.Driver() if driverName == "" { - log.Log.V(2).Info("discoverVDPAType(): VDPA device has no driver", "device", pciAddr) + funcLog.V(2).Info("DiscoverVDPAType(): VDPA device has no driver") return "" } vdpaType := vdpaDriverToType(driverName) if vdpaType == "" { - log.Log.Error(nil, "getVfInfo(): WARNING: unknown VDPA device type for VF, ignore", - "device", pciAddr, "driver", driverName) + funcLog.Error(nil, "DiscoverVDPAType(): WARNING: unknown VDPA device type for VF, ignore") } return vdpaType } @@ -110,9 +125,9 @@ func generateVDPADevName(pciAddr string) string { func vdpaTypeToDriver(vdpaType string) string { switch vdpaType { case constants.VdpaTypeVhost: - return kvdpa.VhostVdpaDriver + return VhostVdpaDriver case constants.VdpaTypeVirtio: - return kvdpa.VirtioVdpaDriver + return VirtioVdpaDriver default: return "" } @@ -121,9 +136,9 @@ func vdpaTypeToDriver(vdpaType string) string { // vdpa driver name to type conversion func vdpaDriverToType(driver string) string { switch driver { - case kvdpa.VhostVdpaDriver: + case VhostVdpaDriver: return constants.VdpaTypeVhost - case kvdpa.VirtioVdpaDriver: + case VirtioVdpaDriver: return constants.VdpaTypeVirtio default: return "" diff --git a/pkg/host/internal/vdpa/vdpa_test.go b/pkg/host/internal/vdpa/vdpa_test.go index 3c668aa44..d58bf0b97 100644 --- a/pkg/host/internal/vdpa/vdpa_test.go +++ b/pkg/host/internal/vdpa/vdpa_test.go @@ -5,31 +5,30 @@ import ( "syscall" "github.com/golang/mock/gomock" + "github.com/vishvananda/netlink" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" - govdpaMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa/mock" + netlinkMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" hostMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" ) var _ = Describe("VDPA", func() { var ( - v types.VdpaInterface - libMock *govdpaMock.MockGoVdpaLib - vdpaDevMock *govdpaMock.MockVdpaDevice - kernelMock *hostMock.MockHostManagerInterface + v types.VdpaInterface + libMock *netlinkMock.MockNetlinkLib + kernelMock *hostMock.MockHostManagerInterface testCtrl *gomock.Controller testErr = fmt.Errorf("test-error") ) BeforeEach(func() { testCtrl = gomock.NewController(GinkgoT()) - libMock = govdpaMock.NewMockGoVdpaLib(testCtrl) - vdpaDevMock = govdpaMock.NewMockVdpaDevice(testCtrl) + libMock = netlinkMock.NewMockNetlinkLib(testCtrl) kernelMock = hostMock.NewMockHostManagerInterface(testCtrl) v = New(kernelMock, libMock) }) @@ -41,27 +40,34 @@ var _ = Describe("VDPA", func() { return v.CreateVDPADevice("0000:d8:00.2", constants.VdpaTypeVhost) } It("Created", func() { - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) - libMock.EXPECT().AddVdpaDevice("pci/"+"0000:d8:00.2", "vdpa:0000:d8:00.2").Return(nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + libMock.EXPECT().VDPANewDev("vdpa:0000:d8:00.2", "pci", "0000:d8:00.2", netlink.VDPANewDevParams{MaxVQP: 32}).Return(nil) + kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(nil) + Expect(callFunc()).NotTo(HaveOccurred()) + }) + It("Created without MaxVQP", func() { + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + libMock.EXPECT().VDPANewDev("vdpa:0000:d8:00.2", "pci", "0000:d8:00.2", netlink.VDPANewDevParams{MaxVQP: 32}).Return(syscall.ENOTSUP) + libMock.EXPECT().VDPANewDev("vdpa:0000:d8:00.2", "pci", "0000:d8:00.2", netlink.VDPANewDevParams{}).Return(nil) kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(nil) Expect(callFunc()).NotTo(HaveOccurred()) }) It("Already exist", func() { - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(nil) Expect(callFunc()).NotTo(HaveOccurred()) }) It("Fail to Get device", func() { - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, testErr) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(nil, testErr) Expect(callFunc()).To(MatchError(testErr)) }) It("Fail to Create device", func() { - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) - libMock.EXPECT().AddVdpaDevice("pci/"+"0000:d8:00.2", "vdpa:0000:d8:00.2").Return(testErr) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + libMock.EXPECT().VDPANewDev("vdpa:0000:d8:00.2", "pci", "0000:d8:00.2", netlink.VDPANewDevParams{MaxVQP: 32}).Return(testErr) Expect(callFunc()).To(MatchError(testErr)) }) It("Fail to Bind device", func() { - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(testErr) Expect(callFunc()).To(MatchError(testErr)) }) @@ -71,15 +77,15 @@ var _ = Describe("VDPA", func() { return v.DeleteVDPADevice("0000:d8:00.2") } It("Removed", func() { - libMock.EXPECT().DeleteVdpaDevice("vdpa:0000:d8:00.2").Return(nil) + libMock.EXPECT().VDPADelDev("vdpa:0000:d8:00.2").Return(nil) Expect(callFunc()).NotTo(HaveOccurred()) }) It("Not found", func() { - libMock.EXPECT().DeleteVdpaDevice("vdpa:0000:d8:00.2").Return(syscall.ENODEV) + libMock.EXPECT().VDPADelDev("vdpa:0000:d8:00.2").Return(syscall.ENODEV) Expect(callFunc()).NotTo(HaveOccurred()) }) It("Fail to delete device", func() { - libMock.EXPECT().DeleteVdpaDevice("vdpa:0000:d8:00.2").Return(testErr) + libMock.EXPECT().VDPADelDev("vdpa:0000:d8:00.2").Return(testErr) Expect(callFunc()).To(MatchError(testErr)) }) }) @@ -88,28 +94,37 @@ var _ = Describe("VDPA", func() { return v.DiscoverVDPAType("0000:d8:00.2") } It("No device", func() { - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + Expect(callFunc()).To(BeEmpty()) + }) + It("Fail to read device", func() { + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(nil, testErr) Expect(callFunc()).To(BeEmpty()) }) It("No driver", func() { - vdpaDevMock.EXPECT().Driver().Return("") - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) + kernelMock.EXPECT().GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2").Return("", nil) Expect(callFunc()).To(BeEmpty()) }) It("Unknown driver", func() { - vdpaDevMock.EXPECT().Driver().Return("something") - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) + kernelMock.EXPECT().GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2").Return("something", nil) Expect(callFunc()).To(BeEmpty()) }) It("Vhost driver", func() { - vdpaDevMock.EXPECT().Driver().Return("vhost_vdpa") - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) + kernelMock.EXPECT().GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2").Return("vhost_vdpa", nil) Expect(callFunc()).To(Equal("vhost")) }) It("Virtio driver", func() { - vdpaDevMock.EXPECT().Driver().Return("virtio_vdpa") - libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) + kernelMock.EXPECT().GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2").Return("virtio_vdpa", nil) Expect(callFunc()).To(Equal("virtio")) }) + It("Fail to read driver", func() { + libMock.EXPECT().VDPAGetDevByName("vdpa:0000:d8:00.2").Return(&netlink.VDPADev{}, nil) + kernelMock.EXPECT().GetDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2").Return("", testErr) + Expect(callFunc()).To(BeEmpty()) + }) }) }) diff --git a/pkg/host/manager.go b/pkg/host/manager.go index 5b397dbb9..7e5a92001 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -3,7 +3,6 @@ package host import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/service" @@ -38,13 +37,13 @@ type hostManager struct { func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { dpUtils := dputils.New() - nl := netlink.New() + netlinkLib := netlink.New() k := kernel.New(utilsInterface) - n := network.New(utilsInterface, dpUtils, nl) + n := network.New(utilsInterface, dpUtils, netlinkLib) sv := service.New(utilsInterface) u := udev.New(utilsInterface) - sr := sriov.New(utilsInterface, k, n, u, nl, dpUtils) - v := vdpa.New(k, govdpa.New()) + sr := sriov.New(utilsInterface, k, n, u, netlinkLib, dpUtils) + v := vdpa.New(k, netlinkLib) return &hostManager{ utilsInterface, diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index 3e9da3d59..ea69d9f42 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -268,6 +268,21 @@ func (mr *MockHostManagerInterfaceMockRecorder) GetCurrentKernelArgs() *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentKernelArgs", reflect.TypeOf((*MockHostManagerInterface)(nil).GetCurrentKernelArgs)) } +// GetDriverByBusAndDevice mocks base method. +func (m *MockHostManagerInterface) GetDriverByBusAndDevice(bus, device string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDriverByBusAndDevice", bus, device) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDriverByBusAndDevice indicates an expected call of GetDriverByBusAndDevice. +func (mr *MockHostManagerInterfaceMockRecorder) GetDriverByBusAndDevice(bus, device interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDriverByBusAndDevice", reflect.TypeOf((*MockHostManagerInterface)(nil).GetDriverByBusAndDevice), bus, device) +} + // GetLinkType mocks base method. func (m *MockHostManagerInterface) GetLinkType(name string) string { m.ctrl.T.Helper() diff --git a/pkg/host/types/interfaces.go b/pkg/host/types/interfaces.go index 7ceaa3e90..be0c824ad 100644 --- a/pkg/host/types/interfaces.go +++ b/pkg/host/types/interfaces.go @@ -37,6 +37,11 @@ type KernelInterface interface { BindDriverByBusAndDevice(bus, device, driver string) error // HasDriver returns try if the virtual function is bind to a driver HasDriver(pciAddr string) (bool, string) + // GetDriverByBusAndDevice returns driver for the device or error. + // returns "", nil if the device has no driver. + // bus - the bus path in the sysfs, e.g. "pci" or "vdpa" + // device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA + GetDriverByBusAndDevice(bus, device string) (string, error) // RebindVfToDefaultDriver rebinds the virtual function to is default driver RebindVfToDefaultDriver(pciAddr string) error // UnbindDriverByBusAndDevice unbind device identified by bus and device ID from the driver