Add VdpaInterface to host package

The interface provides implementation
for the following functions:
* CreateVDPADevice - creates VDPA device for
VF with required type(vhost, virtio)
* DeleteVDPADevice - removes VDPA device
for provided VF
* DiscoverVDPAType - return type of existing
VDPA device for VF

Signed-off-by: Yury Kulazhenkov <ykulazhenkov@nvidia.com>

pkg/host/interface.go

@@ -2,6 +2,7 @@ package host
 
 import (
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
 )
 
@@ -26,6 +27,7 @@ type HostManagerInterface interface {
 	ServiceInterface
 	UdevInterface
 	SriovInterface
+	VdpaInterface
 }
 
 type hostManager struct {
@@ -35,6 +37,7 @@ type hostManager struct {
 	ServiceInterface
 	UdevInterface
 	SriovInterface
+	VdpaInterface
 }
 
 func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface {
@@ -43,6 +46,7 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface {
 	sv := newServiceInterface(utilsInterface)
 	u := newUdevInterface(utilsInterface)
 	sr := newSriovInterface(utilsInterface, k, n, u)
+	v := newVdpaInterface(k, govdpa.New())
 
 	return &hostManager{
 		utilsInterface,
@@ -51,5 +55,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface {
 		sv,
 		u,
 		sr,
+		v,
 	}
 }

pkg/host/vdpa.go

@@ -0,0 +1,140 @@
+package host
+
+import (
+	"errors"
+	"fmt"
+	"syscall"
+
+	"github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa"
+)
+
+type VdpaInterface interface {
+	// CreateVDPADevice creates VDPA device for VF with required type
+	CreateVDPADevice(pciAddr, vdpaType string) error
+	// DeleteVDPADevice removes VDPA device for provided pci address
+	DeleteVDPADevice(pciAddr string) error
+	// DiscoverVDPAType returns typo of existing VDPA device for VF,
+	// returns empty string if VDPA device not found or unknown driver is in use
+	DiscoverVDPAType(pciAddr string) string
+}
+
+type vdpa struct {
+	kernel  KernelInterface
+	vdpaLib govdpa.GoVdpaLib
+}
+
+func newVdpaInterface(k KernelInterface, vdpaLib govdpa.GoVdpaLib) VdpaInterface {
+	return &vdpa{kernel: k, vdpaLib: vdpaLib}
+}
+
+// CreateVDPADevice creates VDPA device for VF with required type,
+// pciAddr - PCI address of the VF
+// vdpaType - type of the VDPA device to create: virtio of vhost
+func (v *vdpa) CreateVDPADevice(pciAddr, vdpaType string) error {
+	log.Log.V(2).Info("CreateVDPADevice(): create VDPA device for VF",
+		"device", pciAddr, "vdpaType", vdpaType)
+	expectedDriver := vdpaTypeToDriver(vdpaType)
+	if expectedDriver == "" {
+		return fmt.Errorf("unknown VDPA device type: %s", vdpaType)
+	}
+	expectedVDPAName := generateVDPADevName(pciAddr)
+	_, err := v.vdpaLib.GetVdpaDevice(expectedVDPAName)
+	if err != nil {
+		if !errors.Is(err, syscall.ENODEV) {
+			log.Log.Error(err, "CreateVDPADevice(): fail to check if VDPA device exist",
+				"device", pciAddr, "vdpaDev", expectedVDPAName)
+			return err
+		}
+		if err := v.vdpaLib.AddVdpaDevice("pci/"+pciAddr, expectedVDPAName); err != nil {
+			log.Log.Error(err, "CreateVDPADevice(): fail to create VDPA device",
+				"device", pciAddr, "vdpaDev", expectedVDPAName)
+			return err
+		}
+	}
+	err = v.kernel.BindDriverByBusAndDevice(constants.BusVdpa, expectedVDPAName, expectedDriver)
+	if err != nil {
+		log.Log.Error(err, "CreateVDPADevice(): fail to bind VDPA device to the driver",
+			"device", pciAddr, "vdpaDev", expectedVDPAName, "driver", expectedDriver)
+		return err
+	}
+	return nil
+}
+
+// DeleteVDPADevice removes VDPA device for provided pci address
+// pciAddr - PCI address of the VF
+func (v *vdpa) DeleteVDPADevice(pciAddr string) error {
+	log.Log.V(2).Info("DeleteVDPADevice(): delete VDPA device for VF",
+		"device", pciAddr)
+	expectedVDPAName := generateVDPADevName(pciAddr)
+	if err := v.vdpaLib.DeleteVdpaDevice(expectedVDPAName); err != nil {
+		if errors.Is(err, syscall.ENODEV) {
+			log.Log.V(2).Info("DeleteVDPADevice(): VDPA device not found",
+				"device", pciAddr, "name", expectedVDPAName)
+			return nil
+		}
+		log.Log.Error(err, "DeleteVDPADevice(): fail to remove VDPA device",
+			"device", pciAddr, "name", expectedVDPAName)
+		return err
+	}
+	return nil
+}
+
+// DiscoverVDPAType returns type of existing VDPA device for VF,
+// returns empty string if VDPA device not found or unknown driver is in use
+// pciAddr - PCI address of the VF
+func (v *vdpa) DiscoverVDPAType(pciAddr string) string {
+	expectedVDPADevName := generateVDPADevName(pciAddr)
+	vdpaDev, err := v.vdpaLib.GetVdpaDevice(expectedVDPADevName)
+	if err != nil {
+		if errors.Is(err, syscall.ENODEV) {
+			log.Log.V(2).Info("discoverVDPAType(): VDPA device for VF not found", "device", pciAddr)
+			return ""
+		}
+		log.Log.Error(err, "getVfInfo(): unable to get VF VDPA devices", "device", pciAddr)
+		return ""
+	}
+	driverName := vdpaDev.Driver()
+	if driverName == "" {
+		log.Log.V(2).Info("discoverVDPAType(): VDPA device has no driver", "device", pciAddr)
+		return ""
+	}
+	vdpaType := vdpaDriverToType(driverName)
+	if vdpaType == "" {
+		log.Log.Error(nil, "getVfInfo(): WARNING: unknown VDPA device type for VF, ignore",
+			"device", pciAddr, "driver", driverName)
+	}
+	return vdpaType
+}
+
+// generates predictable name for VDPA device, example: vpda:0000:03:00.1
+func generateVDPADevName(pciAddr string) string {
+	return "vdpa:" + pciAddr
+}
+
+// vdpa type to driver name conversion
+func vdpaTypeToDriver(vdpaType string) string {
+	switch vdpaType {
+	case constants.VdpaTypeVhost:
+		return kvdpa.VhostVdpaDriver
+	case constants.VdpaTypeVirtio:
+		return kvdpa.VirtioVdpaDriver
+	default:
+		return ""
+	}
+}
+
+// vdpa driver name to type conversion
+func vdpaDriverToType(driver string) string {
+	switch driver {
+	case kvdpa.VhostVdpaDriver:
+		return constants.VdpaTypeVhost
+	case kvdpa.VirtioVdpaDriver:
+		return constants.VdpaTypeVirtio
+	default:
+		return ""
+	}
+}

pkg/host/vdpa_test.go

@@ -0,0 +1,121 @@
+package host
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/golang/mock/gomock"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
+	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
+	govdpaMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa/mock"
+	hostMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock"
+)
+
+const (
+	testVdpaVFPciAddr = "0000:d8:00.2"
+)
+
+var _ = Describe("VDPA", func() {
+	var (
+		v           VdpaInterface
+		libMock     *govdpaMock.MockGoVdpaLib
+		vdpaDevMock *govdpaMock.MockVdpaDevice
+		kernelMock  *hostMock.MockKernelInterface
+
+		testCtrl *gomock.Controller
+
+		expectedName   = "vdpa:" + testVdpaVFPciAddr
+		expectedDriver = "vhost_vdpa"
+		testErr        = fmt.Errorf("test-error")
+	)
+	BeforeEach(func() {
+		testCtrl = gomock.NewController(GinkgoT())
+		libMock = govdpaMock.NewMockGoVdpaLib(testCtrl)
+		vdpaDevMock = govdpaMock.NewMockVdpaDevice(testCtrl)
+		kernelMock = hostMock.NewMockKernelInterface(testCtrl)
+		v = newVdpaInterface(kernelMock, libMock)
+	})
+	AfterEach(func() {
+		testCtrl.Finish()
+	})
+	Context("CreateVDPADevice", func() {
+		callFunc := func() error {
+			return v.CreateVDPADevice(testVdpaVFPciAddr, constants.VdpaTypeVhost)
+		}
+		It("Created", func() {
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(nil, syscall.ENODEV)
+			libMock.EXPECT().AddVdpaDevice("pci/"+testVdpaVFPciAddr, expectedName).Return(nil)
+			kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, expectedName, expectedDriver).Return(nil)
+			Expect(callFunc()).NotTo(HaveOccurred())
+		})
+		It("Already exist", func() {
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(vdpaDevMock, nil)
+			kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, expectedName, expectedDriver).Return(nil)
+			Expect(callFunc()).NotTo(HaveOccurred())
+		})
+		It("Fail to Get device", func() {
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(nil, testErr)
+			Expect(callFunc()).To(MatchError(testErr))
+		})
+		It("Fail to Create device", func() {
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(nil, syscall.ENODEV)
+			libMock.EXPECT().AddVdpaDevice("pci/"+testVdpaVFPciAddr, expectedName).Return(testErr)
+			Expect(callFunc()).To(MatchError(testErr))
+		})
+		It("Fail to Bind device", func() {
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(vdpaDevMock, nil)
+			kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, expectedName, expectedDriver).Return(testErr)
+			Expect(callFunc()).To(MatchError(testErr))
+		})
+	})
+	Context("DeleteVDPADevice", func() {
+		callFunc := func() error {
+			return v.DeleteVDPADevice(testVdpaVFPciAddr)
+		}
+		It("Removed", func() {
+			libMock.EXPECT().DeleteVdpaDevice(expectedName).Return(nil)
+			Expect(callFunc()).NotTo(HaveOccurred())
+		})
+		It("Not found", func() {
+			libMock.EXPECT().DeleteVdpaDevice(expectedName).Return(syscall.ENODEV)
+			Expect(callFunc()).NotTo(HaveOccurred())
+		})
+		It("Fail to delete device", func() {
+			libMock.EXPECT().DeleteVdpaDevice(expectedName).Return(testErr)
+			Expect(callFunc()).To(MatchError(testErr))
+		})
+	})
+	Context("DiscoverVDPAType", func() {
+		callFunc := func() string {
+			return v.DiscoverVDPAType(testVdpaVFPciAddr)
+		}
+		It("No device", func() {
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(nil, syscall.ENODEV)
+			Expect(callFunc()).To(BeEmpty())
+		})
+		It("No driver", func() {
+			vdpaDevMock.EXPECT().Driver().Return("")
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(vdpaDevMock, nil)
+			Expect(callFunc()).To(BeEmpty())
+		})
+		It("Unknown driver", func() {
+			vdpaDevMock.EXPECT().Driver().Return("something")
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(vdpaDevMock, nil)
+			Expect(callFunc()).To(BeEmpty())
+		})
+		It("Vhost driver", func() {
+			vdpaDevMock.EXPECT().Driver().Return("vhost_vdpa")
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(vdpaDevMock, nil)
+			Expect(callFunc()).To(Equal("vhost"))
+		})
+		It("Virtio driver", func() {
+			vdpaDevMock.EXPECT().Driver().Return("virtio_vdpa")
+			libMock.EXPECT().GetVdpaDevice(expectedName).Return(vdpaDevMock, nil)
+			Expect(callFunc()).To(Equal("virtio"))
+		})
+	})
+})