Skip to content

Commit

Permalink
Add additional methods to host/kernel.go
Browse files Browse the repository at this point in the history 
New methods are:
* BindDriverByBusAndDevice - binds device to the provided driver
* UnbindDriverByBusAndDevice
unbind device identified by bus and device ID from the driver

Both methods allows to work with devices not
only on PCI bus.

+refactor driver-related methods
+add unit-tests for changed methods

Signed-off-by: Yury Kulazhenkov <ykulazhenkov@nvidia.com>
  • Loading branch information
@ykulazhenkov
ykulazhenkov committed Jan 12, 2024
1 parent 5752f32 commit 67415cc
Show file tree
Hide file tree
Showing 5 changed files with 412 additions and 67 deletions.
9 changes: 6 additions & 3 deletions pkg/consts/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,16 @@ const (
CheckpointFileName = "sno-initial-node-state.json"
Unknown = "Unknown"

SysBusPciDevices = "/sys/bus/pci/devices"
SysBusPciDrivers = "/sys/bus/pci/drivers"
SysBusPciDriversProbe = "/sys/bus/pci/drivers_probe"
SysBus = "/sys/bus"
SysBusPciDevices = SysBus + "/pci/devices"
SysBusPciDrivers = SysBus + "/pci/drivers"
SysBusPciDriversProbe = SysBus + "/pci/drivers_probe"
SysClassNet = "/sys/class/net"
ProcKernelCmdLine = "/proc/cmdline"
NetClass = 0x02
NumVfsFile = "sriov_numvfs"
BusPci = "pci"
BusVdpa = "vdpa"

UdevFolder = "/etc/udev"
UdevRulesFolder = UdevFolder + "/rules.d"
Expand Down
28 changes: 28 additions & 0 deletions pkg/helper/mock/mock_helper.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

222 changes: 158 additions & 64 deletions pkg/host/kernel.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package host

import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"

dputils "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
"sigs.k8s.io/controller-runtime/pkg/log"

sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
Expand Down Expand Up @@ -36,12 +36,21 @@ type KernelInterface interface {
BindDpdkDriver(string, string) error
// BindDefaultDriver binds the virtual function to is default driver
BindDefaultDriver(string) error
// BindDriverByBusAndDevice binds device to the provided driver
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
// driver - the name of the driver, e.g. vfio-pci or vhost_vdpa.
BindDriverByBusAndDevice(string, string, string) error
// HasDriver returns try if the virtual function is bind to a driver
HasDriver(string) (bool, string)
// RebindVfToDefaultDriver rebinds the virtual function to is default driver
RebindVfToDefaultDriver(string) error
// UnbindDriverIfNeeded unbinds the virtual function from a driver if needed
UnbindDriverIfNeeded(string, bool) error
// UnbindDriverByBusAndDevice unbind device identified by bus and device ID from the driver
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
UnbindDriverByBusAndDevice(bus, device string) error
// LoadKernelModule loads a kernel module to the host
LoadKernelModule(name string, args ...string) error
// IsKernelModuleLoaded returns try if the requested kernel module is loaded
Expand Down Expand Up @@ -165,64 +174,23 @@ func (k *kernel) IsKernelArgsSet(cmdLine string, karg string) bool {
// Unbind unbind driver for one device
func (k *kernel) Unbind(pciAddr string) error {
log.Log.V(2).Info("Unbind(): unbind device driver for device", "device", pciAddr)
yes, driver := k.HasDriver(pciAddr)
if !yes {
return nil
}

filePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDrivers, driver, "unbind")
err := os.WriteFile(filePath, []byte(pciAddr), os.ModeAppend)
if err != nil {
log.Log.Error(err, "Unbind(): fail to unbind driver for device", "device", pciAddr)
return err
}
return nil
return k.UnbindDriverByBusAndDevice(consts.BusPci, pciAddr)
}

// BindDpdkDriver bind dpdk driver for one device
// Bind the device given by "pciAddr" to the driver "driver"
func (k *kernel) BindDpdkDriver(pciAddr, driver string) error {
log.Log.V(2).Info("BindDpdkDriver(): bind device to driver",
"device", pciAddr, "driver", driver)

if yes, d := k.HasDriver(pciAddr); yes {
if driver == d {
log.Log.V(2).Info("BindDpdkDriver(): device already bound to driver",
"device", pciAddr, "driver", driver)
return nil
}

if err := k.Unbind(pciAddr); err != nil {
return err
}
}

driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "driver_override")
err := os.WriteFile(driverOverridePath, []byte(driver), os.ModeAppend)
if err != nil {
log.Log.Error(err, "BindDpdkDriver(): fail to write driver_override for device",
"device", pciAddr, "driver", driver)
return err
}
bindPath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDrivers, driver, "bind")
err = os.WriteFile(bindPath, []byte(pciAddr), os.ModeAppend)
if err != nil {
log.Log.Error(err, "BindDpdkDriver(): fail to bind driver for device",
"driver", driver, "device", pciAddr)
_, err := os.Readlink(filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "iommu_group"))
if err != nil {
if err := k.BindDriverByBusAndDevice(consts.BusPci, pciAddr, driver); err != nil {
_, innerErr := os.Readlink(filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "iommu_group"))
if innerErr != nil {
log.Log.Error(err, "Could not read IOMMU group for device", "device", pciAddr)
return fmt.Errorf(
"cannot bind driver %s to device %s, make sure IOMMU is enabled in BIOS. %w", driver, pciAddr, err)
"cannot bind driver %s to device %s, make sure IOMMU is enabled in BIOS. %w", driver, pciAddr, innerErr)
}
return err
}
err = os.WriteFile(driverOverridePath, []byte(""), os.ModeAppend)
if err != nil {
log.Log.Error(err, "BindDpdkDriver(): failed to clear driver_override for device", "device", pciAddr)
return err
}

return nil
}

Expand All @@ -231,32 +199,58 @@ func (k *kernel) BindDpdkDriver(pciAddr, driver string) error {
func (k *kernel) BindDefaultDriver(pciAddr string) error {
log.Log.V(2).Info("BindDefaultDriver(): bind device to default driver", "device", pciAddr)

if yes, d := k.HasDriver(pciAddr); yes {
if !sriovnetworkv1.StringInArray(d, vars.DpdkDrivers) {
curDriver, err := getDriverByBusAndDevice(consts.BusPci, pciAddr)
if err != nil {
return err
}
if curDriver != "" {
if !sriovnetworkv1.StringInArray(curDriver, vars.DpdkDrivers) {
log.Log.V(2).Info("BindDefaultDriver(): device already bound to default driver",
"device", pciAddr, "driver", d)
"device", pciAddr, "driver", curDriver)
return nil
}
if err := k.Unbind(pciAddr); err != nil {
if err := k.UnbindDriverByBusAndDevice(consts.BusPci, pciAddr); err != nil {
return err
}
}

driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDevices, pciAddr, "driver_override")
err := os.WriteFile(driverOverridePath, []byte("\x00"), os.ModeAppend)
if err != nil {
log.Log.Error(err, "BindDefaultDriver(): failed to write driver_override for device", "device", pciAddr)
if err := setDriverOverride(consts.BusPci, pciAddr, ""); err != nil {
return err
}
if err := probeDriver(consts.BusPci, pciAddr); err != nil {
return err
}
return nil
}

pciDriversProbe := filepath.Join(vars.FilesystemRoot, consts.SysBusPciDriversProbe)
err = os.WriteFile(pciDriversProbe, []byte(pciAddr), os.ModeAppend)
// BindDriverByBusAndDevice binds device to the provided driver
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
// driver - the name of the driver, e.g. vfio-pci or vhost_vdpa.
func (k *kernel) BindDriverByBusAndDevice(bus, device, driver string) error {
log.Log.V(2).Info("BindDriverByBusAndDevice(): bind device to driver",
"bus", bus, "device", device, "driver", driver)

curDriver, err := getDriverByBusAndDevice(bus, device)
if err != nil {
log.Log.Error(err, "BindDefaultDriver(): failed to bind driver for device", "device", pciAddr)
return err
}

return nil
if curDriver != "" {
if curDriver == driver {
log.Log.V(2).Info("BindDriverByBusAndDevice(): device already bound to driver",
"bus", bus, "device", device, "driver", driver)
return nil
}
if err := k.UnbindDriverByBusAndDevice(bus, device); err != nil {
return err
}
}
if err := setDriverOverride(bus, device, driver); err != nil {
return err
}
if err := bindDriver(bus, device, driver); err != nil {
return err
}
return setDriverOverride(bus, device, "")
}

// Workaround function to handle a case where the vf default driver is stuck and not able to create the vf kernel interface.
Expand Down Expand Up @@ -287,14 +281,33 @@ func (k *kernel) UnbindDriverIfNeeded(vfAddr string, isRdma bool) error {
return nil
}

// UnbindDriverByBusAndDevice unbind device identified by bus and device ID from the driver
// bus - the bus path in the sysfs, e.g. "pci" or "vdpa"
// device - the name of the device on the bus, e.g. 0000:85:1e.5 for PCI or vpda1 for VDPA
func (k *kernel) UnbindDriverByBusAndDevice(bus, device string) error {
log.Log.V(2).Info("UnbindDriverByBusAndDevice(): unbind device driver for device", "bus", bus, "device", device)
driver, err := getDriverByBusAndDevice(bus, device)
if err != nil {
return err
}
if driver == "" {
log.Log.V(2).Info("UnbindDriverByBusAndDevice(): device has no driver", "bus", bus, "device", device)
return nil
}
return unbindDriver(bus, device, driver)
}

func (k *kernel) HasDriver(pciAddr string) (bool, string) {
driver, err := dputils.GetDriverName(pciAddr)
driver, err := getDriverByBusAndDevice(consts.BusPci, pciAddr)
if err != nil {
log.Log.V(2).Info("HasDriver(): device driver is empty for device", "device", pciAddr)
return false, ""
}
log.Log.V(2).Info("HasDriver(): device driver for device", "device", pciAddr, "driver", driver)
return true, driver
if driver != "" {
log.Log.V(2).Info("HasDriver(): device driver for device", "device", pciAddr, "driver", driver)
return true, driver
}
return false, ""
}

func (k *kernel) TryEnableRdma() (bool, error) {
Expand Down Expand Up @@ -625,3 +638,84 @@ func (k *kernel) IsKernelLockdownMode() bool {
}
return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]")
}

// returns driver for device on the bus
func getDriverByBusAndDevice(bus, device string) (string, error) {
driverLink := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "devices", device, "driver")
driverInfo, err := os.Readlink(driverLink)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
log.Log.V(2).Info("getDriverByBusAndDevice(): driver path for device not exist", "bus", bus, "device", device, "driver", driverInfo)
return "", nil
}
log.Log.Error(err, "getDriverByBusAndDevice(): error getting driver info for device", "bus", bus, "device", device)
return "", err
}
log.Log.V(2).Info("getDriverByBusAndDevice(): driver for device", "bus", bus, "device", device, "driver", driverInfo)
return filepath.Base(driverInfo), nil
}

// binds device to the provide driver
func bindDriver(bus, device, driver string) error {
log.Log.V(2).Info("bindDriver(): bind to driver", "bus", bus, "device", device, "driver", driver)
bindPath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers", driver, "bind")
err := os.WriteFile(bindPath, []byte(device), os.ModeAppend)
if err != nil {
log.Log.Error(err, "bindDriver(): failed to bind driver", "bus", bus, "device", device, "driver", driver)
return err
}
return nil
}

// unbind device from the driver
func unbindDriver(bus, device, driver string) error {
log.Log.V(2).Info("unbindDriver(): unbind from driver", "bus", bus, "device", device, "driver", driver)
unbindPath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers", driver, "unbind")
err := os.WriteFile(unbindPath, []byte(device), os.ModeAppend)
if err != nil {
log.Log.Error(err, "unbindDriver(): failed to unbind driver", "bus", bus, "device", device, "driver", driver)
return err
}
return nil
}

// probes driver for device on the bus
func probeDriver(bus, device string) error {
log.Log.V(2).Info("probeDriver(): drivers probe", "bus", bus, "device", device)
probePath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "drivers_probe")
err := os.WriteFile(probePath, []byte(device), os.ModeAppend)
if err != nil {
log.Log.Error(err, "probeDriver(): failed to trigger driver probe", "bus", bus, "device", device)
return err
}
return nil
}

// set driver override for the bus/device,
// resets override if override arg is "",
// if device doesn't support overriding (has no driver_override path), does nothing
func setDriverOverride(bus, device, override string) error {
driverOverridePath := filepath.Join(vars.FilesystemRoot, consts.SysBus, bus, "devices", device, "driver_override")
if _, err := os.Stat(driverOverridePath); err != nil {
if os.IsNotExist(err) {
log.Log.V(2).Info("setDriverOverride(): device doesn't support driver override, skip", "bus", bus, "device", device)
return nil
}
return err
}
var overrideData []byte
if override != "" {
log.Log.V(2).Info("setDriverOverride(): configure driver override for device", "bus", bus, "device", device, "driver", override)
overrideData = []byte(override)
} else {
log.Log.V(2).Info("setDriverOverride(): reset driver override for device", "bus", bus, "device", device)
overrideData = []byte("\x00")
}
err := os.WriteFile(driverOverridePath, overrideData, os.ModeAppend)
if err != nil {
log.Log.Error(err, "setDriverOverride(): fail to write driver_override for device",
"bus", bus, "device", device, "driver", override)
return err
}
return nil
}
Loading

0 comments on commit 67415cc

Please sign in to comment.