Add support for bridge configuration to systemd service and daemon

Signed-off-by: Yury Kulazhenkov <ykulazhenkov@nvidia.com>

cmd/sriov-network-config-daemon/service.go

@@ -93,6 +93,7 @@ func runServiceCmd(cmd *cobra.Command, args []string) error {
 	setupLog.V(2).Info("sriov-config-service", "config", sriovConf)
 	vars.DevMode = sriovConf.UnsupportedNics
 	vars.ManageSoftwareBridges = sriovConf.ManageSoftwareBridges
+	vars.OVSDBSocketPath = sriovConf.OVSDBSocketPath
 
 	if err := initSupportedNics(); err != nil {
 		return updateSriovResultErr(setupLog, phaseArg, fmt.Errorf("failed to initialize list of supported NIC ids: %v", err))
@@ -181,7 +182,7 @@ func callPlugin(setupLog logr.Logger, phase string, conf *systemd.SriovConfig, h
 		return nil
 	}
 
-	nodeState, err := getNetworkNodeState(setupLog, conf, hostHelpers)
+	nodeState, err := getNetworkNodeState(setupLog, conf, phase, hostHelpers)
 	if err != nil {
 		return err
 	}
@@ -207,7 +208,9 @@ func getPlugin(setupLog logr.Logger, phase string,
 	case consts.Baremetal:
 		switch phase {
 		case PhasePre:
-			configPlugin, err = newGenericPluginFunc(hostHelpers, generic.WithSkipVFConfiguration())
+			configPlugin, err = newGenericPluginFunc(hostHelpers,
+				generic.WithSkipVFConfiguration(),
+				generic.WithSkipBridgeConfiguration())
 		case PhasePost:
 			configPlugin, err = newGenericPluginFunc(hostHelpers)
 		}
@@ -229,10 +232,11 @@ func getPlugin(setupLog logr.Logger, phase string,
 	return configPlugin, nil
 }
 
-func getNetworkNodeState(setupLog logr.Logger, conf *systemd.SriovConfig,
+func getNetworkNodeState(setupLog logr.Logger, conf *systemd.SriovConfig, phase string,
 	hostHelpers helper.HostHelpersInterface) (*sriovv1.SriovNetworkNodeState, error) {
 	var (
 		ifaceStatuses []sriovv1.InterfaceExt
+		bridges       sriovv1.Bridges
 		err           error
 	)
 	switch conf.PlatformType {
@@ -241,6 +245,13 @@ func getNetworkNodeState(setupLog logr.Logger, conf *systemd.SriovConfig,
 		if err != nil {
 			return nil, fmt.Errorf("failed to discover sriov devices on the host:  %v", err)
 		}
+		if phase != PhasePre && vars.ManageSoftwareBridges {
+			// openvswitch is not available during the pre phase
+			bridges, err = hostHelpers.DiscoverBridges()
+			if err != nil {
+				return nil, fmt.Errorf("failed to discover managed bridges on the host:  %v", err)
+			}
+		}
 	case consts.VirtualOpenStack:
 		platformHelper, err := newPlatformHelperFunc()
 		if err != nil {
@@ -257,7 +268,7 @@ func getNetworkNodeState(setupLog logr.Logger, conf *systemd.SriovConfig,
 	}
 	return &sriovv1.SriovNetworkNodeState{
 		Spec:   conf.Spec,
-		Status: sriovv1.SriovNetworkNodeStateStatus{Interfaces: ifaceStatuses},
+		Status: sriovv1.SriovNetworkNodeStateStatus{Interfaces: ifaceStatuses, Bridges: bridges},
 	}, nil
 }
 

cmd/sriov-network-config-daemon/service_test.go

@@ -38,7 +38,6 @@ func restoreOrigFuncs() {
 
 func getTestSriovInterfaceConfig(platform int) []byte {
 	return []byte(fmt.Sprintf(`spec:
-    dpconfigversion: ""
     interfaces:
         - pciaddress: 0000:d8:00.0
           numvfs: 4
@@ -57,6 +56,7 @@ func getTestSriovInterfaceConfig(platform int) []byte {
           externallymanaged: false
 unsupportedNics: false
 platformType: %d
+manageSoftwareBridges: true
 `, platform))
 }
 
@@ -239,6 +239,7 @@ var _ = Describe("Service", func() {
 		hostHelpers.EXPECT().DiscoverSriovDevices(hostHelpers).Return([]sriovnetworkv1.InterfaceExt{{
 			Name: "enp216s0f0np0",
 		}}, nil)
+		hostHelpers.EXPECT().DiscoverBridges().Return(sriovnetworkv1.Bridges{}, nil)
 		genericPlugin.EXPECT().OnNodeStateChange(newNodeStateContainsDeviceMatcher("enp216s0f0np0")).Return(true, false, nil)
 		genericPlugin.EXPECT().Apply().Return(nil)
 		Expect(runServiceCmd(&cobra.Command{}, []string{})).NotTo(HaveOccurred())

cmd/sriov-network-config-daemon/start.go

@@ -87,6 +87,7 @@ var (
 		disabledPlugins       stringList
 		parallelNicConfig     bool
 		manageSoftwareBridges bool
+		ovsSocketPath         string
 	}
 )
 
@@ -98,6 +99,7 @@ func init() {
 	startCmd.PersistentFlags().VarP(&startOpts.disabledPlugins, "disable-plugins", "", "comma-separated list of plugins to disable")
 	startCmd.PersistentFlags().BoolVar(&startOpts.parallelNicConfig, "parallel-nic-config", false, "perform NIC configuration in parallel")
 	startCmd.PersistentFlags().BoolVar(&startOpts.manageSoftwareBridges, "manage-software-bridges", false, "enable management of software bridges")
+	startCmd.PersistentFlags().StringVar(&startOpts.ovsSocketPath, "ovs-socket-path", vars.OVSDBSocketPath, "path for OVSDB socket")
 }
 
 func runStartCmd(cmd *cobra.Command, args []string) error {
@@ -113,6 +115,7 @@ func runStartCmd(cmd *cobra.Command, args []string) error {
 
 	vars.ParallelNicConfig = startOpts.parallelNicConfig
 	vars.ManageSoftwareBridges = startOpts.manageSoftwareBridges
+	vars.OVSDBSocketPath = startOpts.ovsSocketPath
 
 	if startOpts.nodeName == "" {
 		name, ok := os.LookupEnv("NODE_NAME")

pkg/daemon/writer.go

@@ -117,17 +117,29 @@ func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message
 func (w *NodeStateStatusWriter) pollNicStatus() error {
 	log.Log.V(2).Info("pollNicStatus()")
 	var iface []sriovnetworkv1.InterfaceExt
+	var bridges sriovnetworkv1.Bridges
 	var err error
 
 	if vars.PlatformType == consts.VirtualOpenStack {
 		iface, err = w.platformHelper.DiscoverSriovDevicesVirtual()
+		if err != nil {
+			return err
+		}
 	} else {
 		iface, err = w.hostHelper.DiscoverSriovDevices(w.hostHelper)
+		if err != nil {
+			return err
+		}
+		if vars.ManageSoftwareBridges {
+			bridges, err = w.hostHelper.DiscoverBridges()
+			if err != nil {
+				return err
+			}
+		}
 	}
-	if err != nil {
-		return err
-	}
+
 	w.status.Interfaces = iface
+	w.status.Bridges = bridges
 
 	return nil
 }
@@ -169,6 +181,7 @@ func (w *NodeStateStatusWriter) updateNodeStateStatusRetry(f func(*sriovnetworkv
 func (w *NodeStateStatusWriter) setNodeStateStatus(msg Message) (*sriovnetworkv1.SriovNetworkNodeState, error) {
 	nodeState, err := w.updateNodeStateStatusRetry(func(nodeState *sriovnetworkv1.SriovNetworkNodeState) {
 		nodeState.Status.Interfaces = w.status.Interfaces
+		nodeState.Status.Bridges = w.status.Bridges
 		if msg.lastSyncError != "" || msg.syncStatus == consts.SyncStatusSucceeded {
 			// clear lastSyncError when sync Succeeded
 			nodeState.Status.LastSyncError = msg.lastSyncError

pkg/host/internal/sriov/sriov.go

@@ -43,6 +43,7 @@ type sriov struct {
 	dputilsLib       dputilsPkg.DPUtilsLib
 	sriovnetLib      sriovnetPkg.SriovnetLib
 	ghwLib           ghwPkg.GHWLib
+	bridgeHelper     types.BridgeInterface
 }
 
 func New(utilsHelper utils.CmdInterface,
@@ -54,7 +55,8 @@ func New(utilsHelper utils.CmdInterface,
 	netlinkLib netlinkPkg.NetlinkLib,
 	dputilsLib dputilsPkg.DPUtilsLib,
 	sriovnetLib sriovnetPkg.SriovnetLib,
-	ghwLib ghwPkg.GHWLib) types.SriovInterface {
+	ghwLib ghwPkg.GHWLib,
+	bridgeHelper types.BridgeInterface) types.SriovInterface {
 	return &sriov{utilsHelper: utilsHelper,
 		kernelHelper:     kernelHelper,
 		networkHelper:    networkHelper,
@@ -65,6 +67,7 @@ func New(utilsHelper utils.CmdInterface,
 		dputilsLib:       dputilsLib,
 		sriovnetLib:      sriovnetLib,
 		ghwLib:           ghwLib,
+		bridgeHelper:     bridgeHelper,
 	}
 }
 
@@ -1007,6 +1010,11 @@ func (s *sriov) setEswitchModeAndNumVFs(pciAddr string, desiredEswitchMode strin
 	// always switch NIC to the legacy mode before creating VFs. This is required because some drivers
 	// may not support VF creation in the switchdev mode
 	if s.GetNicSriovMode(pciAddr) != sriovnetworkv1.ESwithModeLegacy {
+		// detach PF from the managed bridge before switching the mode,
+		// changing of eSwitch mode may fail if NIC is part of the bridge (has offloaded TC rules)
+		if err := s.detachPFFromBridge(pciAddr); err != nil {
+			return err
+		}
 		if err := s.setEswitchMode(pciAddr, sriovnetworkv1.ESwithModeLegacy); err != nil {
 			return err
 		}
@@ -1021,6 +1029,19 @@ func (s *sriov) setEswitchModeAndNumVFs(pciAddr string, desiredEswitchMode strin
 	return nil
 }
 
+// detach PF from the managed bridge
+func (s *sriov) detachPFFromBridge(pciAddr string) error {
+	log.Log.V(2).Info("detachPFFromBridge(): detach PF", "device", pciAddr)
+	if !vars.ManageSoftwareBridges {
+		return nil
+	}
+	if err := s.bridgeHelper.DetachInterfaceFromManagedBridge(pciAddr); err != nil {
+		log.Log.Error(err, "detachPFFromBridge(): failed to detach interface from the managed bridge", "device", pciAddr)
+		return err
+	}
+	return nil
+}
+
 // retrieve all VFs for the PF and unbind them from a driver
 func (s *sriov) unbindAllVFsOnPF(addr string) error {
 	log.Log.V(2).Info("unbindAllVFsOnPF(): unbind all VFs on PF", "device", addr)

pkg/host/internal/sriov/sriov_test.go

@@ -49,7 +49,7 @@ var _ = Describe("SRIOV", func() {
 		hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl)
 		storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl)
 
-		s = New(nil, hostMock, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock)
+		s = New(nil, hostMock, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock, hostMock)
 	})
 
 	AfterEach(func() {

pkg/host/manager.go

@@ -1,6 +1,7 @@
 package host
 
 import (
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/bridge"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils"
@@ -28,6 +29,7 @@ type HostManagerInterface interface {
 	types.SriovInterface
 	types.VdpaInterface
 	types.InfinibandInterface
+	types.BridgeInterface
 }
 
 type hostManager struct {
@@ -39,6 +41,7 @@ type hostManager struct {
 	types.SriovInterface
 	types.VdpaInterface
 	types.InfinibandInterface
+	types.BridgeInterface
 }
 
 func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, error) {
@@ -56,8 +59,8 @@ func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, er
 	if err != nil {
 		return nil, err
 	}
-	sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils, sriovnetLib, ghwLib)
-
+	br := bridge.New()
+	sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils, sriovnetLib, ghwLib, br)
 	return &hostManager{
 		utilsInterface,
 		k,
@@ -67,5 +70,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, er
 		sr,
 		v,
 		ib,
+		br,
 	}, nil
 }