RDMA subsystem is implemented via ib_core module config.

api/v1/sriovnetworknodestate_types.go

@@ -27,6 +27,7 @@ import (
 type SriovNetworkNodeStateSpec struct {
 	Interfaces Interfaces `json:"interfaces,omitempty"`
 	Bridges    Bridges    `json:"bridges,omitempty"`
+	System     System     `json:"system,omitempty"`
 }
 
 type Interfaces []Interface
@@ -114,10 +115,16 @@ type OVSUplinkConfigExt struct {
 	Interface OVSInterfaceConfig `json:"interface,omitempty"`
 }
 
+type System struct {
+	// +kubebuilder:validation:Enum=shared;exclusive
+	RdmaMode string `json:"rdmaMode,omitempty"`
+}
+
 // SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState
 type SriovNetworkNodeStateStatus struct {
 	Interfaces    InterfaceExts `json:"interfaces,omitempty"`
 	Bridges       Bridges       `json:"bridges,omitempty"`
+	System        System        `json:"system,omitempty"`
 	SyncStatus    string        `json:"syncStatus,omitempty"`
 	LastSyncError string        `json:"lastSyncError,omitempty"`
 }

api/v1/sriovnetworkpoolconfig_types.go

@@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
 	// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
 	// even if maxUnavailable is greater than one.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// +kubebuilder:validation:Enum=shared;exclusive
+	// RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
 }
 
 type OvsHardwareOffloadConfig struct {

bindata/manifests/ib-core-config/ib_core.conf

@@ -0,0 +1 @@
+options ib_core netns_mode={{.Mode}}

config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -174,6 +174,11 @@ spec:
                   - pciAddress
                   type: object
                 type: array
+              system:
+                properties:
+                  rdmaMode:
+                    type: string
+                type: object
             type: object
           status:
             description: SriovNetworkNodeStateStatus defines the observed state of
@@ -335,6 +340,11 @@ spec:
                 type: string
               syncStatus:
                 type: string
+              system:
+                properties:
+                  rdmaMode:
+                    type: string
+                type: object
             type: object
         type: object
     served: true

config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml

@@ -111,6 +111,12 @@ spec:
                       Name is the name of MachineConfigPool to be enabled with OVS hardware offload
                     type: string
                 type: object
+              rdmaMode:
+                description: RDMA subsystem. Allowed value "shared", "exclusive".
+                enum:
+                - shared
+                - exclusive
+                type: string
             type: object
           status:
             description: SriovNetworkPoolConfigStatus defines the observed state of

controllers/drain_controller.go

@@ -24,11 +24,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -48,13 +45,6 @@ import (
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
 )
 
-var (
-	oneNode     = intstr.FromInt32(1)
-	defaultNpcl = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{
-		MaxUnavailable: &oneNode,
-		NodeSelector:   &metav1.LabelSelector{}}}
-)
-
 type DrainReconcile struct {
 	client.Client
 	Scheme   *runtime.Scheme
@@ -346,94 +336,7 @@ func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (
 }
 
 func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) {
-	logger := log.FromContext(ctx)
-	logger.Info("findNodePoolConfig():")
-	// get all the sriov network pool configs
-	npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{}
-	err := dr.List(ctx, npcl)
-	if err != nil {
-		logger.Error(err, "failed to list sriovNetworkPoolConfig")
-		return nil, nil, err
-	}
-
-	selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{}
-	nodesInPools := map[string]interface{}{}
-
-	for _, npc := range npcl.Items {
-		// we skip hw offload objects
-		if npc.Spec.OvsHardwareOffloadConfig.Name != "" {
-			continue
-		}
-
-		if npc.Spec.NodeSelector == nil {
-			npc.Spec.NodeSelector = &metav1.LabelSelector{}
-		}
-
-		selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector)
-		if err != nil {
-			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		if selector.Matches(labels.Set(node.Labels)) {
-			selectedNpcl = append(selectedNpcl, npc.DeepCopy())
-		}
-
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
-		if err != nil {
-			logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector",
-				"machineConfigPoolName", npc,
-				"nodeSelector", npc.Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		for _, nodeName := range nodeList.Items {
-			nodesInPools[nodeName.Name] = nil
-		}
-	}
-
-	if len(selectedNpcl) > 1 {
-		// don't allow the node to be part of multiple pools
-		err = fmt.Errorf("node is part of more then one pool")
-		logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl)
-		return nil, nil, err
-	} else if len(selectedNpcl) == 1 {
-		// found one pool for our node
-		logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0])
-		selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector)
-		if err != nil {
-			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		// list all the nodes that are also part of this pool and return them
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
-		if err != nil {
-			logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector)
-			return nil, nil, err
-		}
-
-		return selectedNpcl[0], nodeList.Items, nil
-	} else {
-		// in this case we get all the nodes and remove the ones that already part of any pool
-		logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultNpcl)
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList)
-		if err != nil {
-			logger.Error(err, "failed to list all the nodes")
-			return nil, nil, err
-		}
-
-		defaultNodeLists := []corev1.Node{}
-		for _, nodeObj := range nodeList.Items {
-			if _, exist := nodesInPools[nodeObj.Name]; !exist {
-				defaultNodeLists = append(defaultNodeLists, nodeObj)
-			}
-		}
-		return defaultNpcl, defaultNodeLists, nil
-	}
+	return utils.FindNodePoolConfig(ctx, node, dr.Client)
 }
 
 // SetupWithManager sets up the controller with the Manager.

controllers/sriovnetworknodepolicy_controller.go

@@ -46,6 +46,7 @@ import (
 	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
 	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
 )
 
@@ -272,6 +273,13 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con
 		ns.Name = node.Name
 		ns.Namespace = vars.Namespace
 		j, _ := json.Marshal(ns)
+		netPoolConfig, _, err := utils.FindNodePoolConfig(ctx, &node, r.Client)
+		if err != nil {
+			log.Log.Error(err, "nodeStateSyncHandler(): failed to get SriovNetworkPoolConfig for the current node")
+		}
+		if netPoolConfig != nil {
+			ns.Spec.System.RdmaMode = netPoolConfig.Spec.RdmaMode
+		}
 		logger.V(2).Info("SriovNetworkNodeState CR", "content", j)
 		if err := r.syncSriovNetworkNodeState(ctx, dc, npl, ns, &node); err != nil {
 			logger.Error(err, "Fail to sync", "SriovNetworkNodeState", ns.Name)

deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -174,6 +174,11 @@ spec:
                   - pciAddress
                   type: object
                 type: array
+              system:
+                properties:
+                  rdmaMode:
+                    type: string
+                type: object
             type: object
           status:
             description: SriovNetworkNodeStateStatus defines the observed state of
@@ -335,6 +340,11 @@ spec:
                 type: string
               syncStatus:
                 type: string
+              system:
+                properties:
+                  rdmaMode:
+                    type: string
+                type: object
             type: object
         type: object
     served: true

deployment/sriov-network-operator-chart/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml

@@ -111,6 +111,12 @@ spec:
                       Name is the name of MachineConfigPool to be enabled with OVS hardware offload
                     type: string
                 type: object
+              rdmaMode:
+                description: RDMA subsystem. Allowed value "shared", "exclusive".
+                enum:
+                - shared
+                - exclusive
+                type: string
             type: object
           status:
             description: SriovNetworkPoolConfigStatus defines the observed state of

pkg/consts/constants.go

@@ -54,6 +54,9 @@ const (
 	VdpaTypeVirtio      = "virtio"
 	VdpaTypeVhost       = "vhost"
 
+	RdmaSubsystemModeShared    = "shared"
+	RdmaSubsystemModeExclusive = "exclusive"
+
 	ClusterTypeOpenshift  = "openshift"
 	ClusterTypeKubernetes = "kubernetes"
 

pkg/daemon/daemon.go

@@ -429,6 +429,16 @@ func (dn *Daemon) nodeStateSyncHandler() error {
 		reqReboot = reqReboot || r
 	}
 
+	if dn.currentNodeState.Status.System.RdmaMode != dn.desiredNodeState.Spec.System.RdmaMode {
+		err = dn.HostHelpers.SetRDMASubsystem(dn.desiredNodeState.Spec.System.RdmaMode)
+		if err != nil {
+			log.Log.Error(err, "nodeStateSyncHandler(): failed to set RDMA subsystem")
+			return err
+		}
+		reqReboot = true
+		reqDrain = true
+	}
+
 	// When running using systemd check if the applied configuration is the latest one
 	// or there is a new config we need to apply
 	// When using systemd configuration we write the file

pkg/daemon/writer.go

@@ -117,6 +117,7 @@ func (w *NodeStateStatusWriter) Run(stop <-chan struct{}, refresh <-chan Message
 func (w *NodeStateStatusWriter) pollNicStatus() error {
 	log.Log.V(2).Info("pollNicStatus()")
 	var iface []sriovnetworkv1.InterfaceExt
+	var rdmaMode string
 	var err error
 
 	if vars.PlatformType == consts.VirtualOpenStack {
@@ -127,7 +128,13 @@ func (w *NodeStateStatusWriter) pollNicStatus() error {
 	if err != nil {
 		return err
 	}
+	rdmaMode, err = w.hostHelper.DiscoverRDMASubsystem()
+	if err != nil {
+		return err
+	}
+
 	w.status.Interfaces = iface
+	w.status.System.RdmaMode = rdmaMode
 
 	return nil
 }