Merge pull request k8snetworkplumbingwg#436 from SchSeba/externally_m…

…anage

Support Externally Created virtual functions

README.md

@@ -211,6 +211,23 @@ VF groups** (when #-notation is used in pfName field) are merged, otherwise only
 the highest priority policy is applied. In case of same-priority policies and
 overlapping VF groups, only the last processed policy is applied.
 
+#### Externally Manage virtual functions
+
+When `ExternallyManage` is request on a policy the operator will only skip the virtual function creation.
+The operator will only bind the virtual functions to the requested driver and expose them via the device plugin.
+Another difference when this field is requested in the policy is that when this policy is removed the operator
+will not remove the virtual functions from the policy.
+
+*Note:* This means the user must create the virtual functions before they apply the policy or the webhook will reject
+the policy creation.
+
+It's possible to use something like nmstate kubernetes-nmstate or just a simple systemd file to create
+the virtual functions on boot.
+
+This feature was created to support deployments where the user want to use some of the virtual funtions for the host
+communication like storage network or out of band managment and the virtual functions must exist on boot and not only
+after the operator and config-daemon are running.
+
 ## Components and design
 
 This operator is split into 2 components:

api/v1/helper.go

@@ -282,12 +282,13 @@ func (p *SriovNetworkNodePolicy) Apply(state *SriovNetworkNodeState, equalPriori
 		if s.Selected(&iface) {
 			log.Info("Update interface", "name:", iface.Name)
 			result := Interface{
-				PciAddress:  iface.PciAddress,
-				Mtu:         p.Spec.Mtu,
-				Name:        iface.Name,
-				LinkType:    p.Spec.LinkType,
-				EswitchMode: p.Spec.EswitchMode,
-				NumVfs:      p.Spec.NumVfs,
+				PciAddress:        iface.PciAddress,
+				Mtu:               p.Spec.Mtu,
+				Name:              iface.Name,
+				LinkType:          p.Spec.LinkType,
+				EswitchMode:       p.Spec.EswitchMode,
+				NumVfs:            p.Spec.NumVfs,
+				ExternallyManaged: p.Spec.ExternallyManaged,
 			}
 			if p.Spec.NumVfs > 0 {
 				group, err := p.generateVfGroup(&iface)

api/v1/sriovnetworknodepolicy_types.go

@@ -59,6 +59,8 @@ type SriovNetworkNodePolicySpec struct {
 	VdpaType string `json:"vdpaType,omitempty"`
 	// Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false.
 	ExcludeTopology bool `json:"excludeTopology,omitempty"`
+	// don't create the virtual function only allocated them to the device plugin. Defaults to false.
+	ExternallyManaged bool `json:"externallyManaged,omitempty"`
 }
 
 type SriovNetworkNicSelector struct {

api/v1/sriovnetworknodestate_types.go

@@ -29,14 +29,17 @@ type SriovNetworkNodeStateSpec struct {
 	Interfaces      Interfaces `json:"interfaces,omitempty"`
 }
 
+type Interfaces []Interface
+
 type Interface struct {
-	PciAddress  string    `json:"pciAddress"`
-	NumVfs      int       `json:"numVfs,omitempty"`
-	Mtu         int       `json:"mtu,omitempty"`
-	Name        string    `json:"name,omitempty"`
-	LinkType    string    `json:"linkType,omitempty"`
-	EswitchMode string    `json:"eSwitchMode,omitempty"`
-	VfGroups    []VfGroup `json:"vfGroups,omitempty"`
+	PciAddress        string    `json:"pciAddress"`
+	NumVfs            int       `json:"numVfs,omitempty"`
+	Mtu               int       `json:"mtu,omitempty"`
+	Name              string    `json:"name,omitempty"`
+	LinkType          string    `json:"linkType,omitempty"`
+	EswitchMode       string    `json:"eSwitchMode,omitempty"`
+	VfGroups          []VfGroup `json:"vfGroups,omitempty"`
+	ExternallyManaged bool      `json:"externallyManaged,omitempty"`
 }
 
 type VfGroup struct {
@@ -49,23 +52,22 @@ type VfGroup struct {
 	VdpaType     string `json:"vdpaType,omitempty"`
 }
 
-type Interfaces []Interface
-
 type InterfaceExt struct {
-	Name        string            `json:"name,omitempty"`
-	Mac         string            `json:"mac,omitempty"`
-	Driver      string            `json:"driver,omitempty"`
-	PciAddress  string            `json:"pciAddress"`
-	Vendor      string            `json:"vendor,omitempty"`
-	DeviceID    string            `json:"deviceID,omitempty"`
-	NetFilter   string            `json:"netFilter,omitempty"`
-	Mtu         int               `json:"mtu,omitempty"`
-	NumVfs      int               `json:"numVfs,omitempty"`
-	LinkSpeed   string            `json:"linkSpeed,omitempty"`
-	LinkType    string            `json:"linkType,omitempty"`
-	EswitchMode string            `json:"eSwitchMode,omitempty"`
-	TotalVfs    int               `json:"totalvfs,omitempty"`
-	VFs         []VirtualFunction `json:"Vfs,omitempty"`
+	Name              string            `json:"name,omitempty"`
+	Mac               string            `json:"mac,omitempty"`
+	Driver            string            `json:"driver,omitempty"`
+	PciAddress        string            `json:"pciAddress"`
+	Vendor            string            `json:"vendor,omitempty"`
+	DeviceID          string            `json:"deviceID,omitempty"`
+	NetFilter         string            `json:"netFilter,omitempty"`
+	Mtu               int               `json:"mtu,omitempty"`
+	NumVfs            int               `json:"numVfs,omitempty"`
+	LinkSpeed         string            `json:"linkSpeed,omitempty"`
+	LinkType          string            `json:"linkType,omitempty"`
+	EswitchMode       string            `json:"eSwitchMode,omitempty"`
+	ExternallyManaged bool              `json:"externallyManaged,omitempty"`
+	TotalVfs          int               `json:"totalvfs,omitempty"`
+	VFs               []VirtualFunction `json:"Vfs,omitempty"`
 }
 type InterfaceExts []InterfaceExt
 

bindata/scripts/udev-find-sriov-pf.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+cat <<'EOF' > /host/etc/udev/disable-nm-sriov.sh
+#!/bin/bash
+if [ ! -d "/sys/class/net/$1/device/physfn" ]; then
+    exit 0
+fi
+
+pf_path=$(readlink /sys/class/net/$1/device/physfn -n)
+pf_pci_address=${pf_path##*../}
+
+if [ "$2" == "$pf_pci_address" ]; then
+    echo "NM_UNMANAGED=1"
+fi
+EOF
+
+chmod +x /host/etc/udev/disable-nm-sriov.sh

cmd/sriov-network-config-daemon/service.go

@@ -95,6 +95,11 @@ func runServiceCmd(cmd *cobra.Command, args []string) error {
 
 	glog.V(2).Infof("sriov-config-service read config: %v", nodeStateSpec)
 
+	storeManager, err := utils.NewStoreManager(true)
+	if err != nil {
+		glog.Errorf("failed to create store manager: %v", err)
+		return err
+	}
 	// Load kernel modules
 	hostManager := host.NewHostManager(true)
 	_, err = hostManager.TryEnableRdma()
@@ -108,14 +113,14 @@ func runServiceCmd(cmd *cobra.Command, args []string) error {
 	var ifaceStatuses []sriovv1.InterfaceExt
 	if nodeStateSpec.PlatformType == utils.Baremetal {
 		// Bare metal support
-		ifaceStatuses, err = utils.DiscoverSriovDevices(nodeStateSpec.UnsupportedNics)
+		ifaceStatuses, err = utils.DiscoverSriovDevices(nodeStateSpec.UnsupportedNics, storeManager)
 		if err != nil {
 			glog.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err)
 			return fmt.Errorf("sriov-config-service: failed to discover sriov devices on the host:  %v", err)
 		}
 
 		// Create the generic plugin
-		configPlugin, err = generic.NewGenericPlugin(true)
+		configPlugin, err = generic.NewGenericPlugin(true, hostManager, storeManager)
 		if err != nil {
 			glog.Errorf("sriov-config-service: failed to create generic plugin %v", err)
 			return fmt.Errorf("sriov-config-service failed to create generic plugin %v", err)

config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml

@@ -53,6 +53,10 @@ spec:
                 description: Exclude device's NUMA node when advertising this resource
                   by SRIOV network device plugin. Default to false.
                 type: boolean
+              externallyManaged:
+                description: don't create the virtual function only allocated them
+                  to the device plugin. Defaults to false.
+                type: boolean
               isRdma:
                 description: RDMA mode. Defaults to false.
                 type: boolean

config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -50,6 +50,8 @@ spec:
                   properties:
                     eSwitchMode:
                       type: string
+                    externallyManaged:
+                      type: boolean
                     linkType:
                       type: string
                     mtu:
@@ -125,6 +127,8 @@ spec:
                       type: string
                     eSwitchMode:
                       type: string
+                    externallyManaged:
+                      type: boolean
                     linkSpeed:
                       type: string
                     linkType:

deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml

@@ -53,6 +53,10 @@ spec:
                 description: Exclude device's NUMA node when advertising this resource
                   by SRIOV network device plugin. Default to false.
                 type: boolean
+              externallyManaged:
+                description: don't create the virtual function only allocated them
+                  to the device plugin. Defaults to false.
+                type: boolean
               isRdma:
                 description: RDMA mode. Defaults to false.
                 type: boolean

deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -50,6 +50,8 @@ spec:
                   properties:
                     eSwitchMode:
                       type: string
+                    externallyManaged:
+                      type: boolean
                     linkType:
                       type: string
                     mtu:
@@ -125,6 +127,8 @@ spec:
                       type: string
                     eSwitchMode:
                       type: string
+                    externallyManaged:
+                      type: boolean
                     linkSpeed:
                       type: string
                     linkType: