Split systemd service to two parts

The first part is executed before the
NeworkManager or systemd-networkd
starts on the host.
The second part is triggered after the
network configuration is complete.

This change is required to support hw offload
use-case with systemd service.

Signed-off-by: Yury Kulazhenkov <ykulazhenkov@nvidia.com>

bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml

@@ -0,0 +1,16 @@
+contents: |
+  [Unit]
+  Description=Configures SRIOV NIC - post network configuration
+  After=systemd-networkd-wait-online.service NetworkManager-wait-online.service
+  Before=network-online.target
+
+  [Service]
+  Type=oneshot
+  # TODO replace with service call
+  ExecStart=/bin/true
+  StandardOutput=journal+console
+
+  [Install]
+  WantedBy=network-online.target
+enabled: true
+name: sriov-config-post-network.service

bindata/manifests/sriov-config-service/kubernetes/sriov-config-service.yaml

@@ -1,14 +1,15 @@
 contents: |
   [Unit]
-  Description=Configures SRIOV NIC
-  After=network-pre.target
-  Before=NetworkManager.service kubelet.service
-  
+  Description=Configures SRIOV NIC - pre network configuration
+  DefaultDependencies=no
+  After=network-pre.target systemd-udev-settle.service systemd-sysusers.service systemd-sysctl.service
+  Before=network.target NetworkManager.service systemd-networkd.service ovs-vswitchd.service ovsdb-server.service
+
   [Service]
   Type=oneshot
   ExecStart=/var/lib/sriov/sriov-network-config-daemon -v 2 service
   StandardOutput=journal+console
-  
+
   [Install]
   WantedBy=multi-user.target
 enabled: true

bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml

@@ -12,19 +12,37 @@ spec:
       units:
         - contents: |
             [Unit]
-            Description=Configures SRIOV NIC
             # Removal of this file signals firstboot completion
             ConditionPathExists=!/etc/ignition-machine-config-encapsulated.json
-            # This service is used to configure the SR-IOV VFs on NICs
-            After=network-pre.target
-            Before=NetworkManager.service kubelet.service
-            
+            Description=Configures SRIOV NIC - pre network configuration
+            DefaultDependencies=no
+            After=network-pre.target systemd-udev-settle.service systemd-sysusers.service systemd-sysctl.service
+            Before=network.target NetworkManager.service systemd-networkd.service ovs-vswitchd.service ovsdb-server.service
+
             [Service]
             Type=oneshot
             ExecStart=/var/lib/sriov/sriov-network-config-daemon service -v {{ .LogLevel }}
             StandardOutput=journal+console
-            
+
             [Install]
             WantedBy=multi-user.target
           enabled: true
           name: "sriov-config.service"
+        - contents: |
+            [Unit]
+            # Removal of this file signals firstboot completion
+            ConditionPathExists=!/etc/ignition-machine-config-encapsulated.json
+            Description=Configures SRIOV NIC - post network configuration
+            After=systemd-networkd-wait-online.service NetworkManager-wait-online.service
+            Before=network-online.target
+
+            [Service]
+            Type=oneshot
+            # TODO replace with service call
+            ExecStart=/bin/true
+            StandardOutput=journal+console
+
+            [Install]
+            WantedBy=network-online.target
+          enabled: true
+          name: "sriov-config-post-network.service"

pkg/daemon/daemon.go

@@ -431,13 +431,19 @@ func (dn *Daemon) nodeStateSyncHandler() error {
 				log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host")
 				return err
 			}
+			postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovPostNetworkServicePath)
+			if err != nil {
+				log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config-post-network service exist on host")
+				return err
+			}
 
 			// if the service doesn't exist we should continue to let the k8s plugin to create the service files
 			// this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply
 			// the system service and reboot the node the config-daemon doesn't need to do anything.
-			if !serviceEnabled {
+			if !(serviceEnabled && postNetworkServiceEnabled) {
 				sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed,
-					LastSyncError: "sriov-config systemd service is not available on node"}
+					LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+
+						"sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)}
 			} else {
 				sriovResult, err = systemd.ReadSriovResult()
 				if err != nil {

pkg/plugins/k8s/k8s_plugin.go

@@ -29,6 +29,7 @@ type K8sPlugin struct {
 	openVSwitchService         *hostTypes.Service
 	networkManagerService      *hostTypes.Service
 	sriovService               *hostTypes.Service
+	sriovPostNetworkService    *hostTypes.Service
 	updateTarget               *k8sUpdateTarget
 	hostHelper                 helper.HostHelpersInterface
 }
@@ -40,15 +41,29 @@ type k8sUpdateTarget struct {
 	switchdevAfterNMRunScript  bool
 	switchdevUdevScript        bool
 	sriovScript                bool
+	sriovPostNetworkScript     bool
 	systemServices             []*hostTypes.Service
 }
 
 func (u *k8sUpdateTarget) needUpdate() bool {
-	return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript || u.sriovScript || len(u.systemServices) > 0
+	return u.switchdevBeforeNMService ||
+		u.switchdevAfterNMService ||
+		u.switchdevBeforeNMRunScript ||
+		u.switchdevAfterNMRunScript ||
+		u.switchdevUdevScript ||
+		u.sriovScript ||
+		u.sriovPostNetworkScript ||
+		len(u.systemServices) > 0
 }
 
 func (u *k8sUpdateTarget) needReboot() bool {
-	return u.switchdevBeforeNMService || u.switchdevAfterNMService || u.switchdevBeforeNMRunScript || u.switchdevAfterNMRunScript || u.switchdevUdevScript || u.sriovScript
+	return u.switchdevBeforeNMService ||
+		u.switchdevAfterNMService ||
+		u.switchdevBeforeNMRunScript ||
+		u.switchdevAfterNMRunScript ||
+		u.switchdevUdevScript ||
+		u.sriovScript ||
+		u.sriovPostNetworkScript
 }
 
 func (u *k8sUpdateTarget) reset() {
@@ -58,6 +73,7 @@ func (u *k8sUpdateTarget) reset() {
 	u.switchdevAfterNMRunScript = false
 	u.switchdevUdevScript = false
 	u.sriovScript = false
+	u.sriovPostNetworkScript = false
 	u.systemServices = []*hostTypes.Service{}
 }
 
@@ -85,6 +101,7 @@ const (
 	switchdevUnits                    = switchdevManifestPath + "switchdev-units/"
 	sriovUnits                        = bindataManifestPath + "sriov-config-service/kubernetes/"
 	sriovUnitFile                     = sriovUnits + "sriov-config-service.yaml"
+	sriovPostNetworkUnitFile          = sriovUnits + "sriov-config-post-network-service.yaml"
 	switchdevBeforeNMUnitFile         = switchdevUnits + "switchdev-configuration-before-nm.yaml"
 	switchdevAfterNMUnitFile          = switchdevUnits + "switchdev-configuration-after-nm.yaml"
 	networkManagerUnitFile            = switchdevUnits + "NetworkManager.service.yaml"
@@ -140,7 +157,7 @@ func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState)
 
 	if vars.UsingSystemdMode {
 		// Check sriov service
-		err = p.sriovServiceStateUpdate()
+		err = p.sriovServicesStateUpdate()
 		if err != nil {
 			log.Log.Error(err, "k8s-plugin OnNodeStateChange(): failed")
 			return
@@ -168,7 +185,7 @@ func (p *K8sPlugin) Apply() error {
 	}
 
 	if vars.UsingSystemdMode {
-		if err := p.updateSriovService(); err != nil {
+		if err := p.updateSriovServices(); err != nil {
 			return err
 		}
 	}
@@ -256,6 +273,16 @@ func (p *K8sPlugin) readSriovServiceManifest() error {
 	return nil
 }
 
+func (p *K8sPlugin) readSriovPostNetworkServiceManifest() error {
+	sriovService, err := p.hostHelper.ReadServiceManifestFile(sriovPostNetworkUnitFile)
+	if err != nil {
+		return err
+	}
+
+	p.sriovPostNetworkService = sriovService
+	return nil
+}
+
 func (p *K8sPlugin) readManifestFiles() error {
 	if err := p.readSwitchdevManifest(); err != nil {
 		return err
@@ -273,6 +300,10 @@ func (p *K8sPlugin) readManifestFiles() error {
 		return err
 	}
 
+	if err := p.readSriovPostNetworkServiceManifest(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -311,23 +342,31 @@ func (p *K8sPlugin) switchdevServiceStateUpdate() error {
 	return nil
 }
 
-func (p *K8sPlugin) sriovServiceStateUpdate() error {
-	log.Log.Info("sriovServiceStateUpdate()")
-	isServiceEnabled, err := p.hostHelper.IsServiceEnabled(p.sriovService.Path)
-	if err != nil {
-		return err
-	}
-
-	// create and enable the service if it doesn't exist or is not enabled
-	if !isServiceEnabled {
-		p.updateTarget.sriovScript = true
-	} else {
-		p.updateTarget.sriovScript = p.isSystemServiceNeedUpdate(p.sriovService)
+func (p *K8sPlugin) sriovServicesStateUpdate() error {
+	log.Log.Info("sriovServicesStateUpdate()")
+
+	for _, s := range []struct {
+		srv    *hostTypes.Service
+		update *bool
+	}{
+		{srv: p.sriovService, update: &p.updateTarget.sriovScript},
+		{srv: p.sriovPostNetworkService, update: &p.updateTarget.sriovPostNetworkScript},
+	} {
+		isServiceEnabled, err := p.hostHelper.IsServiceEnabled(s.srv.Path)
+		if err != nil {
+			return err
+		}
+		// create and enable the service if it doesn't exist or is not enabled
+		if !isServiceEnabled {
+			*s.update = true
+		} else {
+			*s.update = p.isSystemServiceNeedUpdate(s.srv)
+		}
+		if *s.update {
+			p.updateTarget.systemServices = append(p.updateTarget.systemServices, s.srv)
+		}
 	}
 
-	if p.updateTarget.sriovScript {
-		p.updateTarget.systemServices = append(p.updateTarget.systemServices, p.sriovService)
-	}
 	return nil
 }
 
@@ -420,11 +459,19 @@ func (p *K8sPlugin) switchDevServicesStateUpdate() error {
 	return nil
 }
 
-func (p *K8sPlugin) updateSriovService() error {
-	if p.updateTarget.sriovScript {
-		err := p.hostHelper.EnableService(p.sriovService)
-		if err != nil {
-			return err
+func (p *K8sPlugin) updateSriovServices() error {
+	for _, s := range []struct {
+		srv    *hostTypes.Service
+		update bool
+	}{
+		{srv: p.sriovService, update: p.updateTarget.sriovScript},
+		{srv: p.sriovPostNetworkService, update: p.updateTarget.sriovPostNetworkScript},
+	} {
+		if s.update {
+			err := p.hostHelper.EnableService(s.srv)
+			if err != nil {
+				return err
+			}
 		}
 	}