diff --git a/api/v1/helper.go b/api/v1/helper.go index fc7b01ac4b..8a60b2b26b 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -13,7 +13,6 @@ import ( "github.com/golang/glog" netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,6 +20,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" ) const ( diff --git a/api/v1/sriovoperatorconfig_types.go b/api/v1/sriovoperatorconfig_types.go index fc4fe4b64d..5a0c483394 100644 --- a/api/v1/sriovoperatorconfig_types.go +++ b/api/v1/sriovoperatorconfig_types.go @@ -43,6 +43,8 @@ type SriovOperatorConfigSpec struct { // Default mode: daemon // +kubebuilder:validation:Enum=daemon;systemd ConfigurationMode ConfigurationModeType `json:"configurationMode,omitempty"` + // FeatureGates to enable experimental features + FeatureGates map[string]bool `json:"featureGates,omitempty"` } // SriovOperatorConfigStatus defines the observed state of SriovOperatorConfig diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index f01d358dda..d97a388617 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -725,6 +725,13 @@ func (in *SriovOperatorConfigSpec) DeepCopyInto(out *SriovOperatorConfigSpec) { *out = new(bool) **out = **in } + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SriovOperatorConfigSpec. diff --git a/bindata/manifests/daemon/daemonset.yaml b/bindata/manifests/daemon/daemonset.yaml index 6e1b3ba8d9..2892417c6f 100644 --- a/bindata/manifests/daemon/daemonset.yaml +++ b/bindata/manifests/daemon/daemonset.yaml @@ -94,9 +94,12 @@ spec: privileged: true args: - "start" - {{- if .UsedSystemdMode}} + {{- if .UsedSystemdMode }} - --use-systemd-service {{- end }} + {{ - if .ParallelNicConfig }} + - --parallel-nic-config + {{ - end }} env: - name: NODE_NAME valueFrom: diff --git a/cmd/sriov-network-config-daemon/service.go b/cmd/sriov-network-config-daemon/service.go index bbad726634..01a9edfacd 100644 --- a/cmd/sriov-network-config-daemon/service.go +++ b/cmd/sriov-network-config-daemon/service.go @@ -118,9 +118,9 @@ func runServiceCmd(cmd *cobra.Command, args []string) error { glog.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err) return fmt.Errorf("sriov-config-service: failed to discover sriov devices on the host: %v", err) } - + // TODO(e0ne): read ParallelNicConfig from SriovOperatorConfig CR // Create the generic plugin - configPlugin, err = generic.NewGenericPlugin(true, hostManager, storeManager) + configPlugin, err = generic.NewGenericPlugin(true, hostManager, storeManager, false) if err != nil { glog.Errorf("sriov-config-service: failed to create generic plugin %v", err) return fmt.Errorf("sriov-config-service failed to create generic plugin %v", err) diff --git a/cmd/sriov-network-config-daemon/start.go b/cmd/sriov-network-config-daemon/start.go index 52e4536fa7..8882483d3d 100644 --- a/cmd/sriov-network-config-daemon/start.go +++ b/cmd/sriov-network-config-daemon/start.go @@ -52,9 +52,10 @@ var ( } startOpts struct { - kubeconfig string - nodeName string - systemd bool + kubeconfig string + nodeName string + systemd bool + parallelNicConfig bool } ) @@ -63,6 +64,7 @@ func init() { startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)") startCmd.PersistentFlags().StringVar(&startOpts.nodeName, "node-name", "", "kubernetes node name daemon is managing") startCmd.PersistentFlags().BoolVar(&startOpts.systemd, "use-systemd-service", false, "use config daemon in systemd mode") + startCmd.PersistentFlags().BoolVar(&startOpts.parallelNicConfig, "parallel-nic-config", false, "NICs configuration in a parallel on the same node") } func runStartCmd(cmd *cobra.Command, args []string) { @@ -214,6 +216,7 @@ func runStartCmd(cmd *cobra.Command, args []string) { startOpts.systemd, eventRecorder, devMode, + startOpts.parallelNicConfig, ).Run(stopCh, exitCh) if err != nil { glog.Errorf("failed to run daemon: %v", err) diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index f1666e78e3..f2cc8b5737 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -65,6 +65,11 @@ spec: provision switchdev-configuration.service and enable OpenvSwitch hw-offload on nodes. type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates to enable experimental features + type: object logLevel: description: Flag to control the log verbose level of the operator. Set to '0' to show only the basic logs. And set to '2' to show all diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go index 7e9c548fbd..9d6850a803 100644 --- a/controllers/sriovoperatorconfig_controller.go +++ b/controllers/sriovoperatorconfig_controller.go @@ -196,6 +196,11 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context, } else { data.Data["UsedSystemdMode"] = false } + if parallelConfig, ok := dc.Spec.FeatureGates["parallelNicConfig"]; ok { + data.Data["ParallelNicConfig"] = parallelConfig + } else { + data.Data["ParallelNicConfig"] = false + } envCniBinPath := os.Getenv("SRIOV_CNI_BIN_PATH") if envCniBinPath == "" { diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index f1666e78e3..f2cc8b5737 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -65,6 +65,11 @@ spec: provision switchdev-configuration.service and enable OpenvSwitch hw-offload on nodes. type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates to enable experimental features + type: object logLevel: description: Flag to control the log verbose level of the operator. Set to '0' to show only the basic logs. And set to '2' to show all diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 5feb37e12a..346bbe6c3b 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -69,6 +69,8 @@ type Daemon struct { useSystemdService bool + parallelNicConfig bool + devMode bool client snclientset.Interface @@ -153,11 +155,13 @@ func New( useSystemdService bool, er *EventRecorder, devMode bool, + parallelNicConfig bool, ) *Daemon { return &Daemon{ name: nodeName, platform: platformType, useSystemdService: useSystemdService, + parallelNicConfig: parallelNicConfig, devMode: devMode, client: client, kubeClient: kubeClient, @@ -540,7 +544,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { // load plugins if it has not loaded if len(dn.enabledPlugins) == 0 { - dn.enabledPlugins, err = enablePlugins(dn.platform, dn.useSystemdService, latestState, dn.hostManager, dn.storeManager) + dn.enabledPlugins, err = enablePlugins(dn.platform, dn.useSystemdService, latestState, dn.hostManager, dn.storeManager, dn.parallelNicConfig) if err != nil { glog.Errorf("nodeStateSyncHandler(): failed to enable vendor plugins error: %v", err) return err diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go index 5300a1a653..f12f834175 100644 --- a/pkg/daemon/daemon_test.go +++ b/pkg/daemon/daemon_test.go @@ -118,6 +118,7 @@ var _ = Describe("Config Daemon", func() { false, er, false, + false, ) sut.enabledPlugins = map[string]plugin.VendorPlugin{generic.PluginName: &fake.FakePlugin{}} diff --git a/pkg/daemon/plugin.go b/pkg/daemon/plugin.go index ef48b28172..c038a456f1 100644 --- a/pkg/daemon/plugin.go +++ b/pkg/daemon/plugin.go @@ -29,7 +29,7 @@ var ( K8sPlugin = k8splugin.NewK8sPlugin ) -func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *sriovnetworkv1.SriovNetworkNodeState, hostManager host.HostManagerInterface, storeManager utils.StoreManagerInterface) (map[string]plugin.VendorPlugin, error) { +func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *sriovnetworkv1.SriovNetworkNodeState, hostManager host.HostManagerInterface, storeManager utils.StoreManagerInterface, parallelNicConfig bool) (map[string]plugin.VendorPlugin, error) { glog.Infof("enableVendorPlugins(): enabling plugins") enabledPlugins := map[string]plugin.VendorPlugin{} @@ -55,7 +55,7 @@ func enablePlugins(platform utils.PlatformType, useSystemdService bool, ns *srio } enabledPlugins[k8sPlugin.Name()] = k8sPlugin } - genericPlugin, err := GenericPlugin(false, hostManager, storeManager) + genericPlugin, err := GenericPlugin(false, hostManager, storeManager, parallelNicConfig) if err != nil { glog.Errorf("enableVendorPlugins(): failed to load the generic plugin error: %v", err) return nil, err diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 844c5e9716..2976d8efc3 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -55,6 +55,7 @@ type GenericPlugin struct { DriverStateMap DriverStateMapType DesiredKernelArgs map[string]bool RunningOnHost bool + ParallelNicConfig bool HostManager host.HostManagerInterface StoreManager utils.StoreManagerInterface } @@ -62,7 +63,7 @@ type GenericPlugin struct { const scriptsPath = "bindata/scripts/enable-kargs.sh" // Initialize our plugin and set up initial values -func NewGenericPlugin(runningOnHost bool, hostManager host.HostManagerInterface, storeManager utils.StoreManagerInterface) (plugin.VendorPlugin, error) { +func NewGenericPlugin(runningOnHost bool, hostManager host.HostManagerInterface, storeManager utils.StoreManagerInterface, parallelNicConfig bool) (plugin.VendorPlugin, error) { driverStateMap := make(map[uint]*DriverState) driverStateMap[Vfio] = &DriverState{ DriverName: vfioPciDriver, @@ -91,6 +92,7 @@ func NewGenericPlugin(runningOnHost bool, hostManager host.HostManagerInterface, DriverStateMap: driverStateMap, DesiredKernelArgs: make(map[string]bool), RunningOnHost: runningOnHost, + ParallelNicConfig: parallelNicConfig, HostManager: hostManager, StoreManager: storeManager, }, nil @@ -173,7 +175,7 @@ func (p *GenericPlugin) Apply() error { defer exit() } - if err := utils.SyncNodeState(p.DesireState, pfsToSkip); err != nil { + if err := utils.SyncNodeState(p.DesireState, pfsToSkip, p.ParallelNicConfig); err != nil { // Catch the "cannot allocate memory" error and try to use PCI realloc if errors.Is(err, syscall.ENOMEM) { p.addToDesiredKernelArgs(utils.KernelArgPciRealloc) diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index e1211b3924..2e3df38dfb 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -33,7 +33,7 @@ var _ = Describe("Generic plugin", func() { ctrl = gomock.NewController(t) mockHost = mock_host.NewMockHostManagerInterface(ctrl) mockStore = mock_utils.NewMockStoreManagerInterface(ctrl) - genericPlugin, err = NewGenericPlugin(false, mockHost, mockStore) + genericPlugin, err = NewGenericPlugin(false, mockHost, mockStore, false) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index c107abcb75..c1b48c8616 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -13,6 +13,7 @@ import ( "regexp" "strconv" "strings" + "sync" "syscall" "time" @@ -197,8 +198,71 @@ func DiscoverSriovDevices(withUnsupported bool, storeManager StoreManagerInterfa } // SyncNodeState Attempt to update the node state to match the desired state -func SyncNodeState(newState *sriovnetworkv1.SriovNetworkNodeState, pfsToConfig map[string]bool) error { - return ConfigSriovInterfaces(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig) +func SyncNodeState(newState *sriovnetworkv1.SriovNetworkNodeState, pfsToConfig map[string]bool, parallelNicConfig bool) error { + if !parallelNicConfig { + return ConfigSriovInterfaces(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig) + } + return ConfigSriovInterfacesInParallel(newState.Spec.Interfaces, newState.Status.Interfaces, pfsToConfig) +} + +func ConfigSriovInterfacesInParallel(interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt, pfsToConfig map[string]bool) error { + glog.V(2).Infof("ConfigSriovInterfacesInParallel(): start sriov configuration") + if IsKernelLockdownMode(true) && hasMellanoxInterfacesInSpec(ifaceStatuses, interfaces) { + glog.Warningf("cannot use mellanox devices when in kernel lockdown mode") + return fmt.Errorf("cannot use mellanox devices when in kernel lockdown mode") + } + // TODO(e0ne): store all errors in SriovNetworkNodeState + var result error + wg := sync.WaitGroup{} + for _, ifaceStatus := range ifaceStatuses { + configured := false + for _, iface := range interfaces { + if iface.PciAddress == ifaceStatus.PciAddress { + configured = true + + if skip := pfsToConfig[iface.PciAddress]; skip { + break + } + + if !NeedUpdate(&iface, &ifaceStatus) { + glog.V(2).Infof("syncNodeState(): no need update interface %s", iface.PciAddress) + break + } + + wg.Add(1) + go func(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetworkv1.InterfaceExt) { + if err := configSriovDevice(iface, ifaceStatus); err != nil { + glog.Errorf("ConfigSriovInterfacesInParallel(): fail to configure sriov interface %s: %v. resetting interface.", iface.PciAddress, err) + result = err + if resetErr := resetSriovDevice(*ifaceStatus); resetErr != nil { + glog.Errorf("SyncNodeState(): fail to reset on error SR-IOV interface: %s", resetErr) + result = resetErr + } + } + wg.Done() + }(&iface, &ifaceStatus) + + break + } + } + if !configured && ifaceStatus.NumVfs > 0 { + if skip := pfsToConfig[ifaceStatus.PciAddress]; skip { + continue + } + + if err := resetSriovDevice(ifaceStatus); err != nil { + glog.V(2).Infof("ConfigSriovInterfacesInParallel(): reset failed %v", ifaceStatus.PciAddress) + result = err + } + } + } + wg.Wait() + if result != nil { + glog.Errorf("ConfigSriovInterfacesInParallel(): fail to configure sriov interface: %v", result) + return result + } + glog.V(2).Infof("ConfigSriovInterfacesInParallel(): sriov configuration finished") + return nil } func ConfigSriovInterfaces(interfaces []sriovnetworkv1.Interface, ifaceStatuses []sriovnetworkv1.InterfaceExt, pfsToConfig map[string]bool) error { @@ -235,6 +299,7 @@ func ConfigSriovInterfaces(interfaces []sriovnetworkv1.Interface, ifaceStatuses break } + if err = configSriovDevice(&iface, &ifaceStatus); err != nil { glog.Errorf("SyncNodeState(): fail to configure sriov interface %s: %v. resetting interface.", iface.PciAddress, err) if iface.ExternallyManaged { @@ -540,6 +605,7 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor return err } } + glog.V(2).Infof("configSriovDevice(): config interface %s completed", ifaceStatus.PciAddress) return nil } @@ -584,6 +650,7 @@ func setNetdevMTU(pciAddr string, mtu int) error { glog.Warningf("setNetdevMTU(): fail to write mtu file after retrying: %v", err) return err } + glog.V(2).Infof("setNetdevMTU(): set MTU for device %s to %d completed", pciAddr, mtu) return nil }