diff --git a/pkg/operator/controller.go b/pkg/operator/controller.go index 507b324bc..4d42d494a 100644 --- a/pkg/operator/controller.go +++ b/pkg/operator/controller.go @@ -645,7 +645,7 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error { } klog.V(2).Infof("syncProfile(): Profile %s not found, creating one [%s]", profileMf.Name, computed.TunedProfileName) - profileMf.Annotations = util.ToggleDeferredUpdateAnnotation(profileMf.Annotations, computed.Deferred) + profileMf.Annotations = toggleDeferred(profileMf.Annotations, computed.Deferred) profileMf.Spec.Config.TunedProfile = computed.TunedProfileName profileMf.Spec.Config.Debug = computed.Operand.Debug profileMf.Spec.Config.TuneDConfig = computed.Operand.TuneDConfig @@ -706,14 +706,14 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error { } } - anns := util.ToggleDeferredUpdateAnnotation(profile.Annotations, computed.Deferred) + anns := toggleDeferred(profile.Annotations, computed.Deferred) // Minimize updates if profile.Spec.Config.TunedProfile == computed.TunedProfileName && profile.Spec.Config.Debug == computed.Operand.Debug && reflect.DeepEqual(profile.Spec.Config.TuneDConfig, computed.Operand.TuneDConfig) && reflect.DeepEqual(profile.Spec.Profile, computed.AllProfiles) && - util.HasDeferredUpdateAnnotation(profile.Annotations) == util.HasDeferredUpdateAnnotation(anns) && + util.GetDeferredUpdateAnnotation(profile.Annotations) == util.GetDeferredUpdateAnnotation(anns) && profile.Spec.Config.ProviderName == providerName { klog.V(2).Infof("syncProfile(): no need to update Profile %s", nodeName) return nil @@ -732,11 +732,18 @@ func (c *Controller) syncProfile(tuned *tunedv1.Tuned, nodeName string) error { if err != nil { return fmt.Errorf("failed to update Profile %s: %v", profile.Name, err) } - klog.Infof("updated profile %s [%s] (deferred=%v)", profile.Name, computed.TunedProfileName, util.HasDeferredUpdateAnnotation(profile.Annotations)) + klog.Infof("updated profile %s [%s] (deferred=%v)", profile.Name, computed.TunedProfileName, util.GetDeferredUpdateAnnotation(profile.Annotations)) return nil } +func toggleDeferred(anns map[string]string, mode util.DeferMode) map[string]string { + if util.IsDeferredUpdate(mode) { + return util.SetDeferredUpdateAnnotation(anns, mode) + } + return util.DeleteDeferredUpdateAnnotation(anns) +} + func (c *Controller) getProviderName(nodeName string) (string, error) { node, err := c.listers.Nodes.Get(nodeName) if err != nil { diff --git a/pkg/operator/profilecalculator.go b/pkg/operator/profilecalculator.go index 19667c7ee..87cc9c872 100644 --- a/pkg/operator/profilecalculator.go +++ b/pkg/operator/profilecalculator.go @@ -153,7 +153,7 @@ func (pc *ProfileCalculator) nodeChangeHandler(nodeName string) (bool, error) { type ComputedProfile struct { TunedProfileName string AllProfiles []tunedv1.TunedProfile - Deferred bool + Deferred util.DeferMode MCLabels map[string]string NodePoolName string Operand tunedv1.OperandConfig @@ -161,7 +161,7 @@ type ComputedProfile struct { type RecommendedProfile struct { TunedProfileName string - Deferred bool + Deferred util.DeferMode Labels map[string]string Config tunedv1.OperandConfig } @@ -302,7 +302,7 @@ func (pc *ProfileCalculator) calculateProfile(nodeName string) (ComputedProfile, type HypershiftRecommendedProfile struct { TunedProfileName string - Deferred bool + Deferred util.DeferMode NodePoolName string Config tunedv1.OperandConfig } @@ -732,7 +732,7 @@ func tunedProfiles(tunedSlice []*tunedv1.Tuned) []tunedv1.TunedProfile { type TunedRecommendInfo struct { tunedv1.TunedRecommend - Deferred bool + Deferred util.DeferMode } // TunedRecommend returns a priority-sorted TunedRecommend slice out of @@ -752,7 +752,7 @@ func TunedRecommend(tunedSlice []*tunedv1.Tuned) []TunedRecommendInfo { for _, recommend := range tuned.Spec.Recommend { recommendAll = append(recommendAll, TunedRecommendInfo{ TunedRecommend: recommend, - Deferred: util.HasDeferredUpdateAnnotation(tuned.Annotations), + Deferred: util.GetDeferredUpdateAnnotation(tuned.Annotations), }) } } diff --git a/pkg/tuned/controller.go b/pkg/tuned/controller.go index 420a9ecf1..aa700cdb6 100644 --- a/pkg/tuned/controller.go +++ b/pkg/tuned/controller.go @@ -149,7 +149,7 @@ type Change struct { recommendedProfile string // Is the current Change triggered by an object with the deferred annotation? - deferred bool + deferredMode util.DeferMode // Text to convey in status message, if present. message string } @@ -180,8 +180,8 @@ func (ch Change) String() string { if ch.recommendedProfile != "" { items = append(items, fmt.Sprintf("recommendedProfile:%q", ch.recommendedProfile)) } - if ch.deferred { - items = append(items, "deferred:true") + if ch.deferredMode != "" { + items = append(items, fmt.Sprintf("deferredMode:%q", string(ch.deferredMode))) } if ch.message != "" { items = append(items, fmt.Sprintf("message:%q", ch.message)) @@ -354,7 +354,7 @@ func (c *Controller) sync(key wqKeyKube) error { if profile.Spec.Config.TuneDConfig.ReapplySysctl != nil { change.reapplySysctl = *profile.Spec.Config.TuneDConfig.ReapplySysctl } - change.deferred = util.HasDeferredUpdateAnnotation(profile.Annotations) + change.deferredMode = util.GetDeferredUpdateAnnotation(profile.Annotations) // Notify the event processor that the Profile k8s object containing information about which TuneD profile to apply changed. c.wqTuneD.Add(wqKeyTuned{kind: wqKindDaemon, change: change}) @@ -987,6 +987,7 @@ func (c *Controller) changeSyncerProfileStatus(change Change) (synced bool) { func (c *Controller) changeSyncerTuneD(change Change) (synced bool, err error) { var restart bool var reload bool + var inplaceUpdate bool // updating a profile already recommended var cfgUpdated bool var changeRecommend bool @@ -1010,15 +1011,16 @@ func (c *Controller) changeSyncerTuneD(change Change) (synced bool, err error) { } reload = reload || changeProvider - if (c.daemon.recommendedProfile != change.recommendedProfile) || change.nodeRestart { + inplaceUpdate = (c.daemon.recommendedProfile == change.recommendedProfile) + if !inplaceUpdate || change.nodeRestart { if err = TunedRecommendFileWrite(change.recommendedProfile); err != nil { return false, err } - klog.V(1).Infof("recommended TuneD profile changed from %q to %q [deferred=%v nodeRestart=%v]", c.daemon.recommendedProfile, change.recommendedProfile, change.deferred, change.nodeRestart) + klog.V(1).Infof("recommended TuneD profile changed from %q to %q [deferred=%v nodeRestart=%v]", c.daemon.recommendedProfile, change.recommendedProfile, change.deferredMode, change.nodeRestart) // Cache the value written to tunedRecommendFile. c.daemon.recommendedProfile = change.recommendedProfile reload = true - } else if !change.deferred && (c.daemon.status&scDeferred != 0) { + } else if util.IsImmediateUpdate(change.deferredMode) && (c.daemon.status&scDeferred != 0) { klog.V(1).Infof("detected deferred update changed to immediate after object update") reload = true } else { @@ -1077,7 +1079,7 @@ func (c *Controller) changeSyncerTuneD(change Change) (synced bool, err error) { } // failures pertaining to deferred updates are not critical - _ = c.handleDaemonReloadRestartRequest(change, reload, restart) + _ = c.handleDaemonReloadRestartRequest(change, reload, restart, inplaceUpdate) cfgUpdated, err = c.changeSyncerRestartOrReloadTuneD() klog.V(2).Infof("changeSyncerTuneD() configuration updated: %v", cfgUpdated) @@ -1088,19 +1090,32 @@ func (c *Controller) changeSyncerTuneD(change Change) (synced bool, err error) { return err == nil, err } -func (c *Controller) handleDaemonReloadRestartRequest(change Change, reload, restart bool) error { +func treatAsImmediate(change Change, inplaceUpdate bool) (bool, string) { + if change.nodeRestart { + return true, "node restart" + } + if util.IsImmediateUpdate(change.deferredMode) { + return true, "immediate update" + } + if !inplaceUpdate && change.deferredMode == util.DeferUpdate { + return true, "recommend change with deferredMode=update" + } + return false, "" +} + +func (c *Controller) handleDaemonReloadRestartRequest(change Change, reload, restart, inplaceUpdate bool) error { if !reload && !restart { // nothing to do return nil } - if !change.deferred || change.nodeRestart { + if ok, reason := treatAsImmediate(change, inplaceUpdate); ok { if reload { - klog.V(2).Infof("immediate update, setting reload flag") + klog.V(2).Infof("%s: setting reload flag", reason) c.daemon.restart |= ctrlReload } if restart { - klog.V(2).Infof("immediate update, setting restart flag") + klog.V(2).Infof("%s: setting restart flag", reason) c.daemon.restart |= ctrlRestart } return nil @@ -1321,7 +1336,7 @@ func (c *Controller) updateTunedProfileStatus(ctx context.Context, change Change } var message string - wantsDeferred := util.HasDeferredUpdateAnnotation(profile.Annotations) + wantsDeferred := util.IsDeferredUpdate(util.GetDeferredUpdateAnnotation(profile.Annotations)) isApplied := (c.daemon.profileFingerprintUnpacked == c.daemon.profileFingerprintEffective) daemonStatus := c.daemon.status diff --git a/pkg/tuned/controller_test.go b/pkg/tuned/controller_test.go index d36e314ca..679653ae5 100644 --- a/pkg/tuned/controller_test.go +++ b/pkg/tuned/controller_test.go @@ -7,6 +7,7 @@ import ( "testing" tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" + "github.com/openshift/cluster-node-tuning-operator/pkg/util" ) func TestRecommendFileRoundTrip(t *testing.T) { @@ -295,7 +296,7 @@ func fullChange() Change { provider: "test-provider", reapplySysctl: true, recommendedProfile: "test-profile", - deferred: true, + deferredMode: util.DeferAlways, message: "test-message", } } diff --git a/pkg/util/annotations.go b/pkg/util/annotations.go index 1577fe459..90635aae3 100644 --- a/pkg/util/annotations.go +++ b/pkg/util/annotations.go @@ -4,28 +4,53 @@ import ( tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" ) -func HasDeferredUpdateAnnotation(anns map[string]string) bool { - if anns == nil { - return false - } - _, ok := anns[tunedv1.TunedDeferredUpdate] - return ok +type DeferMode string + +const ( + DeferNever DeferMode = "never" // aka defer mode disabled aka immediate update + DeferAlways DeferMode = "always" + DeferUpdate DeferMode = "update" +) + +func (dm DeferMode) String() string { + return string(dm) +} + +func IsImmediateUpdate(value DeferMode) bool { + return value == DeferNever } -func SetDeferredUpdateAnnotation(anns map[string]string, tuned *tunedv1.Tuned) map[string]string { +func IsDeferredUpdate(value DeferMode) bool { + return value == DeferAlways || value == DeferUpdate +} + +func GetDeferredUpdateAnnotation(anns map[string]string) DeferMode { if anns == nil { - anns = make(map[string]string) + return DeferNever + } + val, ok := anns[tunedv1.TunedDeferredUpdate] + if !ok { + return DeferNever } - return ToggleDeferredUpdateAnnotation(anns, HasDeferredUpdateAnnotation(tuned.Annotations)) + value := DeferMode(val) + if !IsDeferredUpdate(value) { + return DeferNever + } + return value } -func ToggleDeferredUpdateAnnotation(anns map[string]string, toggle bool) map[string]string { +func SetDeferredUpdateAnnotation(anns map[string]string, value DeferMode) map[string]string { ret := cloneMapStringString(anns) - if toggle { - ret[tunedv1.TunedDeferredUpdate] = "" - } else { - delete(ret, tunedv1.TunedDeferredUpdate) + if value == DeferNever { + return ret } + ret[tunedv1.TunedDeferredUpdate] = string(value) + return ret +} + +func DeleteDeferredUpdateAnnotation(anns map[string]string) map[string]string { + ret := cloneMapStringString(anns) + delete(ret, tunedv1.TunedDeferredUpdate) return ret } diff --git a/pkg/util/annotations_test.go b/pkg/util/annotations_test.go index a20d17bbe..e81615a66 100644 --- a/pkg/util/annotations_test.go +++ b/pkg/util/annotations_test.go @@ -3,25 +3,22 @@ package util import ( "reflect" "testing" - - tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestHasDeferredUpdateAnnotation(t *testing.T) { testCases := []struct { name string anns map[string]string - expected bool + expected DeferMode }{ { name: "nil", - expected: false, + expected: DeferNever, }, { name: "empty", anns: map[string]string{}, - expected: false, + expected: DeferNever, }, { name: "no-ann", @@ -29,26 +26,40 @@ func TestHasDeferredUpdateAnnotation(t *testing.T) { "foo": "bar", "baz": "2", }, - expected: false, + expected: DeferNever, }, { name: "wrong-case", anns: map[string]string{ "tuned.openshift.io/Deferred": "", }, - expected: false, + expected: DeferNever, }, { name: "found", anns: map[string]string{ "tuned.openshift.io/deferred": "", }, - expected: true, + expected: DeferNever, + }, + { + name: "found-explicit", + anns: map[string]string{ + "tuned.openshift.io/deferred": "always", + }, + expected: DeferAlways, + }, + { + name: "found-wrong-case", + anns: map[string]string{ + "tuned.openshift.io/deferred": "Always", + }, + expected: DeferNever, }, } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { - got := HasDeferredUpdateAnnotation(tt.anns) + got := GetDeferredUpdateAnnotation(tt.anns) if got != tt.expected { t.Errorf("got=%v expected=%v", got, tt.expected) } @@ -60,74 +71,48 @@ func TestSetDeferredUpdateAnnotation(t *testing.T) { testCases := []struct { name string anns map[string]string - tuned *tunedv1.Tuned + mode DeferMode expected map[string]string }{ { name: "nil", - tuned: &tunedv1.Tuned{}, + mode: DeferNever, expected: map[string]string{}, }, { - name: "nil-add", - tuned: &tunedv1.Tuned{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "tuned.openshift.io/deferred": "", - }, - }, - }, + name: "add", + mode: DeferAlways, expected: map[string]string{ - "tuned.openshift.io/deferred": "", + "tuned.openshift.io/deferred": "always", }, }, { - name: "existing-add", + name: "add-existing", anns: map[string]string{ "foobar": "42", }, - tuned: &tunedv1.Tuned{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "tuned.openshift.io/deferred": "", - }, - }, - }, + mode: DeferAlways, expected: map[string]string{ "foobar": "42", - "tuned.openshift.io/deferred": "", + "tuned.openshift.io/deferred": "always", }, }, { - name: "nil-remove", - tuned: &tunedv1.Tuned{}, - expected: map[string]string{}, - }, - { - name: "existing-remove", + name: "add-overwrite", anns: map[string]string{ "foobar": "42", "tuned.openshift.io/deferred": "", }, - tuned: &tunedv1.Tuned{}, + mode: DeferAlways, expected: map[string]string{ - "foobar": "42", - }, - }, - { - name: "missing-remove", - anns: map[string]string{ - "foobar": "42", - }, - tuned: &tunedv1.Tuned{}, - expected: map[string]string{ - "foobar": "42", + "foobar": "42", + "tuned.openshift.io/deferred": "always", }, }, } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { - got := SetDeferredUpdateAnnotation(tt.anns, tt.tuned) + got := SetDeferredUpdateAnnotation(tt.anns, tt.mode) if !reflect.DeepEqual(got, tt.expected) { t.Errorf("got=%v expected=%v", got, tt.expected) } @@ -135,39 +120,16 @@ func TestSetDeferredUpdateAnnotation(t *testing.T) { } } -func TestToggleDeferredUpdateAnnotation(t *testing.T) { +func TestDeleteDeferredUpdateAnnotation(t *testing.T) { testCases := []struct { name string anns map[string]string - toggle bool expected map[string]string }{ { name: "nil", expected: map[string]string{}, }, - { - name: "nil-add", - toggle: true, - expected: map[string]string{ - "tuned.openshift.io/deferred": "", - }, - }, - { - name: "existing-add", - anns: map[string]string{ - "foobar": "42", - }, - toggle: true, - expected: map[string]string{ - "foobar": "42", - "tuned.openshift.io/deferred": "", - }, - }, - { - name: "nil-remove", - expected: map[string]string{}, - }, { name: "existing-remove", anns: map[string]string{ @@ -191,7 +153,7 @@ func TestToggleDeferredUpdateAnnotation(t *testing.T) { for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { anns := cloneMapStringString(tt.anns) - got := ToggleDeferredUpdateAnnotation(tt.anns, tt.toggle) + got := DeleteDeferredUpdateAnnotation(tt.anns) // must not mutate the argument if tt.anns != nil && !reflect.DeepEqual(anns, tt.anns) { t.Errorf("toggle must return a new copy") diff --git a/test/e2e/deferred/basic.go b/test/e2e/deferred/basic.go index 1218ab1ce..1cd5c7e74 100644 --- a/test/e2e/deferred/basic.go +++ b/test/e2e/deferred/basic.go @@ -85,8 +85,8 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La verifData := util.MustExtractVerificationOutputAndCommand(cs, targetNode, tuned) gomega.Expect(verifData.OutputCurrent).ToNot(gomega.Equal(verifData.OutputExpected), "current output %q already matches expected %q", verifData.OutputCurrent, verifData.OutputExpected) - tunedMutated := setDeferred(tuned.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tuned.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -138,10 +138,10 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La } for _, condition := range curProf.Status.Conditions { if condition.Type == tunedv1.TunedProfileApplied && condition.Status != corev1.ConditionFalse && condition.Reason != "Deferred" { - return fmt.Errorf("Profile deferred=%v %s applied", ntoutil.HasDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) + return fmt.Errorf("Profile deferred=%v %s applied", ntoutil.GetDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) } if condition.Type == tunedv1.TunedDegraded && condition.Status != corev1.ConditionTrue && condition.Reason != "TunedDeferredUpdate" { - return fmt.Errorf("Profile deferred=%v %s not degraded", ntoutil.HasDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) + return fmt.Errorf("Profile deferred=%v %s not degraded", ntoutil.GetDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) } } ginkgo.By(fmt.Sprintf("checking real node conditions for profile %q are not changed from pristine state", curProf.Name)) @@ -174,8 +174,8 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La verifData := util.MustExtractVerificationOutputAndCommand(cs, targetNode, tuned) gomega.Expect(verifData.OutputCurrent).ToNot(gomega.Equal(verifData.OutputExpected), "current output %q already matches expected %q", verifData.OutputCurrent, verifData.OutputExpected) - tunedMutated := setDeferred(tuned.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tuned.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -266,8 +266,8 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La tunedDeferred, err := util.LoadTuned(tunedPathSHMMNI) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - tunedMutated := setDeferred(tunedDeferred.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tunedDeferred.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -316,8 +316,8 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) tunedDeferred2 := tunedObjVMLatency - tunedMutated2 := setDeferred(tunedDeferred2.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated2.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated2.Annotations))) + tunedMutated2 := setDeferred(tunedDeferred2.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated2.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated2.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated2, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -362,8 +362,8 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La ginkgo.It("should be overridden by a immediate update by edit", func(ctx context.Context) { tunedImmediate := tunedObjVMLatency - tunedMutated := setDeferred(tunedImmediate.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tunedImmediate.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err := cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -416,7 +416,7 @@ var _ = ginkgo.Describe("[deferred][profile-status] Profile deferred", ginkgo.La curTuned = curTuned.DeepCopy() ginkgo.By(fmt.Sprintf("removing the deferred annotation from Tuned %q", tunedImmediate.Name)) - curTuned.Annotations = ntoutil.ToggleDeferredUpdateAnnotation(curTuned.Annotations, false) + curTuned.Annotations = ntoutil.DeleteDeferredUpdateAnnotation(curTuned.Annotations) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Update(ctx, curTuned, metav1.UpdateOptions{}) return err diff --git a/test/e2e/deferred/non_regression.go b/test/e2e/deferred/non_regression.go index 7952ad92f..9702124a2 100644 --- a/test/e2e/deferred/non_regression.go +++ b/test/e2e/deferred/non_regression.go @@ -56,7 +56,7 @@ var _ = ginkgo.Describe("[deferred][non-regression] Profile non-deferred", ginkg ginkgo.It("should trigger changes", func(ctx context.Context) { tuned, err := util.LoadTuned(tunedPathVMLatency) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tuned.Name, ntoutil.HasDeferredUpdateAnnotation(tuned.Annotations))) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tuned.Name, ntoutil.GetDeferredUpdateAnnotation(tuned.Annotations))) verifications := extractVerifications(tuned) gomega.Expect(len(verifications)).To(gomega.Equal(1), "unexpected verification count, check annotations") diff --git a/test/e2e/deferred/operator_test.go b/test/e2e/deferred/operator_test.go index 55af91863..2d079421b 100644 --- a/test/e2e/deferred/operator_test.go +++ b/test/e2e/deferred/operator_test.go @@ -22,6 +22,10 @@ import ( "github.com/openshift/cluster-node-tuning-operator/test/framework" ) +const ( + defaultWorkerProfile = "openshift-node" +) + const ( verifyCommandAnnotation = "verificationCommand" verifyOutputAnnotation = "verificationOutput" @@ -34,6 +38,9 @@ const ( tunedSHMMNI = "../testing_manifests/deferred/tuned-basic-00.yaml" tunedCPUEnergy = "../testing_manifests/deferred/tuned-basic-10.yaml" tunedVMLatency = "../testing_manifests/deferred/tuned-basic-20.yaml" + + tunedVMLatInplaceBootstrap = "../testing_manifests/deferred/tuned-basic-updates-00.yaml" + tunedVMLatInplaceUpdate = "../testing_manifests/deferred/tuned-basic-updates-10.yaml" ) var ( @@ -118,14 +125,14 @@ func prepend(strs []string, s string) []string { return append([]string{s}, strs...) } -func setDeferred(obj *tunedv1.Tuned) *tunedv1.Tuned { +func setDeferred(obj *tunedv1.Tuned, mode ntoutil.DeferMode) *tunedv1.Tuned { if obj == nil { return obj } if obj.Annotations == nil { obj.Annotations = make(map[string]string) } - obj.Annotations = ntoutil.ToggleDeferredUpdateAnnotation(obj.Annotations, true) + obj.Annotations = ntoutil.SetDeferredUpdateAnnotation(obj.Annotations, mode) return obj } @@ -187,3 +194,40 @@ func checkAppliedConditionStaysOKForNode(ctx context.Context, nodeName, expected return err }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) } + +func checkNontargetWorkerNodesAreUnaffected(ctx context.Context, workerNodes []corev1.Node, targetNodeName string) { + ginkgo.GinkgoHelper() + + // all the other nodes should be fine as well. For them the state should be settled now, so we just check once + // why: because of bugs, we had once allegedly unaffected nodes stuck in deferred updates (obvious bug), let's + // be prudent and add a check this never happens again. Has sky fell yet? + for _, workerNode := range workerNodes { + if workerNode.Name == targetNodeName { + continue // checked previously, if we got this far it's OK + } + + ginkgo.By(fmt.Sprintf("checking non-target worker node after rollback: %q", workerNode.Name)) + + gomega.Eventually(func() error { + wrkNodeProf, err := cs.Profiles(ntoconfig.WatchNamespace()).Get(ctx, workerNode.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + ginkgo.By(fmt.Sprintf("checking profile for target node %q matches expectations about %q", wrkNodeProf.Name, defaultWorkerProfile)) + if wrkNodeProf.Spec.Config.TunedProfile != defaultWorkerProfile { + return fmt.Errorf("checking profile for worker node %q matches expectations %q", wrkNodeProf.Name, defaultWorkerProfile) + } + + ginkgo.By(fmt.Sprintf("checking condition for target node %q matches expectations about %q", wrkNodeProf.Name, defaultWorkerProfile)) + if len(wrkNodeProf.Status.Conditions) == 0 { + return fmt.Errorf("missing status conditions") + } + cond := findCondition(wrkNodeProf.Status.Conditions, tunedv1.TunedProfileApplied) + if cond == nil { + return fmt.Errorf("missing status applied condition") + } + return checkAppliedConditionOK(cond) + }).WithPolling(10. * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + } +} diff --git a/test/e2e/deferred/restart.go b/test/e2e/deferred/restart.go index 56800b794..5e677f921 100644 --- a/test/e2e/deferred/restart.go +++ b/test/e2e/deferred/restart.go @@ -25,6 +25,7 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de var ( createdTuneds []string targetNode *corev1.Node + workerNodes []corev1.Node dirPath string tunedPathSHMMNI string @@ -34,12 +35,13 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de ) ginkgo.BeforeEach(func() { + var err error ginkgo.By("getting a list of worker nodes") - nodes, err := util.GetNodesByRole(cs, "worker") + workerNodes, err = util.GetNodesByRole(cs, "worker") gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(nodes)).NotTo(gomega.BeZero(), "number of worker nodes is 0") + gomega.Expect(workerNodes).ToNot(gomega.BeEmpty(), "number of worker nodes is 0") - targetNode = &nodes[0] + targetNode = &workerNodes[0] ginkgo.By(fmt.Sprintf("using node %q as reference", targetNode.Name)) createdTuneds = []string{} @@ -61,6 +63,11 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de ginkgo.By(fmt.Sprintf("cluster changes rollback: %q", createdTuned)) util.ExecAndLogCommand("oc", "delete", "-n", ntoconfig.WatchNamespace(), "-f", createdTuned) } + + if len(createdTuneds) == 0 { + return + } + checkNontargetWorkerNodesAreUnaffected(context.TODO(), workerNodes, targetNode.Name) }) ginkgo.Context("[slow][pod]the tuned daemon", func() { @@ -75,8 +82,8 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de verifData := util.MustExtractVerificationOutputAndCommand(cs, targetNode, tunedImmediate) gomega.Expect(verifData.OutputCurrent).ToNot(gomega.Equal(verifData.OutputExpected), "current output %q already matches expected %q", verifData.OutputCurrent, verifData.OutputExpected) - tunedMutated := setDeferred(tunedImmediate.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tunedImmediate.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -163,10 +170,10 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de } for _, condition := range curProf.Status.Conditions { if condition.Type == tunedv1.TunedProfileApplied && condition.Status != corev1.ConditionFalse && condition.Reason != "Deferred" { - return fmt.Errorf("Profile deferred=%v %s applied", ntoutil.HasDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) + return fmt.Errorf("Profile deferred=%v %s applied", ntoutil.GetDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) } if condition.Type == tunedv1.TunedDegraded && condition.Status != corev1.ConditionTrue && condition.Reason != "TunedDeferredUpdate" { - return fmt.Errorf("Profile deferred=%v %s not degraded", ntoutil.HasDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) + return fmt.Errorf("Profile deferred=%v %s not degraded", ntoutil.GetDeferredUpdateAnnotation(curProf.Annotations), curProf.Name) } } @@ -196,8 +203,8 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("got the tuned pod running on %q: %s/%s %s", targetNode.Name, targetTunedPod.Namespace, targetTunedPod.Name, targetTunedPod.UID)) - tunedMutated := setDeferred(tunedImmediate.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tunedImmediate.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -297,8 +304,8 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de verifData := util.MustExtractVerificationOutputAndCommand(cs, targetNode, tunedImmediate) gomega.Expect(verifData.OutputCurrent).ToNot(gomega.Equal(verifData.OutputExpected), "verification output current %q matches expected %q", verifData.OutputCurrent, verifData.OutputExpected) - tunedMutated := setDeferred(tunedImmediate.DeepCopy()) - ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.HasDeferredUpdateAnnotation(tunedMutated.Annotations))) + tunedMutated := setDeferred(tunedImmediate.DeepCopy(), ntoutil.DeferAlways) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedMutated.Name, ntoutil.GetDeferredUpdateAnnotation(tunedMutated.Annotations))) _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedMutated, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) @@ -383,10 +390,11 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de return nil }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) - ginkgo.By(fmt.Sprintf("cluster changes rollback: %q", tunedPathSHMMNI)) + ginkgo.By(fmt.Sprintf("doing cluster changes rollback: %q", tunedPathSHMMNI)) _, _, err = util.ExecAndLogCommand("oc", "delete", "-n", ntoconfig.WatchNamespace(), "-f", tunedPathSHMMNI) gomega.Expect(err).ToNot(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("checking target node after rollback: %q", targetNode.Name)) _, createdTuneds, _ = popleft(createdTuneds) gomega.Eventually(func() error { curProf, err := cs.Profiles(ntoconfig.WatchNamespace()).Get(ctx, targetNode.Name, metav1.GetOptions{}) @@ -394,6 +402,7 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de return err } ginkgo.By(fmt.Sprintf("checking profile for target node %q matches expectations about %q", curProf.Name, expectedProfile)) + gomega.Expect(curProf.Name).To(gomega.Equal(expectedProfile), "checking profile for worker node %q matches expectations %q", curProf.Name, expectedProfile) if len(curProf.Status.Conditions) == 0 { return fmt.Errorf("missing status conditions") @@ -418,6 +427,8 @@ var _ = ginkgo.Describe("[deferred][restart][slow][disruptive][flaky] Profile de } return nil }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + checkNontargetWorkerNodesAreUnaffected(ctx, workerNodes, targetNode.Name) }) }) }) diff --git a/test/e2e/deferred/updates.go b/test/e2e/deferred/updates.go new file mode 100644 index 000000000..42940e689 --- /dev/null +++ b/test/e2e/deferred/updates.go @@ -0,0 +1,230 @@ +package e2e + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1" + ntoconfig "github.com/openshift/cluster-node-tuning-operator/pkg/config" + ntoutil "github.com/openshift/cluster-node-tuning-operator/pkg/util" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/util" +) + +var _ = ginkgo.Describe("[deferred][inplace-update] Profile deferred", ginkgo.Label("deferred", "inplace-update"), func() { + ginkgo.Context("when applied", func() { + var ( + createdTuneds []string + referenceNode *corev1.Node // control plane + targetNode *corev1.Node + referenceTunedPod *corev1.Pod // control plane + referenceProfile string + + dirPath string + ) + + ginkgo.BeforeEach(func() { + ginkgo.By("getting a list of worker nodes") + workerNodes, err := util.GetNodesByRole(cs, "worker") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(workerNodes)).NotTo(gomega.BeZero(), "number of worker nodes is 0") + + targetNode = &workerNodes[0] + ginkgo.By(fmt.Sprintf("using node %q as target for workers", targetNode.Name)) + + ginkgo.By("getting a list of control-plane nodes") + controlPlaneNodes, err := util.GetNodesByRole(cs, "control-plane") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(controlPlaneNodes)).NotTo(gomega.BeZero(), "number of control plane nodes is 0") + + referenceNode = &controlPlaneNodes[0] + ginkgo.By(fmt.Sprintf("using node %q as reference control plane", referenceNode.Name)) + + referenceTunedPod, err = util.GetTunedForNode(cs, referenceNode) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(referenceTunedPod.Status.Phase).To(gomega.Equal(corev1.PodRunning)) + + referenceProfile, err = getRecommendedProfile(referenceTunedPod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("using profile %q as reference control plane", referenceProfile)) + + createdTuneds = []string{} + + dirPath, err = util.GetCurrentDirPath() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.AfterEach(func() { + for _, createdTuned := range createdTuneds { + ginkgo.By(fmt.Sprintf("cluster changes rollback: %q", createdTuned)) + util.ExecAndLogCommand("oc", "delete", "-n", ntoconfig.WatchNamespace(), "-f", createdTuned) + } + }) + + ginkgo.It("should not trigger changes when applied fist", func(ctx context.Context) { + /* + * notes about manifests to be used in this test: + * - manifests are already annotated natively (no need to call setDeferred) + * - we need fresh manifests never seen before in the node; this is why we also + * set the annoation natively. This restriction many be lifted in the future. + */ + + tunedPathBootstrap := filepath.Join(dirPath, tunedVMLatInplaceBootstrap) + ginkgo.By(fmt.Sprintf("loading tuned data from %s (basepath=%s)", tunedPathBootstrap, dirPath)) + tunedPathUpdate := filepath.Join(dirPath, tunedVMLatInplaceUpdate) + ginkgo.By(fmt.Sprintf("loading tuned data from %s (basepath=%s)", tunedPathUpdate, dirPath)) + + tunedBootstrap, err := util.LoadTuned(tunedPathBootstrap) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + + tunedUpdate, err := util.LoadTuned(tunedPathUpdate) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("creating tuned object %s deferred=%v", tunedUpdate.Name, ntoutil.GetDeferredUpdateAnnotation(tunedUpdate.Annotations))) + + verifDataBootstrap := util.MustExtractVerificationOutputAndCommand(cs, targetNode, tunedBootstrap) + gomega.Expect(verifDataBootstrap.OutputCurrent).ToNot(gomega.Equal(verifDataBootstrap.OutputExpected), "current output %q already matches expected %q", verifDataBootstrap.OutputCurrent, verifDataBootstrap.OutputExpected) + + _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Create(ctx, tunedBootstrap, metav1.CreateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + createdTuneds = prepend(createdTuneds, tunedPathBootstrap) // we need the path, not the name + ginkgo.By(fmt.Sprintf("create tuneds: %v", createdTuneds)) + + gomega.Expect(tunedBootstrap.Spec.Recommend).ToNot(gomega.BeEmpty(), "tuned %q has empty recommendations", tunedBootstrap.Name) + gomega.Expect(tunedBootstrap.Spec.Recommend[0].Profile).ToNot(gomega.BeNil(), "tuned %q has empty recommended tuned profile", tunedBootstrap.Name) + expectedProfile := *tunedBootstrap.Spec.Recommend[0].Profile + ginkgo.By(fmt.Sprintf("expecting Tuned Profile %q to be picked up", expectedProfile)) + + curProfName := "" + ginkgo.By(fmt.Sprintf("waiting for tuned object %s deferred=%v to be in effect", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + gomega.Eventually(func() error { + curProf, err := cs.Profiles(ntoconfig.WatchNamespace()).Get(ctx, targetNode.Name, metav1.GetOptions{}) + if err != nil { + return err + } + ginkgo.By(fmt.Sprintf("checking profile for target node %q matches expectations about %q", curProf.Name, expectedProfile)) + + if len(curProf.Status.Conditions) == 0 { + return fmt.Errorf("missing status conditions") + } + cond := findCondition(curProf.Status.Conditions, tunedv1.TunedProfileApplied) + if cond == nil { + return fmt.Errorf("missing status applied condition") + } + err = checkAppliedConditionOK(cond) + if err != nil { + util.Logf("profile for target node %q does not match expectations about %q: %v", curProf.Name, expectedProfile, err) + } + recommended, err := getRecommendedProfile(verifDataBootstrap.TargetTunedPod) + if err != nil { + return err + } + if recommended != expectedProfile { + return fmt.Errorf("recommended profile is %q expected %q", recommended, expectedProfile) + } + curProfName = recommended + return err + }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("ensuring tuned object %s deferred=%v are applied", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + gomega.Eventually(func() error { + ginkgo.By(fmt.Sprintf("checking real node conditions for profile %q are changed as expected [command=%v]", curProfName, verifDataBootstrap.CommandArgs)) + out, err := util.ExecCmdInPod(verifDataBootstrap.TargetTunedPod, verifDataBootstrap.CommandArgs...) + if err != nil { + return err + } + out = strings.TrimSpace(out) + if out != verifDataBootstrap.OutputExpected { + return fmt.Errorf("got: %s; expected: %s", out, verifDataBootstrap.OutputExpected) + } + return nil + }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("ensuring tuned object %s deferred=%v changes stay applied", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + gomega.Consistently(func() error { + ginkgo.By(fmt.Sprintf("checking real node conditions for profile %q are changed as expected [command=%v]", curProfName, verifDataBootstrap.CommandArgs)) + out, err := util.ExecCmdInPod(verifDataBootstrap.TargetTunedPod, verifDataBootstrap.CommandArgs...) + if err != nil { + return err + } + out = strings.TrimSpace(out) + if out != verifDataBootstrap.OutputExpected { + return fmt.Errorf("got: %s; expected: %s", out, verifDataBootstrap.OutputExpected) + } + return nil + }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("updating tuned object %s deferred=%v in place", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + gomega.Eventually(func() error { + curTuned, err := cs.Tuneds(ntoconfig.WatchNamespace()).Get(ctx, tunedBootstrap.Name, metav1.GetOptions{}) + if err != nil { + return err + } + curTuned = curTuned.DeepCopy() + curTuned.Spec = tunedUpdate.Spec + _, err = cs.Tuneds(ntoconfig.WatchNamespace()).Update(ctx, curTuned, metav1.UpdateOptions{}) + return err + }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("ensuring tuned object %s deferred=%v is now NOT picked up and stays deferred, being in-place update", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + gomega.Eventually(func() error { + curProf, err := cs.Profiles(ntoconfig.WatchNamespace()).Get(ctx, targetNode.Name, metav1.GetOptions{}) + if err != nil { + return err + } + ginkgo.By(fmt.Sprintf("checking profile for target node %q matches expectations about %q", curProf.Name, expectedProfile)) + + if len(curProf.Status.Conditions) == 0 { + return fmt.Errorf("missing status conditions") + } + ginkgo.By(fmt.Sprintf("checking UPDATED conditions for profile %q: %#v", curProf.Name, curProf.Status.Conditions)) + + cond := findCondition(curProf.Status.Conditions, tunedv1.TunedProfileApplied) + if cond == nil { + return fmt.Errorf("missing status applied condition") + } + + ginkgo.By(fmt.Sprintf("checking UPDATED condition is DEFERRED: %#v", cond)) + err = checkAppliedConditionDeferred(cond, expectedProfile) + if err != nil { + util.Logf("profile for target node %q does not match expectations about %q: %v", curProf.Name, expectedProfile, err) + return err + } + recommended, err := getRecommendedProfile(verifDataBootstrap.TargetTunedPod) + if err != nil { + return err + } + if recommended != expectedProfile { + return fmt.Errorf("recommended profile is %q expected %q", recommended, expectedProfile) + } + return err + }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + ginkgo.By(fmt.Sprintf("ensuring tuned object %s deferred=%v INITIAL changes stay applied", tunedBootstrap.Name, ntoutil.GetDeferredUpdateAnnotation(tunedBootstrap.Annotations))) + gomega.Consistently(func() error { + ginkgo.By(fmt.Sprintf("checking real node conditions for profile %q are changed as expected [command=%v]", curProfName, verifDataBootstrap.CommandArgs)) + out, err := util.ExecCmdInPod(verifDataBootstrap.TargetTunedPod, verifDataBootstrap.CommandArgs...) + if err != nil { + return err + } + out = strings.TrimSpace(out) + if out != verifDataBootstrap.OutputExpected { + return fmt.Errorf("got: %s; expected: %s", out, verifDataBootstrap.OutputExpected) + } + return nil + }).WithPolling(10 * time.Second).WithTimeout(1 * time.Minute).Should(gomega.Succeed()) + + // on non-targeted nodes, like control plane, nothing should have changed + checkAppliedConditionStaysOKForNode(ctx, referenceNode.Name, referenceProfile) + }) + }) +}) diff --git a/test/e2e/testing_manifests/deferred/tuned-basic-updates-00.yaml b/test/e2e/testing_manifests/deferred/tuned-basic-updates-00.yaml new file mode 100644 index 000000000..4c1021076 --- /dev/null +++ b/test/e2e/testing_manifests/deferred/tuned-basic-updates-00.yaml @@ -0,0 +1,26 @@ +apiVersion: tuned.openshift.io/v1 +kind: Tuned +metadata: + name: ocp-prof-deferred-updates-00 + namespace: openshift-cluster-node-tuning-operator + annotations: + tuned.openshift.io/deferred: "update" + verificationCommand: "[\"/usr/sbin/sysctl\", \"-n\", \"vm.swappiness\"]" + verificationOutput: "17" +spec: + profile: + - data: | + [main] + summary=Custom OpenShift profile + include=openshift-node + [sysctl] + vm.swappiness=17 + kernel.shmmni=8192 + vm.dirty_ratio=10 + vm.dirty_background_ratio=3 + name: test-vmlat-inplace + recommend: + - match: + - label: node-role.kubernetes.io/worker + priority: 15 + profile: test-vmlat-inplace diff --git a/test/e2e/testing_manifests/deferred/tuned-basic-updates-10.yaml b/test/e2e/testing_manifests/deferred/tuned-basic-updates-10.yaml new file mode 100644 index 000000000..ae7ce0726 --- /dev/null +++ b/test/e2e/testing_manifests/deferred/tuned-basic-updates-10.yaml @@ -0,0 +1,26 @@ +apiVersion: tuned.openshift.io/v1 +kind: Tuned +metadata: + name: ocp-prof-deferred-updates-00 + namespace: openshift-cluster-node-tuning-operator + annotations: + tuned.openshift.io/deferred: "update" + verificationCommand: "[\"/usr/sbin/sysctl\", \"-n\", \"vm.swappiness\"]" + verificationOutput: "41" +spec: + profile: + - data: | + [main] + summary=Custom OpenShift profile + include=openshift-node + [sysctl] + vm.swappiness=41 + kernel.shmmni=8192 + vm.dirty_ratio=10 + vm.dirty_background_ratio=3 + name: test-vmlat-inplace + recommend: + - match: + - label: node-role.kubernetes.io/worker + priority: 15 + profile: test-vmlat-inplace