From 20c08f5c989c56f91a863497b7d88c9174624d14 Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Thu, 1 Feb 2024 19:15:02 +0100 Subject: [PATCH] e2e: Fix `Debug logging should be visible in multus pod` flake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the SriovNetwork has been created, test routine must wait for NetAttachDefinition to be created by the operator before spawing pods. Create a function `waitForNetAttachDef(...)` and use it in similar situations. The flake error this commit is supposed to fix is: ``` • [FAILED] [2.695 seconds] [sriov] operator Generic SriovNetworkNodePolicy CNI Logging level [It] Debug logging should be visible in multus pod /root/opr-k8s-1/data/sriov-network-operator/sriov-network-operator/test/conformance/tests/test_sriov_operator.go:1077 [FAILED] Unexpected error: <*errors.StatusError | 0xc000fd8640>: admission webhook "network-resources-injector-config.k8s.io" denied the request: could not find network attachment definition 'sriov-conformance-testing/test-log-level-debug-no-file': could not get Network Attachment Definition sriov-conformance-testing/test-log-level-debug-no-file: the server could not find the requested resource { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: { SelfLink: "", ResourceVersion: "", Continue: "", RemainingItemCount: nil, }, Status: "Failure", Message: "admission webhook \"network-resources-injector-config.k8s.io\" denied the request: could not find network attachment definition 'sriov-conformance-testing/test-log-level-debug-no-file': could not get Network Attachment Definition sriov-conformance-testing/test-log-level-debug-no-file: the server could not find the requested resource", Reason: "", Details: nil, Code: 400, }, } occurred In [It] at: /root/opr-k8s-1/data/sriov-network-operator/sriov-network-operator/test/conformance/tests/test_sriov_operator.go:2505 @ 02/01/24 12:44:25.702 ``` Signed-off-by: Andrea Panattoni --- test/conformance/tests/test_sriov_operator.go | 59 ++++++------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/test/conformance/tests/test_sriov_operator.go b/test/conformance/tests/test_sriov_operator.go index bc3ca194e..4abc5e0f5 100644 --- a/test/conformance/tests/test_sriov_operator.go +++ b/test/conformance/tests/test_sriov_operator.go @@ -776,10 +776,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv4 err := network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) pod := createTestPod(node, []string{sriovNetworkName, sriovNetworkName}) nics, err := network.GetNicsByPrefix(pod, "net") @@ -795,10 +792,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv6 err := network.CreateSriovNetwork(clients, sriovDevice, ipv6NetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: ipv6NetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(ipv6NetworkName, namespaces.Test) pod := createTestPod(node, []string{ipv6NetworkName}) ips, err := network.GetSriovNicIPs(pod, "net1") @@ -826,10 +820,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv4 err = network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, ns1, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: ns1}, netAttDef) - }, (30+snoTimeoutMultiplier*90)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, ns1) body, _ := json.Marshal([]patchBody{{ Op: "replace", @@ -838,10 +829,8 @@ var _ = Describe("[sriov] operator", func() { }}) clients.SriovnetworkV1Interface.RESTClient().Patch(types.JSONPatchType).Namespace(operatorNamespace).Resource("sriovnetworks").Name(sriovNetworkName).Body(body).Do(context.Background()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: ns2}, netAttDef) - }, (30+snoTimeoutMultiplier*90)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, ns2) + Consistently(func() error { netAttDef := &netattdefv1.NetworkAttachmentDefinition{} return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: ns1}, netAttDef) @@ -891,10 +880,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv4 err := network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) macvlanNadName := "macvlan-nad" macvlanNad := network.CreateMacvlanNetworkAttachmentDefinition(macvlanNadName, namespaces.Test) @@ -934,10 +920,7 @@ var _ = Describe("[sriov] operator", func() { } err := network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam, []network.SriovNetworkOptions{config}...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) testPod := createTestPod(node, []string{sriovNetworkName}) stdout, _, err := pod.ExecCommand(clients, testPod, "more", "/proc/sys/net/ipv4/conf/net1/accept_redirects") @@ -964,10 +947,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv6 err = network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) testPodA := pod.RedefineWithNodeSelector( pod.DefineWithNetworks([]string{sriovNetworkName, sriovNetworkName, sriovNetworkName, sriovNetworkName, sriovNetworkName}), @@ -1031,6 +1011,7 @@ var _ = Describe("[sriov] operator", func() { sn.Spec.LogLevel = "debug" }) Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) podDeployTime := time.Now() @@ -1381,10 +1362,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv4 err := network.CreateSriovNetwork(clients, mainDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) createTestPod(nodeToTest, []string{sriovNetworkName}) }) }) @@ -1430,10 +1408,7 @@ var _ = Describe("[sriov] operator", func() { err = network.CreateSriovNetwork(clients, unusedSriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(sriovNetworkName, namespaces.Test) changeNodeInterfaceState(testNode, unusedSriovDevice.Name, false) pod := createTestPod(testNode, []string{sriovNetworkName}) ips, err := network.GetSriovNicIPs(pod, "net1") @@ -1794,10 +1769,7 @@ var _ = Describe("[sriov] operator", func() { ipam := ipamIpv4 err = network.CreateSriovNetwork(clients, &nic, ipv6NetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: ipv6NetworkName, Namespace: namespaces.Test}, netAttDef) - }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + waitForNetAttachDef(ipv6NetworkName, namespaces.Test) By("creating a pod") pod := createTestPod(node, []string{ipv6NetworkName}) @@ -2719,3 +2691,10 @@ func getMultusPodLogs(nodeName string, since time.Time) []string { return strings.Split(string(rawLogs), "\n") } + +func waitForNetAttachDef(name, namespace string) { + Eventually(func() error { + netAttDef := &netattdefv1.NetworkAttachmentDefinition{} + return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: name, Namespace: namespace}, netAttDef) + }, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) +}