From 6e138a7bc16dbb144a66cff81d06590e54182868 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Tue, 10 Sep 2024 13:23:56 +0200 Subject: [PATCH] udn, e2e, svc: external client, externalIP and lb Add testing for: - External clients to LB, NodePort and ExternalIPs - Podify clients to LB and ExternalIPs Also skipping testing not working yet with local gw mode. Signed-off-by: Enrique Llorente --- test/e2e/network_segmentation_services.go | 167 +++++++++++++++++++++- 1 file changed, 163 insertions(+), 4 deletions(-) diff --git a/test/e2e/network_segmentation_services.go b/test/e2e/network_segmentation_services.go index a2fa671410..7566b67f7a 100644 --- a/test/e2e/network_segmentation_services.go +++ b/test/e2e/network_segmentation_services.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + + corev1 "k8s.io/api/core/v1" kapi "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -86,7 +88,7 @@ var _ = Describe("Network Segmentation: services", func() { // + clusterIP fails // + nodeIP:nodePort fails FOR NOW, when we only target the local node (*) - "should be reachable through their cluster IP and node port", + "should be reachable through their cluster IP, external IP, node port and load balancer", func( netConfigParams networkAttachmentConfigParams, ) { @@ -119,7 +121,7 @@ var _ = Describe("Network Segmentation: services", func() { ) Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Creating a UDN NodePort service")) + By(fmt.Sprintf("Creating a UDN LoadBalancer service")) policy := v1.IPFamilyPolicyPreferDualStack udnService, err := jig.CreateUDPService(context.TODO(), func(s *v1.Service) { s.Spec.Ports = []v1.ServicePort{ @@ -130,11 +132,17 @@ var _ = Describe("Network Segmentation: services", func() { TargetPort: intstr.FromInt(int(serviceTargetPort)), }, } - s.Spec.Type = v1.ServiceTypeNodePort + s.Spec.Type = v1.ServiceTypeLoadBalancer s.Spec.IPFamilyPolicy = &policy + // FIXME (quique): NodePort is failing if we use node IPs as externalIPs + //s.Spec.ExternalIPs = internalIPsFromNodes(nodes.Items) }) framework.ExpectNoError(err) + By("Wait for UDN LoadBalancer Ingress to pop up") + udnService, err = jig.WaitForLoadBalancer(context.TODO(), 180*time.Second) + framework.ExpectNoError(err) + By("Creating a UDN backend pod") udnServerPod := e2epod.NewAgnhostPod( namespace, "backend-pod", nil, nil, @@ -161,7 +169,9 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* // UDN -> UDN By("Connect to the UDN service cluster IP from the UDN client pod on the same node") checkConnectionToClusterIPs(f, udnClientPod, udnService, udnServerPod.Name) + checkConnectionToLoadBalancers(f, udnClientPod, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[0], "endpoint node", udnServerPod.Name) + checkConnectionToExternalIPs(f, udnClientPod, udnService, udnServerPod.Name) // FIXME(dceara): Remove this check when Local Gateway external->service support is implemented. if !IsGatewayModeLocal() { // FIXME(kyrtapz): Remove once l2 external->svc is fixed. Client node is nodes.Items[0] @@ -178,7 +188,9 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* By("Connect to the UDN service from the UDN client pod on a different node") checkConnectionToClusterIPs(f, udnClientPod2, udnService, udnServerPod.Name) + checkConnectionToLoadBalancers(f, udnClientPod2, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[1], "local node", udnServerPod.Name) + checkConnectionToExternalIPs(f, udnClientPod2, udnService, udnServerPod.Name) // FIXME(dceara): Remove this check when Local Gateway external->service support is implemented. if !IsGatewayModeLocal() { // FIXME(kyrtapz): Remove once l2 external->svc is fixed. Client node is nodes.Items[1] @@ -188,6 +200,17 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* } } + By("Connect to the UDN service from the UDN client external container") + clientContainer := "frr" + //FIXME(qinqon) Remove this check when Local Gateway external->service support is implemented. + if !IsGatewayModeLocal() { + checkConnectionToLoadBalancersFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[0], "server node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[1], "other node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + } + checkConnectionToExternalIPsFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name) + // Default network -> UDN // Check that it cannot connect By(fmt.Sprintf("Create a client pod in the default network on node %s", clientNode)) @@ -204,7 +227,8 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* By("Verify the client in the default network connection to the UDN service") checkNoConnectionToClusterIPs(f, defaultClient, udnService) - + checkNoConnectionToLoadBalancers(f, defaultClient, udnService) + checkNoConnectionToExternalIPs(f, defaultClient, udnService) checkNoConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[1], "local node") // TODO change to checkConnectionToNodePort when we have full UDN support in ovnkube-node // FIXME(dceara): Remove this check when Local Gateway external->service support is implemented. @@ -250,6 +274,8 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* Expect(err).NotTo(HaveOccurred()) By("Verify the UDN client connection to the default network service") + checkNoConnectionToLoadBalancers(f, udnClientPod2, defaultService) + checkNoConnectionToExternalIPs(f, udnClientPod2, defaultService) checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[0], "server node", defaultServerPod.Name) checkNoConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[1], "local node") checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[2], "other node", defaultServerPod.Name) @@ -434,3 +460,136 @@ func checkConnectionOrNoConnectionToNodePort(f *framework.Framework, clientPod * framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) } } + +func checkConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) { + checkConnectionOrNoConnectionToExternalIPs(f, clientPod, service, expectedOutput, true) +} + +func checkNoConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) { + checkConnectionOrNoConnectionToExternalIPs(f, clientPod, service, "", false) +} + +func checkConnectionOrNoConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) { + var err error + targetPort := service.Spec.Ports[0].TargetPort.String() + notStr := "" + if !shouldConnect { + notStr = "not " + } + + for _, externalIP := range service.Spec.ExternalIPs { + msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on external IP %s port %s", + clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, externalIP, targetPort) + By(msg) + + cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %s '`, externalIP, targetPort) + + if shouldConnect { + err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd) + } else { + err = checkNoConnectionToAgnhostPod(f, clientPod, cmd) + } + framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) + } +} + +func checkConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) { + checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, expectedOutput, true) +} + +func checkNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) { + checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, "", false) +} + +func checkConnectionOrNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) { + var err error + targetPort := service.Spec.Ports[0].TargetPort.String() + notStr := "" + if !shouldConnect { + notStr = "not " + } + for _, lbIngress := range service.Status.LoadBalancer.Ingress { + msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on LoadBalancer IP %s port %s", + clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, lbIngress.IP, targetPort) + By(msg) + + cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %s '`, lbIngress.IP, targetPort) + + if shouldConnect { + err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd) + } else { + err = checkNoConnectionToAgnhostPod(f, clientPod, cmd) + } + framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) + } +} + +func checkConnectionToNodePortFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, node *v1.Node, nodeRoleMsg, expectedOutput string) { + GinkgoHelper() + var err error + nodePort := service.Spec.Ports[0].NodePort + nodeIPs, err := ParseNodeHostIPDropNetMask(node) + Expect(err).NotTo(HaveOccurred()) + + for nodeIP := range nodeIPs { + msg := fmt.Sprintf("Client at external container %s should connect to NodePort service %s/%s on %s:%d (node %s, %s)", + containerName, service.Namespace, service.Name, nodeIP, nodePort, node.Name, nodeRoleMsg) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %d", nodeIP, nodePort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(5*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func checkConnectionToLoadBalancersFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) { + GinkgoHelper() + targetPort := service.Spec.Ports[0].TargetPort.String() + + for _, lbIngress := range service.Status.LoadBalancer.Ingress { + msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on LoadBalancer IP %s port %s", + containerName, service.Namespace, service.Name, lbIngress.IP, targetPort) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %s", lbIngress.IP, targetPort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(20*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func checkConnectionToExternalIPsFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) { + GinkgoHelper() + targetPort := service.Spec.Ports[0].TargetPort.String() + + for _, externalIP := range service.Spec.ExternalIPs { + msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on External IP %s port %s", + containerName, service.Namespace, service.Name, externalIP, targetPort) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %s", externalIP, targetPort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(5*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func internalIPsFromNodes(nodes []corev1.Node) []string { + internalIPs := []string{} + for _, node := range nodes { + for _, nodeAddress := range node.Status.Addresses { + if nodeAddress.Type != corev1.NodeInternalIP { + continue + } + internalIPs = append(internalIPs, nodeAddress.Address) + } + } + return internalIPs +}