diff --git a/test/e2e/network_segmentation_services.go b/test/e2e/network_segmentation_services.go index 1c5d000336..99e4621fb4 100644 --- a/test/e2e/network_segmentation_services.go +++ b/test/e2e/network_segmentation_services.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" kapi "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -89,7 +90,7 @@ var _ = Describe("Network Segmentation: services", func() { // that is when https://github.com/ovn-org/ovn-kubernetes/pull/4648 and // https://github.com/ovn-org/ovn-kubernetes/pull/4554 merge - "should be reachable through their cluster IP and node port", + "should be reachable through their cluster IP, external IP, node port and load balancer", func( netConfigParams networkAttachmentConfigParams, ) { @@ -115,7 +116,7 @@ var _ = Describe("Network Segmentation: services", func() { ) Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Creating a UDN NodePort service")) + By(fmt.Sprintf("Creating a UDN LoadBalancer service")) policy := v1.IPFamilyPolicyPreferDualStack udnService, err := jig.CreateUDPService(context.TODO(), func(s *v1.Service) { s.Spec.Ports = []v1.ServicePort{ @@ -126,11 +127,17 @@ var _ = Describe("Network Segmentation: services", func() { TargetPort: intstr.FromInt(int(serviceTargetPort)), }, } - s.Spec.Type = v1.ServiceTypeNodePort + s.Spec.Type = v1.ServiceTypeLoadBalancer s.Spec.IPFamilyPolicy = &policy + // FIXME (quique): NodePort is failing if we use node IPs as externalIPs + //s.Spec.ExternalIPs = internalIPsFromNodes(nodes.Items) }) framework.ExpectNoError(err) + By("Wait for UDN LoadBalancer Ingress to pop up") + udnService, err = jig.WaitForLoadBalancer(context.TODO(), 180*time.Second) + framework.ExpectNoError(err) + By("Creating a UDN backend pod") udnServerPod := e2epod.NewAgnhostPod( namespace, "backend-pod", nil, nil, @@ -157,7 +164,9 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* // UDN -> UDN By("Connect to the UDN service cluster IP from the UDN client pod on the same node") checkConnectionToClusterIPs(f, udnClientPod, udnService, udnServerPod.Name) + checkConnectionToLoadBalancers(f, udnClientPod, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[0], "endpoint node", udnServerPod.Name) + checkConnectionToExternalIPs(f, udnClientPod, udnService, udnServerPod.Name) // FIXME(dceara): Remove this check when Local Gateway external->service support is implemented. if !IsGatewayModeLocal() { checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[1], "client node", udnServerPod.Name) @@ -171,13 +180,26 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* By("Connect to the UDN service from the UDN client pod on a different node") checkConnectionToClusterIPs(f, udnClientPod2, udnService, udnServerPod.Name) + checkConnectionToLoadBalancers(f, udnClientPod2, udnService, udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[1], "local node", udnServerPod.Name) + checkConnectionToExternalIPs(f, udnClientPod2, udnService, udnServerPod.Name) // FIXME(dceara): Remove this check when Local Gateway external->service support is implemented. if !IsGatewayModeLocal() { checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[0], "server node", udnServerPod.Name) checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[2], "other node", udnServerPod.Name) } + By("Connect to the UDN service from the UDN client external container") + clientContainer := "frr" + //FIXME(qinqon) Remove this check when Local Gateway external->service support is implemented. + if !IsGatewayModeLocal() { + checkConnectionToLoadBalancersFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[0], "server node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[1], "other node", udnServerPod.Name) + checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[2], "other node", udnServerPod.Name) + } + checkConnectionToExternalIPsFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name) + // Default network -> UDN // Check that it cannot connect By(fmt.Sprintf("Create a client pod in the default network on node %s", clientNode)) @@ -194,6 +216,8 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* By("Verify the client in the default network connection to the UDN service") checkNoConnectionToClusterIPs(f, defaultClient, udnService) + checkNoConnectionToLoadBalancers(f, defaultClient, udnService) + checkNoConnectionToExternalIPs(f, defaultClient, udnService) checkNoConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[1], "local node") // TODO change to checkConnectionToNodePort when we have full UDN support in ovnkube-node // FIXME(dceara): Remove this check when Local Gateway external->service support is implemented. if !IsGatewayModeLocal() { @@ -239,6 +263,8 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.* // TODO uncomment below when below OVN_DISABLE_SNAT_MULTIPLE_GWS=true is supported // checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[0], "server node", defaultServerPod.Name) // TODO change line below to checkConnectionToNodePort when we have full UDN support in ovnkube-node + checkNoConnectionToLoadBalancers(f, udnClientPod2, defaultService) + checkNoConnectionToExternalIPs(f, udnClientPod2, defaultService) checkNoConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[1], "local node") // TODO uncomment below when OVN_DISABLE_SNAT_MULTIPLE_GWS=true is supported // checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[2], "other node", defaultServerPod.Name) @@ -422,3 +448,136 @@ func checkConnectionOrNoConnectionToNodePort(f *framework.Framework, clientPod * framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) } } + +func checkConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) { + checkConnectionOrNoConnectionToExternalIPs(f, clientPod, service, expectedOutput, true) +} + +func checkNoConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) { + checkConnectionOrNoConnectionToExternalIPs(f, clientPod, service, "", false) +} + +func checkConnectionOrNoConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) { + var err error + targetPort := service.Spec.Ports[0].TargetPort.String() + notStr := "" + if !shouldConnect { + notStr = "not " + } + + for _, externalIP := range service.Spec.ExternalIPs { + msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on external IP %s port %s", + clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, externalIP, targetPort) + By(msg) + + cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %s '`, externalIP, targetPort) + + if shouldConnect { + err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd) + } else { + err = checkNoConnectionToAgnhostPod(f, clientPod, cmd) + } + framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) + } +} + +func checkConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) { + checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, expectedOutput, true) +} + +func checkNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) { + checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, "", false) +} + +func checkConnectionOrNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) { + var err error + targetPort := service.Spec.Ports[0].TargetPort.String() + notStr := "" + if !shouldConnect { + notStr = "not " + } + for _, lbIngress := range service.Status.LoadBalancer.Ingress { + msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on LoadBalancer IP %s port %s", + clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, lbIngress.IP, targetPort) + By(msg) + + cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %s '`, lbIngress.IP, targetPort) + + if shouldConnect { + err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd) + } else { + err = checkNoConnectionToAgnhostPod(f, clientPod, cmd) + } + framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg)) + } +} + +func checkConnectionToNodePortFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, node *v1.Node, nodeRoleMsg, expectedOutput string) { + GinkgoHelper() + var err error + nodePort := service.Spec.Ports[0].NodePort + nodeIPs, err := ParseNodeHostIPDropNetMask(node) + Expect(err).NotTo(HaveOccurred()) + + for nodeIP := range nodeIPs { + msg := fmt.Sprintf("Client at external container %s should connect to NodePort service %s/%s on %s:%d (node %s, %s)", + containerName, service.Namespace, service.Name, nodeIP, nodePort, node.Name, nodeRoleMsg) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %d", nodeIP, nodePort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(5*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func checkConnectionToLoadBalancersFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) { + GinkgoHelper() + targetPort := service.Spec.Ports[0].TargetPort.String() + + for _, lbIngress := range service.Status.LoadBalancer.Ingress { + msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on LoadBalancer IP %s port %s", + containerName, service.Namespace, service.Name, lbIngress.IP, targetPort) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %s", lbIngress.IP, targetPort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(20*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func checkConnectionToExternalIPsFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) { + GinkgoHelper() + targetPort := service.Spec.Ports[0].TargetPort.String() + + for _, externalIP := range service.Spec.ExternalIPs { + msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on External IP %s port %s", + containerName, service.Namespace, service.Name, externalIP, targetPort) + By(msg) + cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %s", externalIP, targetPort)} + Eventually(func() (string, error) { + return runCommand(cmd...) + }). + WithTimeout(5*time.Second). + WithPolling(200*time.Millisecond). + Should(Equal(expectedOutput), "Failed to verify that %s", msg) + } +} + +func internalIPsFromNodes(nodes []corev1.Node) []string { + internalIPs := []string{} + for _, node := range nodes { + for _, nodeAddress := range node.Status.Addresses { + if nodeAddress.Type != corev1.NodeInternalIP { + continue + } + internalIPs = append(internalIPs, nodeAddress.Address) + } + } + return internalIPs +}