Skip to content

Commit

Permalink
udn, e2e, svc: external client, externalIP and lb
Browse files Browse the repository at this point in the history 
Add testing for:
- External clients to LB, NodePort and ExternalIPs
- Podify clients to LB and ExternalIPs

Also skipping testing not working yet with local gw mode.

Signed-off-by: Enrique Llorente <ellorent@redhat.com>
  • Loading branch information
@qinqon
qinqon committed Sep 13, 2024
1 parent 297be61 commit 5709618
Showing 1 changed file with 162 additions and 3 deletions.
165 changes: 162 additions & 3 deletions test/e2e/network_segmentation_services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
kapi "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -89,7 +90,7 @@ var _ = Describe("Network Segmentation: services", func() {
// that is when https://github.com/ovn-org/ovn-kubernetes/pull/4648 and
// https://github.com/ovn-org/ovn-kubernetes/pull/4554 merge

"should be reachable through their cluster IP and node port",
"should be reachable through their cluster IP, external IP, node port and load balancer",
func(
netConfigParams networkAttachmentConfigParams,
) {
Expand All @@ -115,7 +116,7 @@ var _ = Describe("Network Segmentation: services", func() {
)
Expect(err).NotTo(HaveOccurred())

By(fmt.Sprintf("Creating a UDN NodePort service"))
By(fmt.Sprintf("Creating a UDN LoadBalancer service"))
policy := v1.IPFamilyPolicyPreferDualStack
udnService, err := jig.CreateUDPService(context.TODO(), func(s *v1.Service) {
s.Spec.Ports = []v1.ServicePort{
Expand All @@ -126,11 +127,17 @@ var _ = Describe("Network Segmentation: services", func() {
TargetPort: intstr.FromInt(int(serviceTargetPort)),
},
}
s.Spec.Type = v1.ServiceTypeNodePort
s.Spec.Type = v1.ServiceTypeLoadBalancer
s.Spec.IPFamilyPolicy = &policy
// FIXME (quique): NodePort is failing if we use node IPs as externalIPs
//s.Spec.ExternalIPs = internalIPsFromNodes(nodes.Items)
})
framework.ExpectNoError(err)

By("Wait for UDN LoadBalancer Ingress to pop up")
udnService, err = jig.WaitForLoadBalancer(context.TODO(), 180*time.Second)
framework.ExpectNoError(err)

By("Creating a UDN backend pod")
udnServerPod := e2epod.NewAgnhostPod(
namespace, "backend-pod", nil, nil,
Expand All @@ -157,7 +164,9 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.*
// UDN -> UDN
By("Connect to the UDN service cluster IP from the UDN client pod on the same node")
checkConnectionToClusterIPs(f, udnClientPod, udnService, udnServerPod.Name)
checkConnectionToLoadBalancers(f, udnClientPod, udnService, udnServerPod.Name)
checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[0], "endpoint node", udnServerPod.Name)
checkConnectionToExternalIPs(f, udnClientPod, udnService, udnServerPod.Name)
// FIXME(dceara): Remove this check when Local Gateway external->service support is implemented.
if !IsGatewayModeLocal() {
checkConnectionToNodePort(f, udnClientPod, udnService, &nodes.Items[1], "client node", udnServerPod.Name)
Expand All @@ -171,13 +180,26 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.*

By("Connect to the UDN service from the UDN client pod on a different node")
checkConnectionToClusterIPs(f, udnClientPod2, udnService, udnServerPod.Name)
checkConnectionToLoadBalancers(f, udnClientPod2, udnService, udnServerPod.Name)
checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[1], "local node", udnServerPod.Name)
checkConnectionToExternalIPs(f, udnClientPod2, udnService, udnServerPod.Name)
// FIXME(dceara): Remove this check when Local Gateway external->service support is implemented.
if !IsGatewayModeLocal() {
checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[0], "server node", udnServerPod.Name)
checkConnectionToNodePort(f, udnClientPod2, udnService, &nodes.Items[2], "other node", udnServerPod.Name)
}

By("Connect to the UDN service from the UDN client external container")
clientContainer := "frr"
//FIXME(qinqon) Remove this check when Local Gateway external->service support is implemented.
if !IsGatewayModeLocal() {
checkConnectionToLoadBalancersFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name)
checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[0], "server node", udnServerPod.Name)
checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[1], "other node", udnServerPod.Name)
checkConnectionToNodePortFromExternalContainer(f, clientContainer, udnService, &nodes.Items[2], "other node", udnServerPod.Name)
}
checkConnectionToExternalIPsFromExternalContainer(f, clientContainer, udnService, udnServerPod.Name)

// Default network -> UDN
// Check that it cannot connect
By(fmt.Sprintf("Create a client pod in the default network on node %s", clientNode))
Expand All @@ -194,6 +216,8 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.*

By("Verify the client in the default network connection to the UDN service")
checkNoConnectionToClusterIPs(f, defaultClient, udnService)
checkNoConnectionToLoadBalancers(f, defaultClient, udnService)
checkNoConnectionToExternalIPs(f, defaultClient, udnService)
checkNoConnectionToNodePort(f, defaultClient, udnService, &nodes.Items[1], "local node") // TODO change to checkConnectionToNodePort when we have full UDN support in ovnkube-node
// FIXME(dceara): Remove this check when Local Gateway external->service support is implemented.
if !IsGatewayModeLocal() {
Expand Down Expand Up @@ -239,6 +263,8 @@ ipv6=$(ip -6 a show dev ovn-udn1 | grep 'global' | awk '{print $2}' | sed 's#/.*
// TODO uncomment below when below OVN_DISABLE_SNAT_MULTIPLE_GWS=true is supported
// checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[0], "server node", defaultServerPod.Name)
// TODO change line below to checkConnectionToNodePort when we have full UDN support in ovnkube-node
checkNoConnectionToLoadBalancers(f, udnClientPod2, defaultService)
checkNoConnectionToExternalIPs(f, udnClientPod2, defaultService)
checkNoConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[1], "local node")
// TODO uncomment below when OVN_DISABLE_SNAT_MULTIPLE_GWS=true is supported
// checkConnectionToNodePort(f, udnClientPod2, defaultService, &nodes.Items[2], "other node", defaultServerPod.Name)
Expand Down Expand Up @@ -422,3 +448,136 @@ func checkConnectionOrNoConnectionToNodePort(f *framework.Framework, clientPod *
framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg))
}
}

func checkConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) {
checkConnectionOrNoConnectionToExternalIPs(f, clientPod, service, expectedOutput, true)
}

func checkNoConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) {
checkConnectionOrNoConnectionToExternalIPs(f, clientPod, service, "", false)
}

func checkConnectionOrNoConnectionToExternalIPs(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) {
var err error
targetPort := service.Spec.Ports[0].TargetPort.String()
notStr := ""
if !shouldConnect {
notStr = "not "
}

for _, externalIP := range service.Spec.ExternalIPs {
msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on external IP %s port %s",
clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, externalIP, targetPort)
By(msg)

cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %s '`, externalIP, targetPort)

if shouldConnect {
err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd)
} else {
err = checkNoConnectionToAgnhostPod(f, clientPod, cmd)
}
framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg))
}
}

func checkConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string) {
checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, expectedOutput, true)
}

func checkNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service) {
checkConnectionOrNoConnectionToLoadBalancers(f, clientPod, service, "", false)
}

func checkConnectionOrNoConnectionToLoadBalancers(f *framework.Framework, clientPod *v1.Pod, service *v1.Service, expectedOutput string, shouldConnect bool) {
var err error
targetPort := service.Spec.Ports[0].TargetPort.String()
notStr := ""
if !shouldConnect {
notStr = "not "
}
for _, lbIngress := range service.Status.LoadBalancer.Ingress {
msg := fmt.Sprintf("Client %s/%s should %sreach service %s/%s on LoadBalancer IP %s port %s",
clientPod.Namespace, clientPod.Name, notStr, service.Namespace, service.Name, lbIngress.IP, targetPort)
By(msg)

cmd := fmt.Sprintf(`/bin/sh -c 'echo hostname | nc -u -w 1 %s %s '`, lbIngress.IP, targetPort)

if shouldConnect {
err = checkConnectionToAgnhostPod(f, clientPod, expectedOutput, cmd)
} else {
err = checkNoConnectionToAgnhostPod(f, clientPod, cmd)
}
framework.ExpectNoError(err, fmt.Sprintf("Failed to verify that %s", msg))
}
}

func checkConnectionToNodePortFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, node *v1.Node, nodeRoleMsg, expectedOutput string) {
GinkgoHelper()
var err error
nodePort := service.Spec.Ports[0].NodePort
nodeIPs, err := ParseNodeHostIPDropNetMask(node)
Expect(err).NotTo(HaveOccurred())

for nodeIP := range nodeIPs {
msg := fmt.Sprintf("Client at external container %s should connect to NodePort service %s/%s on %s:%d (node %s, %s)",
containerName, service.Namespace, service.Name, nodeIP, nodePort, node.Name, nodeRoleMsg)
By(msg)
cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %d", nodeIP, nodePort)}
Eventually(func() (string, error) {
return runCommand(cmd...)
}).
WithTimeout(5*time.Second).
WithPolling(200*time.Millisecond).
Should(Equal(expectedOutput), "Failed to verify that %s", msg)
}
}

func checkConnectionToLoadBalancersFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) {
GinkgoHelper()
targetPort := service.Spec.Ports[0].TargetPort.String()

for _, lbIngress := range service.Status.LoadBalancer.Ingress {
msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on LoadBalancer IP %s port %s",
containerName, service.Namespace, service.Name, lbIngress.IP, targetPort)
By(msg)
cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %s", lbIngress.IP, targetPort)}
Eventually(func() (string, error) {
return runCommand(cmd...)
}).
WithTimeout(20*time.Second).
WithPolling(200*time.Millisecond).
Should(Equal(expectedOutput), "Failed to verify that %s", msg)
}
}

func checkConnectionToExternalIPsFromExternalContainer(f *framework.Framework, containerName string, service *v1.Service, expectedOutput string) {
GinkgoHelper()
targetPort := service.Spec.Ports[0].TargetPort.String()

for _, externalIP := range service.Spec.ExternalIPs {
msg := fmt.Sprintf("Client at external container %s should reach service %s/%s on External IP %s port %s",
containerName, service.Namespace, service.Name, externalIP, targetPort)
By(msg)
cmd := []string{containerRuntime, "exec", containerName, "/bin/bash", "-c", fmt.Sprintf("echo hostname | nc -u -w 1 %s %s", externalIP, targetPort)}
Eventually(func() (string, error) {
return runCommand(cmd...)
}).
WithTimeout(5*time.Second).
WithPolling(200*time.Millisecond).
Should(Equal(expectedOutput), "Failed to verify that %s", msg)
}
}

func internalIPsFromNodes(nodes []corev1.Node) []string {
internalIPs := []string{}
for _, node := range nodes {
for _, nodeAddress := range node.Status.Addresses {
if nodeAddress.Type != corev1.NodeInternalIP {
continue
}
internalIPs = append(internalIPs, nodeAddress.Address)
}
}
return internalIPs
}

0 comments on commit 5709618

Please sign in to comment.