Migrate utils from test/kube2e to test/kubernetes (#9720)

changelog/v1.18.0-beta5/utils.yaml

@@ -0,0 +1,8 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6303
+    resolvesIssue: false
+    description: >-
+      Migrate test utilities from kube2e to facilitate test migration.
+
+      skipCI-docs-build:true

pkg/utils/fsutils/fsutils.go

@@ -0,0 +1,35 @@
+package fsutils
+
+import (
+	"fmt"
+	"os"
+)
+
+// ToTempFile takes a string to write to a temp file. It returns the filename and an error.
+func ToTempFile(content string) (string, error) {
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	n, err := f.WriteString(content)
+	if err != nil {
+		return "", err
+	}
+
+	if n != len(content) {
+		return "", fmt.Errorf("expected to write %d bytes, actually wrote %d", len(content), n)
+	}
+	return f.Name(), nil
+}
+
+// IsDirectory checks the provided path is a directory by first checking something exists at that path
+// and then checking that it is a directory.
+func IsDirectory(dir string) bool {
+	stat, err := os.Stat(dir)
+	if err != nil {
+		return false
+	}
+	return stat.IsDir()
+}

pkg/utils/helmutils/client.go

@@ -2,6 +2,7 @@ package helmutils
 
 import (
 	"context"
+	"fmt"
 	"io"
 
 	"github.com/solo-io/gloo/pkg/utils/cmdutils"
@@ -15,6 +16,77 @@ type Client struct {
 	namespace string
 }
 
+// InstallOpts is a set of typical options for a helm install which can be passed in
+// instead of requiring the caller to remember the helm cli flags. extraArgs should
+// always be accepted and respected when using InstallOpts.
+type InstallOpts struct {
+	// KubeContext is the kubernetes context to use.
+	KubeContext string
+
+	// Namespace is the namespace to which the release will be installed.
+	Namespace string
+	// CreateNamespace controls whether to create the namespace or error if it doesn't exist.
+	CreateNamespace bool
+
+	// ValuesFile is the path to the YAML values for the installation.
+	ValuesFile string
+
+	// ReleaseName is the name of the release to install. Usually will be "gloo".
+	ReleaseName string
+
+	// Repository is the remote repo to use. Usually will be one of the constants exported
+	// from this package. Ignored if LocalChartPath is set.
+	Repository string
+
+	// ChartName is the name of the chart to use. Usually will be "gloo". Ignored if LocalChartPath is set.
+	ChartName string
+
+	// LocalChartPath is the path to a locally built tarballed chart to install
+	LocalChartPath string
+}
+
+func (o InstallOpts) all() []string {
+	return append([]string{o.chart(), o.release()}, o.flags()...)
+}
+
+func (o InstallOpts) flags() []string {
+	args := []string{}
+	appendIfNonEmpty := func(fld, flag string) {
+		if fld != "" {
+			args = append(args, flag, fld)
+		}
+	}
+
+	appendIfNonEmpty(o.KubeContext, "--kube-context")
+	appendIfNonEmpty(o.Namespace, "--namespace")
+	if o.CreateNamespace {
+		args = append(args, "--create-namespace")
+	}
+	appendIfNonEmpty(o.ValuesFile, "--values")
+
+	return args
+}
+
+func (o InstallOpts) chart() string {
+	if o.LocalChartPath != "" {
+		return o.LocalChartPath
+	}
+
+	if o.Repository == "" || o.ChartName == "" {
+		return RemoteChartName
+	}
+
+	return fmt.Sprintf("%s/%s", o.Repository, o.ChartName)
+}
+
+func (o InstallOpts) release() string {
+	if o.ReleaseName != "" {
+		return o.ReleaseName
+	}
+
+	return ChartName
+}
+
 // NewClient returns an implementation of the helmutils.Client
 func NewClient() *Client {
 	return &Client{
@@ -62,6 +134,14 @@ func (c *Client) Install(ctx context.Context, extraArgs ...string) error {
 	return c.RunCommand(ctx, args...)
 }
 
+func (c *Client) Delete(ctx context.Context, extraArgs ...string) error {
+	args := append([]string{
+		"delete",
+	}, extraArgs...)
+
+	return c.RunCommand(ctx, args...)
+}
+
 func (c *Client) AddRepository(ctx context.Context, chartName string, chartUrl string, extraArgs ...string) error {
 	args := append([]string{
 		"repo",
@@ -75,3 +155,12 @@ func (c *Client) AddRepository(ctx context.Context, chartName string, chartUrl s
 func (c *Client) AddGlooRepository(ctx context.Context, extraArgs ...string) error {
 	return c.AddRepository(ctx, ChartName, ChartRepositoryUrl, extraArgs...)
 }
+
+func (c *Client) AddPrGlooRepository(ctx context.Context, extraArgs ...string) error {
+	return c.AddRepository(ctx, ChartName, PrChartRepositoryUrl, extraArgs...)
+}
+
+func (c *Client) InstallGloo(ctx context.Context, installOpts InstallOpts, extraArgs ...string) error {
+	args := append(installOpts.all(), extraArgs...)
+	return c.Install(ctx, args...)
+}

pkg/utils/helmutils/constants.go

@@ -6,6 +6,7 @@ const (
 	ChartName = "gloo"
 
 	ChartRepositoryUrl     = "https://storage.googleapis.com/solo-public-helm"
+	PrChartRepositoryUrl   = "https://storage.googleapis.com/solo-public-tagged-helm"
 	RemoteChartUriTemplate = "https://storage.googleapis.com/solo-public-helm/charts/gloo-%s.tgz"
 	RemoteChartName        = "gloo/gloo"
 )

test/gomega/assertions/stats.go

@@ -77,7 +77,6 @@ func EventuallyWithOffsetStatisticsMatchAssertions(offset int, statsPortFwd Stat
 		portForwarder.WaitForStop()
 	}()
 
-	By("Ensure port-forward is open before performing assertions")
 	statsRequest, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%d/", statsPortFwd.LocalPort), nil)
 	ExpectWithOffset(offset+1, err).NotTo(HaveOccurred())
 	EventuallyWithOffset(offset+1, func(g Gomega) {
@@ -90,7 +89,6 @@ func EventuallyWithOffsetStatisticsMatchAssertions(offset int, statsPortFwd Stat
 		}))
 	}).Should(Succeed())
 
-	By("Perform the assertions while the port forward is open")
 	for _, assertion := range assertions {
 		assertion.WithOffset(offset + 1).ShouldNot(HaveOccurred())
 	}

test/kube2e/helper/http_echo.go

@@ -6,6 +6,8 @@ const (
 	HttpEchoPort         = 3000
 )
 
+// Deprecated
+// ported to test/kubernetes/e2e/defaults/testdata/http_echo.yaml
 func NewEchoHttp(namespace string) (TestContainer, error) {
 	return newTestContainer(namespace, defaultHttpEchoImage, HttpEchoName, HttpEchoPort)
 }
@@ -16,6 +18,8 @@ const (
 	TcpEchoPort         = 1025
 )
 
+// Deprecated
+// ported to test/kubernetes/e2e/defaults/testdata/tcp_echo.yaml
 func NewEchoTcp(namespace string) (TestContainer, error) {
 	return newTestContainer(namespace, defaultTcpEchoImage, TcpEchoName, TcpEchoPort)
 }

test/kube2e/helper/kube.go

@@ -51,10 +51,11 @@ func (h *SoloTestHelper) WaitForRollout(ctx context.Context, deploymentName stri
 	}, "30s", "1s").Should(BeTrue())
 }
 
+// Can be replaced entirely with Cli
 func (h *SoloTestHelper) GetContainerLogs(ctx context.Context, namespace string, name string) string {
 	GinkgoHelper()
 
-	out, _, err := h.Cli.Execute(ctx, "-n", namespace, "logs", name)
+	out, err := h.Cli.GetContainerLogs(ctx, namespace, name)
 	Expect(err).ToNot(HaveOccurred())
 	return out
 }

test/kube2e/helper/testserver.go

@@ -80,6 +80,8 @@ const (
 </html>`
 )
 
+// tests relying on the test server should be ported using the default nginx deployment located at
+// test/kubernetes/e2e/defaults/testdata/nginx_pod.yaml
 func NewTestServer(namespace string) (TestUpstreamServer, error) {
 	testContainer, err := newTestContainer(namespace, defaultTestServerImage, TestServerName, TestServerPort)
 	if err != nil {

test/kube2e/util.go

@@ -11,6 +11,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	errors "github.com/rotisserie/eris"
+	"github.com/solo-io/gloo/pkg/utils/fsutils"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options"
 	clienthelpers "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers"
@@ -152,13 +153,10 @@ func UpdateSettingsWithPropagationDelay(updateSettings func(settings *v1.Setting
 }
 
 func ToFile(content string) string {
-	f, err := os.CreateTemp("", "")
-	ExpectWithOffset(1, err).NotTo(HaveOccurred())
-	n, err := f.WriteString(content)
-	ExpectWithOffset(1, err).NotTo(HaveOccurred())
-	ExpectWithOffset(1, n).To(Equal(len(content)))
-	_ = f.Close()
-	return f.Name()
+	fname, err := fsutils.ToTempFile(content)
+	Expect(err).ToNot(HaveOccurred())
+
+	return fname
 }
 
 // https://github.com/solo-io/gloo/issues/4043#issuecomment-772706604

test/kubernetes/e2e/README.md

@@ -18,8 +18,10 @@ We define all tests in the [features](./features) package. This is done for a va
 1. We group the tests by feature, so it's easy to identify which behaviors we assert for a given feature.
 2. We can invoke that same test against different `TestInstallation`s. This means we can test a feature against a variety of installation values, or even against OSS and Enterprise installations.
 
+Many examples of testing features may be found in the [features](./features) package. The general pattern for adding a new feature should be to create a directory for the feature under `features/`, write manifest files for the resources the tests will need into `features/my_feature/testdata/`, define Go objects for them in a file called `features/my_feature/types.go`, and finally define the test suite in `features/my_feature/suite.go`. There are occasions where multiple suites will need to be created under a single feature. See [Suites](#test-suites) for more info on this case. 
+
 ## Test Suites
-A Test Suite is a subset of the Feature concept. A single Feature has at minimum one Test Suite, and can have many. Each Test Suite within the feature must have a function which satisfies the signature `NewSuiteFunc` found in [suite.go](./suite.go).
+A Test Suite is a subset of the Feature concept. A single Feature has at minimum one Test Suite, and can have many. Each Test Suite should have its own appropriately named `.go` file from which is exported an appropriately named function which satisfies the signature `NewSuiteFunc` found in [suite.go](./suite.go).
 
 These test suites are registered by a name and this func in [Tests](#tests) to be run against various `TestInstallation`s.
 

test/kubernetes/e2e/defaults/defaults.go

@@ -1,10 +1,13 @@
 package defaults
 
 import (
+	"path/filepath"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl"
+	"github.com/solo-io/skv2/codegen/util"
 )
 
 var (
@@ -20,4 +23,57 @@ var (
 			Namespace: "curl",
 		},
 	}
+
+	CurlPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "curl_pod.yaml")
+
+	HttpEchoPod = &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "http-echo",
+			Namespace: "http-echo",
+		},
+	}
+
+	HttpEchoPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "http_echo.yaml")
+
+	TcpEchoPod = &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-echo",
+			Namespace: "tcp-echo",
+		},
+	}
+
+	TcpEchoPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "tcp_echo.yaml")
+
+	NginxPod = &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "nginx",
+			Namespace: "nginx",
+		},
+	}
+
+	NginxPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "nginx_pod.yaml")
+
+	NginxResponse = `<!DOCTYPE html>
+<html>
+<head>
+<title>Welcome to nginx!</title>
+<style>
+html { color-scheme: light dark; }
+body { width: 35em; margin: 0 auto;
+font-family: Tahoma, Verdana, Arial, sans-serif; }
+</style>
+</head>
+<body>
+<h1>Welcome to nginx!</h1>
+<p>If you see this page, the nginx web server is successfully installed and
+working. Further configuration is required.</p>
+
+<p>For online documentation and support please refer to
+<a href="http://nginx.org/">nginx.org</a>.<br/>
+Commercial support is available at
+<a href="http://nginx.com/">nginx.com</a>.</p>
+
+<p><em>Thank you for using nginx.</em></p>
+</body>
+</html>`
 )

test/kubernetes/e2e/defaults/testdata/http_echo.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: http-echo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-echo
+  namespace: http-echo
+spec:
+  selector:
+    app.kubernetes.io/name: http-echo
+  ports:
+    - protocol: TCP
+      port: 3000
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: http-echo
+  namespace: http-echo
+  labels:
+    app.kubernetes.io/name: http-echo
+spec:
+  containers:
+  - name: http-echo
+    image: kennship/http-echo@sha256:144322e8e96be2be6675dcf6e3ee15697c5d052d14d240e8914871a2a83990af
+    ports:
+      - containerPort: 3000
+    resources:
+      requests:
+        cpu: "100m"
+      limits:
+        cpu: "200m"