From 69d12f82284c4984f232a70892b1658bbe2664a2 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 4 Jul 2024 07:48:23 -0400 Subject: [PATCH] Migrate utils from `test/kube2e` to `test/kubernetes` (#9720) --- changelog/v1.18.0-beta5/utils.yaml | 8 + pkg/utils/fsutils/fsutils.go | 35 ++ pkg/utils/helmutils/client.go | 89 ++++ pkg/utils/helmutils/constants.go | 1 + test/gomega/assertions/stats.go | 2 - test/kube2e/helper/http_echo.go | 4 + test/kube2e/helper/kube.go | 3 +- test/kube2e/helper/testserver.go | 2 + test/kube2e/util.go | 12 +- test/kubernetes/e2e/README.md | 4 +- test/kubernetes/e2e/defaults/defaults.go | 56 +++ .../e2e/defaults/testdata/http_echo.yaml | 36 ++ .../e2e/defaults/testdata/nginx_pod.yaml | 138 ++++++ .../e2e/defaults/testdata/tcp_echo.yaml | 37 ++ .../e2e/example/debug_logging_test.go | 4 +- test/kubernetes/e2e/test.go | 15 +- .../e2e/tests/automtls_istio_edge_api_test.go | 4 +- .../e2e/tests/automtls_istio_test.go | 4 +- test/kubernetes/e2e/tests/edge_gw_test.go | 4 +- .../e2e/tests/glooctl_istio_inject_test.go | 4 +- .../e2e/tests/istio_edge_api_test.go | 4 +- .../e2e/tests/istio_regression_test.go | 4 +- .../kubernetes/e2e/tests/k8s_gw_istio_test.go | 4 +- ..._minimal_default_gatewayparameters_test.go | 4 +- .../e2e/tests/k8s_gw_no_validation_test.go | 4 +- test/kubernetes/e2e/tests/k8s_gw_test.go | 4 +- .../testutils/assertions/deployments.go | 33 +- .../testutils/assertions/glooctl.go | 35 ++ .../testutils/helper/docker/Dockerfile | 10 + test/kubernetes/testutils/helper/install.go | 404 ++++++++++++++++++ test/kubernetes/testutils/helper/util.go | 147 +++++++ 31 files changed, 1076 insertions(+), 39 deletions(-) create mode 100644 changelog/v1.18.0-beta5/utils.yaml create mode 100644 pkg/utils/fsutils/fsutils.go create mode 100644 test/kubernetes/e2e/defaults/testdata/http_echo.yaml create mode 100644 test/kubernetes/e2e/defaults/testdata/nginx_pod.yaml create mode 100644 test/kubernetes/e2e/defaults/testdata/tcp_echo.yaml create mode 100644 test/kubernetes/testutils/helper/docker/Dockerfile create mode 100644 test/kubernetes/testutils/helper/install.go create mode 100644 test/kubernetes/testutils/helper/util.go diff --git a/changelog/v1.18.0-beta5/utils.yaml b/changelog/v1.18.0-beta5/utils.yaml new file mode 100644 index 00000000000..0d03569aa6f --- /dev/null +++ b/changelog/v1.18.0-beta5/utils.yaml @@ -0,0 +1,8 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/6303 + resolvesIssue: false + description: >- + Migrate test utilities from kube2e to facilitate test migration. + + skipCI-docs-build:true diff --git a/pkg/utils/fsutils/fsutils.go b/pkg/utils/fsutils/fsutils.go new file mode 100644 index 00000000000..d4e45c09167 --- /dev/null +++ b/pkg/utils/fsutils/fsutils.go @@ -0,0 +1,35 @@ +package fsutils + +import ( + "fmt" + "os" +) + +// ToTempFile takes a string to write to a temp file. It returns the filename and an error. +func ToTempFile(content string) (string, error) { + f, err := os.CreateTemp("", "") + if err != nil { + return "", err + } + defer f.Close() + + n, err := f.WriteString(content) + if err != nil { + return "", err + } + + if n != len(content) { + return "", fmt.Errorf("expected to write %d bytes, actually wrote %d", len(content), n) + } + return f.Name(), nil +} + +// IsDirectory checks the provided path is a directory by first checking something exists at that path +// and then checking that it is a directory. +func IsDirectory(dir string) bool { + stat, err := os.Stat(dir) + if err != nil { + return false + } + return stat.IsDir() +} diff --git a/pkg/utils/helmutils/client.go b/pkg/utils/helmutils/client.go index 919307a662f..cc0edafa63d 100644 --- a/pkg/utils/helmutils/client.go +++ b/pkg/utils/helmutils/client.go @@ -2,6 +2,7 @@ package helmutils import ( "context" + "fmt" "io" "github.com/solo-io/gloo/pkg/utils/cmdutils" @@ -15,6 +16,77 @@ type Client struct { namespace string } +// InstallOpts is a set of typical options for a helm install which can be passed in +// instead of requiring the caller to remember the helm cli flags. extraArgs should +// always be accepted and respected when using InstallOpts. +type InstallOpts struct { + // KubeContext is the kubernetes context to use. + KubeContext string + + // Namespace is the namespace to which the release will be installed. + Namespace string + // CreateNamespace controls whether to create the namespace or error if it doesn't exist. + CreateNamespace bool + + // ValuesFile is the path to the YAML values for the installation. + ValuesFile string + + // ReleaseName is the name of the release to install. Usually will be "gloo". + ReleaseName string + + // Repository is the remote repo to use. Usually will be one of the constants exported + // from this package. Ignored if LocalChartPath is set. + Repository string + + // ChartName is the name of the chart to use. Usually will be "gloo". Ignored if LocalChartPath is set. + ChartName string + + // LocalChartPath is the path to a locally built tarballed chart to install + LocalChartPath string +} + +func (o InstallOpts) all() []string { + return append([]string{o.chart(), o.release()}, o.flags()...) +} + +func (o InstallOpts) flags() []string { + args := []string{} + appendIfNonEmpty := func(fld, flag string) { + if fld != "" { + args = append(args, flag, fld) + } + } + + appendIfNonEmpty(o.KubeContext, "--kube-context") + appendIfNonEmpty(o.Namespace, "--namespace") + if o.CreateNamespace { + args = append(args, "--create-namespace") + } + appendIfNonEmpty(o.ValuesFile, "--values") + + return args +} + +func (o InstallOpts) chart() string { + if o.LocalChartPath != "" { + return o.LocalChartPath + } + + if o.Repository == "" || o.ChartName == "" { + return RemoteChartName + } + + return fmt.Sprintf("%s/%s", o.Repository, o.ChartName) +} + +func (o InstallOpts) release() string { + if o.ReleaseName != "" { + return o.ReleaseName + } + + return ChartName +} + // NewClient returns an implementation of the helmutils.Client func NewClient() *Client { return &Client{ @@ -62,6 +134,14 @@ func (c *Client) Install(ctx context.Context, extraArgs ...string) error { return c.RunCommand(ctx, args...) } +func (c *Client) Delete(ctx context.Context, extraArgs ...string) error { + args := append([]string{ + "delete", + }, extraArgs...) + + return c.RunCommand(ctx, args...) +} + func (c *Client) AddRepository(ctx context.Context, chartName string, chartUrl string, extraArgs ...string) error { args := append([]string{ "repo", @@ -75,3 +155,12 @@ func (c *Client) AddRepository(ctx context.Context, chartName string, chartUrl s func (c *Client) AddGlooRepository(ctx context.Context, extraArgs ...string) error { return c.AddRepository(ctx, ChartName, ChartRepositoryUrl, extraArgs...) } + +func (c *Client) AddPrGlooRepository(ctx context.Context, extraArgs ...string) error { + return c.AddRepository(ctx, ChartName, PrChartRepositoryUrl, extraArgs...) +} + +func (c *Client) InstallGloo(ctx context.Context, installOpts InstallOpts, extraArgs ...string) error { + args := append(installOpts.all(), extraArgs...) + return c.Install(ctx, args...) +} diff --git a/pkg/utils/helmutils/constants.go b/pkg/utils/helmutils/constants.go index c0427682d84..0bd38724220 100644 --- a/pkg/utils/helmutils/constants.go +++ b/pkg/utils/helmutils/constants.go @@ -6,6 +6,7 @@ const ( ChartName = "gloo" ChartRepositoryUrl = "https://storage.googleapis.com/solo-public-helm" + PrChartRepositoryUrl = "https://storage.googleapis.com/solo-public-tagged-helm" RemoteChartUriTemplate = "https://storage.googleapis.com/solo-public-helm/charts/gloo-%s.tgz" RemoteChartName = "gloo/gloo" ) diff --git a/test/gomega/assertions/stats.go b/test/gomega/assertions/stats.go index 5fe61be16a7..e89cc8fecca 100644 --- a/test/gomega/assertions/stats.go +++ b/test/gomega/assertions/stats.go @@ -77,7 +77,6 @@ func EventuallyWithOffsetStatisticsMatchAssertions(offset int, statsPortFwd Stat portForwarder.WaitForStop() }() - By("Ensure port-forward is open before performing assertions") statsRequest, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%d/", statsPortFwd.LocalPort), nil) ExpectWithOffset(offset+1, err).NotTo(HaveOccurred()) EventuallyWithOffset(offset+1, func(g Gomega) { @@ -90,7 +89,6 @@ func EventuallyWithOffsetStatisticsMatchAssertions(offset int, statsPortFwd Stat })) }).Should(Succeed()) - By("Perform the assertions while the port forward is open") for _, assertion := range assertions { assertion.WithOffset(offset + 1).ShouldNot(HaveOccurred()) } diff --git a/test/kube2e/helper/http_echo.go b/test/kube2e/helper/http_echo.go index 6f2f6af1edc..b0d1745c854 100644 --- a/test/kube2e/helper/http_echo.go +++ b/test/kube2e/helper/http_echo.go @@ -6,6 +6,8 @@ const ( HttpEchoPort = 3000 ) +// Deprecated +// ported to test/kubernetes/e2e/defaults/testdata/http_echo.yaml func NewEchoHttp(namespace string) (TestContainer, error) { return newTestContainer(namespace, defaultHttpEchoImage, HttpEchoName, HttpEchoPort) } @@ -16,6 +18,8 @@ const ( TcpEchoPort = 1025 ) +// Deprecated +// ported to test/kubernetes/e2e/defaults/testdata/tcp_echo.yaml func NewEchoTcp(namespace string) (TestContainer, error) { return newTestContainer(namespace, defaultTcpEchoImage, TcpEchoName, TcpEchoPort) } diff --git a/test/kube2e/helper/kube.go b/test/kube2e/helper/kube.go index 5cd556c2998..343621b0989 100644 --- a/test/kube2e/helper/kube.go +++ b/test/kube2e/helper/kube.go @@ -51,10 +51,11 @@ func (h *SoloTestHelper) WaitForRollout(ctx context.Context, deploymentName stri }, "30s", "1s").Should(BeTrue()) } +// Can be replaced entirely with Cli func (h *SoloTestHelper) GetContainerLogs(ctx context.Context, namespace string, name string) string { GinkgoHelper() - out, _, err := h.Cli.Execute(ctx, "-n", namespace, "logs", name) + out, err := h.Cli.GetContainerLogs(ctx, namespace, name) Expect(err).ToNot(HaveOccurred()) return out } diff --git a/test/kube2e/helper/testserver.go b/test/kube2e/helper/testserver.go index fe94329b5fa..d86c4dcaea8 100644 --- a/test/kube2e/helper/testserver.go +++ b/test/kube2e/helper/testserver.go @@ -80,6 +80,8 @@ const ( ` ) +// tests relying on the test server should be ported using the default nginx deployment located at +// test/kubernetes/e2e/defaults/testdata/nginx_pod.yaml func NewTestServer(namespace string) (TestUpstreamServer, error) { testContainer, err := newTestContainer(namespace, defaultTestServerImage, TestServerName, TestServerPort) if err != nil { diff --git a/test/kube2e/util.go b/test/kube2e/util.go index 68d96f5ccd2..0c1cda49947 100644 --- a/test/kube2e/util.go +++ b/test/kube2e/util.go @@ -11,6 +11,7 @@ import ( "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" errors "github.com/rotisserie/eris" + "github.com/solo-io/gloo/pkg/utils/fsutils" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" clienthelpers "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" @@ -152,13 +153,10 @@ func UpdateSettingsWithPropagationDelay(updateSettings func(settings *v1.Setting } func ToFile(content string) string { - f, err := os.CreateTemp("", "") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - n, err := f.WriteString(content) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - ExpectWithOffset(1, n).To(Equal(len(content))) - _ = f.Close() - return f.Name() + fname, err := fsutils.ToTempFile(content) + Expect(err).ToNot(HaveOccurred()) + + return fname } // https://github.com/solo-io/gloo/issues/4043#issuecomment-772706604 diff --git a/test/kubernetes/e2e/README.md b/test/kubernetes/e2e/README.md index 20e856f068c..18df3f02f7f 100644 --- a/test/kubernetes/e2e/README.md +++ b/test/kubernetes/e2e/README.md @@ -18,8 +18,10 @@ We define all tests in the [features](./features) package. This is done for a va 1. We group the tests by feature, so it's easy to identify which behaviors we assert for a given feature. 2. We can invoke that same test against different `TestInstallation`s. This means we can test a feature against a variety of installation values, or even against OSS and Enterprise installations. +Many examples of testing features may be found in the [features](./features) package. The general pattern for adding a new feature should be to create a directory for the feature under `features/`, write manifest files for the resources the tests will need into `features/my_feature/testdata/`, define Go objects for them in a file called `features/my_feature/types.go`, and finally define the test suite in `features/my_feature/suite.go`. There are occasions where multiple suites will need to be created under a single feature. See [Suites](#test-suites) for more info on this case. + ## Test Suites -A Test Suite is a subset of the Feature concept. A single Feature has at minimum one Test Suite, and can have many. Each Test Suite within the feature must have a function which satisfies the signature `NewSuiteFunc` found in [suite.go](./suite.go). +A Test Suite is a subset of the Feature concept. A single Feature has at minimum one Test Suite, and can have many. Each Test Suite should have its own appropriately named `.go` file from which is exported an appropriately named function which satisfies the signature `NewSuiteFunc` found in [suite.go](./suite.go). These test suites are registered by a name and this func in [Tests](#tests) to be run against various `TestInstallation`s. diff --git a/test/kubernetes/e2e/defaults/defaults.go b/test/kubernetes/e2e/defaults/defaults.go index 666cd40e875..3dddaddc5e0 100644 --- a/test/kubernetes/e2e/defaults/defaults.go +++ b/test/kubernetes/e2e/defaults/defaults.go @@ -1,10 +1,13 @@ package defaults import ( + "path/filepath" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + "github.com/solo-io/skv2/codegen/util" ) var ( @@ -20,4 +23,57 @@ var ( Namespace: "curl", }, } + + CurlPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "curl_pod.yaml") + + HttpEchoPod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "http-echo", + Namespace: "http-echo", + }, + } + + HttpEchoPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "http_echo.yaml") + + TcpEchoPod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-echo", + Namespace: "tcp-echo", + }, + } + + TcpEchoPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "tcp_echo.yaml") + + NginxPod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "nginx", + }, + } + + NginxPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "nginx_pod.yaml") + + NginxResponse = ` + + +Welcome to nginx! + + + +

Welcome to nginx!

+

If you see this page, the nginx web server is successfully installed and +working. Further configuration is required.

+ +

For online documentation and support please refer to +nginx.org.
+Commercial support is available at +nginx.com.

+ +

Thank you for using nginx.

+ +` ) diff --git a/test/kubernetes/e2e/defaults/testdata/http_echo.yaml b/test/kubernetes/e2e/defaults/testdata/http_echo.yaml new file mode 100644 index 00000000000..62e43fbad4d --- /dev/null +++ b/test/kubernetes/e2e/defaults/testdata/http_echo.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: http-echo +--- +apiVersion: v1 +kind: Service +metadata: + name: http-echo + namespace: http-echo +spec: + selector: + app.kubernetes.io/name: http-echo + ports: + - protocol: TCP + port: 3000 +--- +apiVersion: v1 +kind: Pod +metadata: + name: http-echo + namespace: http-echo + labels: + app.kubernetes.io/name: http-echo +spec: + containers: + - name: http-echo + image: kennship/http-echo@sha256:144322e8e96be2be6675dcf6e3ee15697c5d052d14d240e8914871a2a83990af + ports: + - containerPort: 3000 + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" diff --git a/test/kubernetes/e2e/defaults/testdata/nginx_pod.yaml b/test/kubernetes/e2e/defaults/testdata/nginx_pod.yaml new file mode 100644 index 00000000000..393413a1f36 --- /dev/null +++ b/test/kubernetes/e2e/defaults/testdata/nginx_pod.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nginx +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + namespace: nginx +spec: + selector: + app.kubernetes.io/name: nginx + ports: + - protocol: TCP + port: 8080 + targetPort: http-web-svc + name: http + - protocol: TCP + port: 8443 + targetPort: https-web-svc + name: https +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf + namespace: nginx +data: + nginx.conf: | + user nginx; + worker_processes 1; + events { + worker_connections 10240; + } + http { + server { + listen 80; + listen 443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/localhost.crt; + ssl_certificate_key /etc/nginx/localhost.key; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + } + } + # localhost cert and key generated with following command from https://letsencrypt.org/docs/certificates-for-localhost/ + # openssl req -x509 -out localhost.crt -keyout localhost.key -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' -extensions EXT -config <(printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") + localhost.crt: | + -----BEGIN CERTIFICATE----- + MIIDDzCCAfegAwIBAgIUMrxrG4aI7TShlLeuu4tmIsTIO1gwDQYJKoZIhvcNAQEL + BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDcwMjIxNDYzNVoXDTI0MDgw + MTIxNDYzNVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF + AAOCAQ8AMIIBCgKCAQEAw6lxj3IX6kBNKWF0LEQiONJN81vbKNDEVpE+w5zwaA1K + zAMSfKxkhdQMPtM+MS64CPkDUZFxdYbUgKygl23uWcuIPWHnD7aqICm+ujMLUMzw + RFXablUCZiO9sFfZegkdLwvecmtnvNjVL5s8jk3HjV8Jetu/tE17HMvP4cMdfs/r + zdYRxoI2tWyYDWUW1XfD6eTxDykWwfLMdJ6UX0ksZSlQ098OheMMA6E+cxH0JMoe + +PLyD4nuAYW8c6tOFTXJjqHUaJzSRlYFg3OG0WRWKcjP9ufeLsPWjWza5M6WSGEj + hiPP2bSxMCfkY3DFSO3K71MrYf5xsP4L70YmD2oUowIDAQABo1kwVzAUBgNVHREE + DTALgglsb2NhbGhvc3QwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMB + MB0GA1UdDgQWBBSC36i8yBxdER9rU3KJvR/Dtop9XTANBgkqhkiG9w0BAQsFAAOC + AQEAXG1LQfCVJZ3R2rXHZirUHkSgXCPMglUv9dN25XMvGiOwVGX8g9QUv+WuMeoS + VK98rLnej7EOLZLb+02lXKqAT8G6eDqXlZONfFCTnS6BWc0+5o2fvnniuJhzxGq0 + qolf2q4P4JNJk7TRlulaLdIxSOMyJukRne4kRcbkz3SaVE+eGAm6IURSEE1x1AXU + BaCZpm5MWgiJtOJulM7/9Nw8SpTir3nKNTcI3Q0M2XGvhWylN9N17AkANrBrNBme + LDyhBvUlZrbnOxfblBxzB8jocGxCLDLLtNNlfuEPquy8J263LVIh+Totibxhg4l/ + MaeHQu7bsnjxU6pWF3x/QsZYzQ== + -----END CERTIFICATE----- + localhost.key: | + -----BEGIN PRIVATE KEY----- + MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDDqXGPchfqQE0p + YXQsRCI40k3zW9so0MRWkT7DnPBoDUrMAxJ8rGSF1Aw+0z4xLrgI+QNRkXF1htSA + rKCXbe5Zy4g9YecPtqogKb66MwtQzPBEVdpuVQJmI72wV9l6CR0vC95ya2e82NUv + mzyOTceNXwl627+0TXscy8/hwx1+z+vN1hHGgja1bJgNZRbVd8Pp5PEPKRbB8sx0 + npRfSSxlKVDT3w6F4wwDoT5zEfQkyh748vIPie4Bhbxzq04VNcmOodRonNJGVgWD + c4bRZFYpyM/2594uw9aNbNrkzpZIYSOGI8/ZtLEwJ+RjcMVI7crvUyth/nGw/gvv + RiYPahSjAgMBAAECggEAPgk8WznWgUwv6t3eQqa7nv52/qDyJMfEaJoWp1jcFHGa + dILI2sSh/piT5Vt+zYM9kIK7XaJaseO0/rM9G6YcO1Y/9Q5Uf3PwDtCNi2XhwZWo + 3FHINtE3OIE+hGnmJz46hY8i0W5ibsrlFPoIQipBCf7G97Ay4Qzr6t5oI/GEKY9e + 9b3/CJ12cvpjrrEnvvgcQv9XJ83dYVpvBLL8Hm9WjwFsmV1erN3JXB+yHe8b3MHK + BGA5rM+bpn+LxV3b7ycnfLXjPVkKY9KXBQCyZDzsGaCfzmQ7dJqIJAdtjR320A3A + M3RU/YCfV7lVxuCuPK59N3xhJJFDnfrikkKLWuGYmQKBgQDX2/iMnvPKjgcgYyui + Gol7h8VqRpQycXHUS89zCeIhZPr6i5ZjzltblOFhsSE26UP6S9QXG8Hw7v6dEKDx + 9zVgNK75e1vzr8Gm5Ld+eqo22PQR1RHeQkwJXgXL+3GfHxFes6TG2h9MR3TNBGJU + EOayfzeNT6l3JSUV/ae913ognQKBgQDoC/jiytQz8auV4p2c/aiMRziL+xgJkV8f + E7q0pw4BZirRf6N6YdNU/mOhgJd1LHkcgYdnPbxHXIhNUUE3uS9Uvc+exujh4R3a + FcmUcbEjBG/yRVFDtgmszQtA+U+1upu4JG5MhuzLizGZvYWD0VrpGrKIBOwxlvEW + zMFp0JvmPwKBgCBuBdtqjgniaKOvAoEqJ3mNnlUnIWCqtoVElngcBgMqXqKBkiiQ + eh06MtoweGL9jJ7wAX8vRmXiIhKKywNPNo+rmpYUuG3V++wM9Jxl5Wi0E4cSUcro + fu/xVkGdFybmzf9CUgEmCAm3uo6KmBM1LtOmVTw/uaASzo2NPERDOS/pAoGAWpwq + MK0RFcN9xAZ8k0v9n+FDtG11Im9QnHsAwgAlmOhDOhFETcqbUioPz4W+HrQiCr6N + mAPkXF1GoCJlfBPk5otD4nU7hNB57qnpT/zhNZJLAGiO5gjUWFSs208/D/BxVANt + ypY5KvYMhUMbOrDqdfHF2xVJAcg2FjgYInCiH9MCgYAYGrAdT7gDLjjNtSuoFur7 + sMtACf59rw24my1UoWCjVOxE1MVWRtCYDV5Qi+7MQ4xNejugKKKNiRwrBa6Z6bCm + JnPicvcV8IaWwKQqHBjDQn47DVQ+xXNHara6Alv/kjpyJztXsh12S7Hn2EJY91// + hcbfh0ySbbsqheBkTXMrvQ== + -----END PRIVATE KEY----- +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx + namespace: nginx + labels: + app.kubernetes.io/name: nginx +spec: + containers: + - name: nginx + image: nginx:stable + ports: + - containerPort: 80 + name: http-web-svc + - containerPort: 443 + name: https-web-svc + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/ + readOnly: true + volumes: + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: nginx.conf + - key: localhost.crt + path: localhost.crt + - key: localhost.key + path: localhost.key diff --git a/test/kubernetes/e2e/defaults/testdata/tcp_echo.yaml b/test/kubernetes/e2e/defaults/testdata/tcp_echo.yaml new file mode 100644 index 00000000000..c68594b8e86 --- /dev/null +++ b/test/kubernetes/e2e/defaults/testdata/tcp_echo.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: tcp-echo +--- +apiVersion: v1 +kind: Service +metadata: + name: tcp-echo + namespace: tcp-echo +spec: + selector: + app.kubernetes.io/name: tcp-echo + ports: + - protocol: TCP + port: 1025 +--- +apiVersion: v1 +kind: Pod +metadata: + name: tcp-echo + namespace: tcp-echo + labels: + app.kubernetes.io/name: tcp-echo +spec: + containers: + - name: tcp-echo + image: soloio/tcp-echo:latest + ports: + - containerPort: 1025 + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" + diff --git a/test/kubernetes/e2e/example/debug_logging_test.go b/test/kubernetes/e2e/example/debug_logging_test.go index e5180091a30..b644bb331eb 100644 --- a/test/kubernetes/e2e/example/debug_logging_test.go +++ b/test/kubernetes/e2e/example/debug_logging_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" "github.com/stretchr/testify/suite" @@ -44,7 +44,7 @@ func TestInstallationWithDebugLogLevel(t *testing.T) { }) testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) // The name here is important for debuggability diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 4c0b5af5757..4c5bd42567b 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -8,12 +8,11 @@ import ( "runtime" "testing" - "github.com/solo-io/gloo/test/kube2e" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/testutils/actions" "github.com/solo-io/gloo/test/kubernetes/testutils/assertions" "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" "github.com/solo-io/gloo/test/testutils" ) @@ -22,12 +21,12 @@ import ( // The SoloTestHelper is a wrapper around `glooctl` and we should eventually phase it out // in favor of using the exact tool that users rely on func MustTestHelper(ctx context.Context, installation *TestInstallation) *helper.SoloTestHelper { - testHelper, err := kube2e.GetTestHelperForRootDir(ctx, testutils.GitRootDirectory(), installation.Metadata.InstallNamespace) + testHelper, err := helper.GetTestHelperForRootDir(ctx, testutils.GitRootDirectory(), installation.Metadata.InstallNamespace) if err != nil { panic(err) } - testHelper.DeployTestServer = false + testHelper.SetKubeCli(installation.ClusterContext.Cli) return testHelper } @@ -155,6 +154,7 @@ func (i *TestInstallation) InstallGlooGateway(ctx context.Context, installFn fun err := installFn(ctx) i.Assertions.Require.NoError(err) i.Assertions.EventuallyInstallationSucceeded(ctx) + i.Assertions.EventuallyGlooReachesConsistentState(i.Metadata.InstallNamespace) } // We can only create the ResourceClients after the CRDs exist in the Cluster @@ -172,6 +172,13 @@ func (i *TestInstallation) UninstallGlooGateway(ctx context.Context, uninstallFn i.Assertions.EventuallyUninstallationSucceeded(ctx) } +func (i *TestInstallation) UpgradeGlooGateway(ctx context.Context, serverVersion string, upgradeFn func(ctx context.Context) error) { + err := upgradeFn(ctx) + i.Assertions.Require.NoError(err) + i.Assertions.EventuallyUpgradeSucceeded(ctx, serverVersion) + i.Assertions.EventuallyGlooReachesConsistentState(i.Metadata.InstallNamespace) +} + // PreFailHandler is the function that is invoked if a test in the given TestInstallation fails func (i *TestInstallation) PreFailHandler(ctx context.Context) { // This is a work in progress diff --git a/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go b/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go index 057979f7b90..7f56312a39a 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go +++ b/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" @@ -62,7 +62,7 @@ func TestAutomtlsIstioEdgeApisGateway(t *testing.T) { // Install Gloo Gateway with only Gloo Edge Gateway APIs enabled testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) AutomtlsIstioEdgeApiSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/automtls_istio_test.go b/test/kubernetes/e2e/tests/automtls_istio_test.go index fd9f5862b9d..1e2c7a67fb5 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_test.go +++ b/test/kubernetes/e2e/tests/automtls_istio_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" ) @@ -64,7 +64,7 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) { // Install Gloo Gateway testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { // istio proxy and sds are added to gateway and take a little longer to start up - return testHelper.InstallGloo(ctx, helper.GATEWAY, 10*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 10*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) AutomtlsIstioSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/edge_gw_test.go b/test/kubernetes/e2e/tests/edge_gw_test.go index b713d61284e..9dba4068704 100644 --- a/test/kubernetes/e2e/tests/edge_gw_test.go +++ b/test/kubernetes/e2e/tests/edge_gw_test.go @@ -6,10 +6,10 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" ) @@ -41,7 +41,7 @@ func TestGlooGatewayEdgeGateway(t *testing.T) { // Install Gloo Gateway with only Gloo Edge Gateway APIs enabled testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) EdgeGwSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go b/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go index 724a877e145..8bbeabf09e4 100644 --- a/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go +++ b/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" "github.com/solo-io/gloo/test/kubernetes/e2e" @@ -61,7 +61,7 @@ func TestGlooctlIstioInjectEdgeApiGateway(t *testing.T) { // Install Gloo Gateway with only Gloo Edge Gateway APIs enabled testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) GlooctlIstioInjectSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/istio_edge_api_test.go b/test/kubernetes/e2e/tests/istio_edge_api_test.go index 9cab29e889a..a462c2ea639 100644 --- a/test/kubernetes/e2e/tests/istio_edge_api_test.go +++ b/test/kubernetes/e2e/tests/istio_edge_api_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" ) @@ -66,7 +66,7 @@ func TestIstioEdgeApiGateway(t *testing.T) { // Install Gloo Gateway with only Edge APIs enabled testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) IstioEdgeApiSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/istio_regression_test.go b/test/kubernetes/e2e/tests/istio_regression_test.go index 11ad0a7f3da..7bfd609555d 100644 --- a/test/kubernetes/e2e/tests/istio_regression_test.go +++ b/test/kubernetes/e2e/tests/istio_regression_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" ) @@ -66,7 +66,7 @@ func TestIstioRegression(t *testing.T) { // Install Gloo Gateway with only Edge APIs enabled testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) IstioRegressionSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/k8s_gw_istio_test.go b/test/kubernetes/e2e/tests/k8s_gw_istio_test.go index 4bc587ad2f9..fefb6a8db3c 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_istio_test.go +++ b/test/kubernetes/e2e/tests/k8s_gw_istio_test.go @@ -6,10 +6,10 @@ import ( "testing" "time" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/skv2/codegen/util" ) @@ -60,7 +60,7 @@ func TestK8sGatewayIstio(t *testing.T) { // Install Gloo Gateway testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { // istio proxy and sds are added to gateway and take a little longer to start up - return testHelper.InstallGloo(ctx, helper.GATEWAY, 10*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 10*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) IstioSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/k8s_gw_minimal_default_gatewayparameters_test.go b/test/kubernetes/e2e/tests/k8s_gw_minimal_default_gatewayparameters_test.go index d899c37855b..80e99524a73 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_minimal_default_gatewayparameters_test.go +++ b/test/kubernetes/e2e/tests/k8s_gw_minimal_default_gatewayparameters_test.go @@ -7,10 +7,10 @@ import ( "time" "github.com/solo-io/gloo/pkg/utils/env" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/gloo/test/testutils" "github.com/solo-io/skv2/codegen/util" ) @@ -44,7 +44,7 @@ func TestK8sGatewayMinimalDefaultGatewayParameters(t *testing.T) { // Install Gloo Gateway testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) KubeGatewayMinimalDefaultGatewayParametersSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/k8s_gw_no_validation_test.go b/test/kubernetes/e2e/tests/k8s_gw_no_validation_test.go index 2810145b1c0..ce28e22f93e 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_no_validation_test.go +++ b/test/kubernetes/e2e/tests/k8s_gw_no_validation_test.go @@ -8,10 +8,10 @@ import ( "github.com/solo-io/skv2/codegen/util" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" ) // TestK8sGatewayNoValidation executes tests against a K8s Gateway gloo install with validation disabled @@ -42,7 +42,7 @@ func TestK8sGatewayNoValidation(t *testing.T) { // Install Gloo Gateway testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) KubeGatewayNoValidationSuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/e2e/tests/k8s_gw_test.go b/test/kubernetes/e2e/tests/k8s_gw_test.go index c2771c27314..afc3942bb89 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_test.go +++ b/test/kubernetes/e2e/tests/k8s_gw_test.go @@ -7,10 +7,10 @@ import ( "time" "github.com/solo-io/gloo/pkg/utils/env" - "github.com/solo-io/gloo/test/kube2e/helper" "github.com/solo-io/gloo/test/kubernetes/e2e" . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/gloo/test/testutils" "github.com/solo-io/skv2/codegen/util" ) @@ -43,7 +43,7 @@ func TestK8sGateway(t *testing.T) { // Install Gloo Gateway testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { - return testHelper.InstallGloo(ctx, helper.GATEWAY, 5*time.Minute, helper.ExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) }) KubeGatewaySuiteRunner().Run(ctx, t, testInstallation) diff --git a/test/kubernetes/testutils/assertions/deployments.go b/test/kubernetes/testutils/assertions/deployments.go index 144ea0bbb51..e94cf010668 100644 --- a/test/kubernetes/testutils/assertions/deployments.go +++ b/test/kubernetes/testutils/assertions/deployments.go @@ -4,11 +4,15 @@ import ( "context" "time" + "github.com/solo-io/gloo/pkg/utils/kubeutils" + "github.com/solo-io/gloo/test/gomega/assertions" + + "github.com/solo-io/go-utils/stats" + . "github.com/onsi/gomega" "github.com/onsi/gomega/types" + "go.uber.org/zap/zapcore" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/solo-io/gloo/pkg/utils/kubeutils" ) func (p *Provider) EventuallyRunningReplicas(ctx context.Context, deploymentMeta metav1.ObjectMeta, replicaMatcher types.GomegaMatcher) { @@ -25,3 +29,28 @@ func (p *Provider) EventuallyRunningReplicas(ctx context.Context, deploymentMeta WithPolling(time.Millisecond * 200). Should(Succeed()) } + +func (p *Provider) EventuallyGlooReachesConsistentState(installNamespace string) { + // We port-forward the Gloo deployment stats port to inspect the metrics and log settings + // TODO(jbohanon) clean this up with newer style portforwarder + glooStatsForwardConfig := assertions.StatsPortFwd{ + ResourceName: "deployment/gloo", + ResourceNamespace: installNamespace, + LocalPort: stats.DefaultPort, + TargetPort: stats.DefaultPort, + } + + // Gloo components are configured to log to the Info level by default + logLevelAssertion := assertions.LogLevelAssertion(zapcore.InfoLevel) + + // The emitter at some point should stabilize and not continue to increase the number of snapshots produced + // We choose 4 here as a bit of a magic number, but we feel comfortable that if 4 consecutive polls of the metrics + // endpoint returns that same value, then we have stabilized + identicalResultInARow := 4 + emitterMetricAssertion, _ := assertions.IntStatisticReachesConsistentValueAssertion("api_gloosnapshot_gloo_solo_io_emitter_snap_out", identicalResultInARow) + + assertions.EventuallyStatisticsMatchAssertions(glooStatsForwardConfig, + logLevelAssertion, + emitterMetricAssertion, + ) +} diff --git a/test/kubernetes/testutils/assertions/glooctl.go b/test/kubernetes/testutils/assertions/glooctl.go index f1cdbae9a78..89e8acd3900 100644 --- a/test/kubernetes/testutils/assertions/glooctl.go +++ b/test/kubernetes/testutils/assertions/glooctl.go @@ -9,6 +9,7 @@ import ( "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/version" "github.com/solo-io/gloo/projects/gloo/cli/pkg/printers" ) @@ -39,6 +40,31 @@ func (p *Provider) EventuallyCheckResourcesOk(ctx context.Context) { Should(Succeed()) } +// EventuallyMatchesVersion asserts that `glooctl version` eventually responds with the expected server version +func (p *Provider) EventuallyMatchesVersion(ctx context.Context, serverVersion string) { + p.expectGlooGatewayContextDefined() + + k := version.NewKube(p.glooGatewayContext.InstallNamespace, p.clusterContext.KubeContext) + + p.Gomega.Eventually(func(innerG Gomega) { + contextWithCancel, cancel := context.WithCancel(ctx) + defer cancel() + csv, err := version.GetClientServerVersions(contextWithCancel, k) + innerG.Expect(err).NotTo(HaveOccurred(), "can get client server versions with glooctl") + innerG.Expect(csv.GetServer()).To(HaveLen(1), "has detected gloo deployment") + kServer := csv.GetServer()[0].GetKubernetes() + innerG.Expect(kServer.GetContainers()).ToNot(BeEmpty(), "has containers for gloo deployment") + innerG.Expect(kServer.GetContainers()[0].GetTag()).To(Equal(serverVersion), "has expected tag") + }). + WithContext(ctx). + // These are some basic defaults that we expect to work in most cases + // We can make these configurable if need be, though most installations + // Should be able to become healthy within this window + WithTimeout(time.Second * 120). + WithPolling(time.Second). + Should(Succeed()) +} + func (p *Provider) EventuallyInstallationSucceeded(ctx context.Context) { p.expectGlooGatewayContextDefined() @@ -51,3 +77,12 @@ func (p *Provider) EventuallyUninstallationSucceeded(ctx context.Context) { p.ExpectNamespaceNotExist(ctx, p.glooGatewayContext.InstallNamespace) } + +func (p *Provider) EventuallyUpgradeSucceeded(ctx context.Context, version string) { + p.expectGlooGatewayContextDefined() + + p.EventuallyMatchesVersion(ctx, version) + + // Check that everything is OK + p.EventuallyCheckResourcesOk(ctx) +} diff --git a/test/kubernetes/testutils/helper/docker/Dockerfile b/test/kubernetes/testutils/helper/docker/Dockerfile new file mode 100644 index 00000000000..1d91b98fff1 --- /dev/null +++ b/test/kubernetes/testutils/helper/docker/Dockerfile @@ -0,0 +1,10 @@ +# This file is used to build soloio/tcp-echo (https://hub.docker.com/r/soloio/tcp-echo) +# which has not received updates since 2019 +FROM cjimti/go-echo:latest + +RUN apk update +RUN apk add curl + +WORKDIR / + +ENTRYPOINT ["/tcp-echo"] diff --git a/test/kubernetes/testutils/helper/install.go b/test/kubernetes/testutils/helper/install.go new file mode 100644 index 00000000000..8dfe22a7732 --- /dev/null +++ b/test/kubernetes/testutils/helper/install.go @@ -0,0 +1,404 @@ +package helper + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "helm.sh/helm/v3/pkg/repo" + + "github.com/pkg/errors" + "github.com/spf13/afero" + + "github.com/rotisserie/eris" + "github.com/solo-io/gloo/pkg/utils/fsutils" + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + "github.com/solo-io/go-utils/log" + "github.com/solo-io/go-utils/testutils/exec" +) + +// Default test configuration +var defaults = TestConfig{ + TestAssetDir: "_test", + BuildAssetDir: "_output", + HelmRepoIndexFileName: "index.yaml", +} + +// supportedArchs is represents the list of architectures we build glooctl for +var supportedArchs = map[string]struct{}{ + "arm64": {}, + "amd64": {}, +} + +// returns true if supported, based on `supportedArchs` +func isSupportedArch() (string, bool) { + if goarch, ok := os.LookupEnv("GOARCH"); ok { + // if the environment's goarch is supported + _, ok := supportedArchs[goarch] + return goarch, ok + } + + // if the runtime's goarch is supported + runtimeArch := runtime.GOARCH + _, ok := supportedArchs[runtimeArch] + return runtimeArch, ok +} + +// Function to provide/override test configuration. Default values will be passed in. +type TestConfigFunc func(defaults TestConfig) TestConfig + +type TestConfig struct { + // All relative paths will assume this as the base directory. This is usually the project base directory. + RootDir string + // The directory holding the test assets. Must be relative to RootDir. + TestAssetDir string + // The directory holding the build assets. Must be relative to RootDir. + BuildAssetDir string + // Helm chart name + HelmChartName string + // Name of the helm index file name + HelmRepoIndexFileName string + // The namespace gloo (and the test server) will be installed to. If empty, will use the helm chart version. + InstallNamespace string + // Name of the glooctl executable + GlooctlExecName string + // If provided, the licence key to install the enterprise version of Gloo + LicenseKey string + // Install a released version of gloo. This is the value of the github tag that may have a leading 'v' + ReleasedVersion string + // If true, glooctl will be run with a -v flag + Verbose bool + + // The version of the Helm chart. Calculated from either the chart or the released version. It will not have a leading 'v' + version string +} + +type SoloTestHelper struct { + *TestConfig + // The kubernetes helper + *kubectl.Cli +} + +// NewSoloTestHelper is meant to provide a standard way of deploying Gloo/GlooE to a k8s cluster during tests. +// It assumes that build and test assets are present in the `_output` and `_test` directories (these are configurable). +// Specifically, it expects the glooctl executable in the BuildAssetDir and a helm chart in TestAssetDir. +// It also assumes that a kubectl executable is on the PATH. +func NewSoloTestHelper(configFunc TestConfigFunc) (*SoloTestHelper, error) { + + // Get and validate test config + testConfig := defaults + if configFunc != nil { + testConfig = configFunc(defaults) + } + // Depending on the testing tool used, GOARCH may always be set if not set already by detecting the local arch + // (`go test`), `ginkgo` and other testing tools may not do this requiring keeping the runtime.GOARCH check + if testConfig.GlooctlExecName == "" { + if arch, ok := isSupportedArch(); ok { + testConfig.GlooctlExecName = "glooctl-" + runtime.GOOS + "-" + arch + } else { + testConfig.GlooctlExecName = "glooctl-" + runtime.GOOS + "-amd64" + } + } + + // Get chart version + if testConfig.ReleasedVersion == "" { + if err := validateConfig(testConfig); err != nil { + return nil, errors.Wrapf(err, "test config validation failed") + } + version, err := getChartVersion(testConfig) + if err != nil { + return nil, errors.Wrapf(err, "getting Helm chart version") + } + testConfig.version = version + } else { + // we use the version field as a chart version and tests assume it doesn't have a leading 'v' + if testConfig.ReleasedVersion[0] == 'v' { + testConfig.version = testConfig.ReleasedVersion[1:] + } else { + testConfig.version = testConfig.ReleasedVersion + } + } + // Default the install namespace to the chart version. + // Currently the test chart version built in CI contains the build id, so the namespace will be unique). + if testConfig.InstallNamespace == "" { + testConfig.InstallNamespace = testConfig.version + } + + testHelper := &SoloTestHelper{ + TestConfig: &testConfig, + } + + return testHelper, nil +} + +func (h *SoloTestHelper) SetKubeCli(cli *kubectl.Cli) { + h.Cli = cli +} + +// Return the version of the Helm chart +func (h *SoloTestHelper) ChartVersion() string { + return h.version +} + +// Return the path to the chart used for installation +func (h *SoloTestHelper) ChartPath() string { + return filepath.Join(h.RootDir, h.TestAssetDir, fmt.Sprintf("%s-%s.tgz", h.HelmChartName, h.ChartVersion())) +} + +type OptionsMutator func(opts *Options) + +type Options struct { + Command []string + Verbose bool +} + +func WithExtraArgs(args ...string) OptionsMutator { + return func(opts *Options) { + opts.Command = append(opts.Command, args...) + } +} + +// InstallGloo calls glooctl to install Gloo. This is where image variants are handled as well. +func (h *SoloTestHelper) InstallGloo(ctx context.Context, timeout time.Duration, options ...OptionsMutator) error { + deploymentType := "gateway" + log.Printf("installing gloo in [%s] mode to namespace [%s]", deploymentType, h.InstallNamespace) + glooctlCommand := []string{ + filepath.Join(h.BuildAssetDir, h.GlooctlExecName), + "install", deploymentType, + } + if h.LicenseKey != "" { + glooctlCommand = append(glooctlCommand, "enterprise", "--license-key", h.LicenseKey) + } + if h.ReleasedVersion != "" { + glooctlCommand = append(glooctlCommand, "-n", h.InstallNamespace, "--version", h.ReleasedVersion) + } else { + glooctlCommand = append(glooctlCommand, + "-n", h.InstallNamespace, + "-f", h.ChartPath()) + } + if h.Verbose { + glooctlCommand = append(glooctlCommand, "-v") + } + variant := os.Getenv("IMAGE_VARIANT") + if variant != "" { + variantValuesFile, err := GenerateVariantValuesFile(variant) + if err != nil { + return err + } + options = append(options, WithExtraArgs("--values", variantValuesFile)) + } + + io := &Options{ + Command: glooctlCommand, + Verbose: true, + } + for _, opt := range options { + opt(io) + } + + if err := glooctlInstallWithTimeout(h.RootDir, io, timeout); err != nil { + return errors.Wrapf(err, "error running glooctl install command") + } + + return nil +} + +func runWithTimeout(rootDir string, opts *Options, timeout time.Duration, operation string) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + errChan := make(chan error, 1) + go func() { + err := exec.RunCommand(rootDir, opts.Verbose, opts.Command...) + if err != nil { + errChan <- errors.Wrapf(err, "error while executing gloo %s", operation) + } + errChan <- nil + }() + + select { + case err := <-errChan: + return err // can be nil + case <-ctx.Done(): + return fmt.Errorf("timed out while executing gloo %s", operation) + } +} + +// Wait for the glooctl install command to respond, err on timeout. +// The command returns as soon as certgen completes and all other +// deployments have been applied, which should only be delayed if +// there's an issue pulling the certgen docker image. +// Without this timeout, it would just hang indefinitely. +func glooctlInstallWithTimeout(rootDir string, opts *Options, timeout time.Duration) error { + return runWithTimeout(rootDir, opts, timeout, "install") +} + +// Upgrades Gloo via a helm upgrade. It returns a method that rolls-back helm to the version prior to this upgrade +func (h *SoloTestHelper) UpgradeGloo(ctx context.Context, timeout time.Duration, options ...OptionsMutator) (revertFunc func() error, err error) { + log.Printf("upgrading gloo in namespace [%s]", h.InstallNamespace) + + revision, err := h.CurrentGlooRevision() + if err != nil { + return nil, err + } + + helmCommand := []string{ + "helm", + "upgrade", + h.HelmChartName, + h.ChartPath(), + "-n", h.InstallNamespace, + } + + if h.Verbose { + helmCommand = append(helmCommand, "-v") + } + + opts := &Options{ + Command: helmCommand, + Verbose: true, + } + for _, opt := range options { + opt(opts) + } + + if err := upgradeGlooWithTimeout(h.RootDir, opts, timeout); err != nil { + return nil, errors.Wrapf(err, "error running glooctl install command") + } + + return func() error { + return h.RevertGlooUpgrade(ctx, timeout, WithExtraArgs([]string{ + strconv.Itoa(revision), + }...)) + }, nil +} + +func (h *SoloTestHelper) CurrentGlooRevision() (int, error) { + command := []string{ + "bash", + "-c", + fmt.Sprintf("helm -n %s ls -o json | jq '.[] | select(.name=\"%s\") | .revision' | tr -d '\"'", h.InstallNamespace, h.HelmChartName), + } + out, err := exec.RunCommandOutput(h.RootDir, false, command...) + if err != nil { + return 0, errors.Wrapf(err, "error while fetching gloo revision") + } + return strconv.Atoi(strings.TrimSpace(out)) +} + +func upgradeGlooWithTimeout(rootDir string, opts *Options, timeout time.Duration) error { + return runWithTimeout(rootDir, opts, timeout, "upgrade") +} + +// Rollback Gloo. The version can be passed via the ExtraArgs option. If not specified it rolls-back to the previous version +// Eg: RevertGlooUpgrade(ctx, timeout, WithExtraArgs([]string{revision})) +func (h *SoloTestHelper) RevertGlooUpgrade(ctx context.Context, timeout time.Duration, options ...OptionsMutator) error { + log.Printf("reverting gloo upgrade in namespace [%s]", h.InstallNamespace) + helmCommand := []string{ + "helm", + "rollback", + h.HelmChartName, + "-n", h.InstallNamespace, + } + + if h.Verbose { + helmCommand = append(helmCommand, "-v") + } + + opts := &Options{ + Command: helmCommand, + Verbose: true, + } + for _, opt := range options { + opt(opts) + } + + if err := upgradeGlooWithTimeout(h.RootDir, opts, timeout); err != nil { + return errors.Wrapf(err, "error running glooctl install command") + } + return nil +} + +// passes the --all flag to glooctl uninstall +func (h *SoloTestHelper) UninstallGlooAll() error { + return h.uninstallGloo(true) +} + +// does not pass the --all flag to glooctl uninstall +func (h *SoloTestHelper) UninstallGloo() error { + return h.uninstallGloo(false) +} + +func (h *SoloTestHelper) uninstallGloo(all bool) error { + log.Printf("uninstalling gloo...") + cmdArgs := []string{ + filepath.Join(h.BuildAssetDir, h.GlooctlExecName), "uninstall", "-n", h.InstallNamespace, "--delete-namespace", "--release-name", h.HelmChartName, + } + if all { + cmdArgs = append(cmdArgs, "--all") + } + return exec.RunCommand(h.RootDir, true, cmdArgs...) +} + +// Parses the Helm index file and returns the version of the chart. +func getChartVersion(config TestConfig) (string, error) { + + // Find helm index file in test asset directory + helmIndexFile := filepath.Join(config.RootDir, config.TestAssetDir, config.HelmRepoIndexFileName) + helmIndex, err := repo.LoadIndexFile(helmIndexFile) + if err != nil { + return "", errors.Wrapf(err, "parsing Helm index file") + } + log.Printf("found Helm index file at: %s", helmIndexFile) + + // Read and return version from helm index file + if chartVersions, ok := helmIndex.Entries[config.HelmChartName]; !ok { + return "", eris.Errorf("index file does not contain entry with key: %s", config.HelmChartName) + } else if len(chartVersions) == 0 || len(chartVersions) > 1 { + return "", eris.Errorf("expected a single entry with name [%s], found: %v", config.HelmChartName, len(chartVersions)) + } else { + version := chartVersions[0].Version + log.Printf("version of [%s] Helm chart is: %s", config.HelmChartName, version) + return version, nil + } +} + +func validateConfig(config TestConfig) error { + for _, dirName := range []string{ + config.RootDir, + filepath.Join(config.RootDir, config.BuildAssetDir), + filepath.Join(config.RootDir, config.TestAssetDir), + } { + if !fsutils.IsDirectory(dirName) { + return fmt.Errorf("%s does not exist or is not a directory", dirName) + } + } + return nil +} + +func GenerateVariantValuesFile(variant string) (string, error) { + content := `global: + image: + variant: ` + variant + + fs := afero.NewOsFs() + dir, err := afero.TempDir(fs, "", "") + if err != nil { + return "", err + } + + tmpFile, err := afero.TempFile(fs, dir, "") + if err != nil { + return "", err + } + _, err = tmpFile.WriteString(content) + if err != nil { + return "", err + } + + return tmpFile.Name(), nil +} diff --git a/test/kubernetes/testutils/helper/util.go b/test/kubernetes/testutils/helper/util.go new file mode 100644 index 00000000000..331b6117657 --- /dev/null +++ b/test/kubernetes/testutils/helper/util.go @@ -0,0 +1,147 @@ +package helper + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/google/go-github/v32/github" + . "github.com/onsi/gomega" + errors "github.com/rotisserie/eris" + "github.com/solo-io/gloo/test/testutils" + "github.com/solo-io/gloo/test/testutils/version" + "github.com/solo-io/go-utils/changelogutils" + "github.com/solo-io/go-utils/githubutils" + "github.com/solo-io/go-utils/versionutils" + "github.com/solo-io/skv2/codegen/util" +) + +// Deprecated; if this is needed create a resource yaml for it. +func GetHttpEchoImage() string { + httpEchoImage := "hashicorp/http-echo" + if runtime.GOARCH == "arm64" { + httpEchoImage = "gcr.io/solo-test-236622/http-echo" + } + return httpEchoImage +} + +// For nightly runs, we want to install a released version rather than using a locally built chart +// To do this, set the environment variable RELEASED_VERSION with either a version name or "LATEST" to get the last release +func GetTestReleasedVersion(ctx context.Context, repoName string) string { + releasedVersion := os.Getenv(testutils.ReleasedVersion) + + if releasedVersion == "" { + // In the case where the released version is empty, we return an empty string + // The function which consumes this value will then use the locally built chart + return releasedVersion + } + + if releasedVersion == "LATEST" { + _, current, err := GetUpgradeVersions(ctx, repoName) + Expect(err).NotTo(HaveOccurred()) + return current.String() + } + + // Assume that releasedVersion is a valid version, for a previously released version of Gloo Edge + return releasedVersion +} + +func GetTestHelperForRootDir(ctx context.Context, rootDir, namespace string) (*SoloTestHelper, error) { + if useVersion := GetTestReleasedVersion(ctx, "gloo"); useVersion != "" { + return NewSoloTestHelper(func(defaults TestConfig) TestConfig { + defaults.RootDir = rootDir + defaults.HelmChartName = "gloo" + defaults.InstallNamespace = namespace + defaults.ReleasedVersion = useVersion + defaults.Verbose = true + return defaults + }) + } else { + return NewSoloTestHelper(func(defaults TestConfig) TestConfig { + defaults.RootDir = rootDir + defaults.HelmChartName = "gloo" + defaults.InstallNamespace = namespace + defaults.Verbose = true + return defaults + }) + } +} + +// GetUpgradeVersions returns two semantic versions of a repository: +// - prevLtsRelease: the latest patch release of v1.m-1.x +// - latestRelease: the latest patch release of v1.m.x +// +// Originally intended for use in upgrade testing, it can return any of: +// - (prevLtsRelease, latestRelease, nil): all release versions computable +// - (prevLtsRelease, nil, nil): only prevLtsRelease computable (ie current branch has never been released) +// - (nil, nil, err): unable to fetch versions for upgrade test +func GetUpgradeVersions(ctx context.Context, repoName string) (*versionutils.Version, *versionutils.Version, error) { + // get the latest and upcoming releases of the current branch + files, changelogReadErr := os.ReadDir(filepath.Join(util.GetModuleRoot(), changelogutils.ChangelogDirectory)) + if changelogReadErr != nil { + return nil, nil, changelogutils.ReadChangelogDirError(changelogReadErr) + } + latestRelease, upcomingRelease, upcomingReleaseErr := version.ChangelogDirForLatestRelease(files...) + if upcomingReleaseErr != nil && !errors.Is(upcomingReleaseErr, version.FirstReleaseError) { + return nil, nil, upcomingReleaseErr + } + + // get latest release of previous LTS branch + // TODO(nfuden): Update goutils to not use a struct but rather interface so we can test this more easily. + client, githubClientErr := githubutils.GetClient(ctx) + if githubClientErr != nil { + return nil, nil, errors.Wrapf(githubClientErr, "unable to create github client") + } + prevLtsRelease, prevLtsReleaseErr := getLatestReleasedPatchVersion(ctx, client, repoName, upcomingRelease.Major, upcomingRelease.Minor-1) + if prevLtsReleaseErr != nil { + return nil, nil, prevLtsReleaseErr + } + + if upcomingReleaseErr != nil { + // if we don't yet have a release for the current branch, we can only upgrade from prevLtsRelease + return prevLtsRelease, nil, nil + } else { + // otherwise, we can upgrade from both prevLtsRelease -and- latestRelease + return prevLtsRelease, latestRelease, nil + } +} + +type latestPatchForMinorPredicate struct { + versionPrefix string +} + +func (s *latestPatchForMinorPredicate) Apply(release *github.RepositoryRelease) bool { + return strings.HasPrefix(*release.Name, s.versionPrefix) && + !release.GetPrerelease() && // we don't want a prerelease version + !strings.Contains(release.GetBody(), "This release build failed") && // we don't want a failed build + release.GetPublishedAt().Before(time.Now().In(time.UTC).Add(time.Duration(-60)*time.Minute)) +} + +func newLatestPatchForMinorPredicate(versionPrefix string) *latestPatchForMinorPredicate { + return &latestPatchForMinorPredicate{ + versionPrefix: versionPrefix, + } +} + +// getLatestReleasedPatchVersion will return the latest released patch version for the given major and minor version +// NOTE: this attempts to reach out to github to get the latest release +func getLatestReleasedPatchVersion(ctx context.Context, client *github.Client, repoName string, majorVersion, minorVersion int) (*versionutils.Version, error) { + + versionPrefix := fmt.Sprintf("v%d.%d", majorVersion, minorVersion) + releases, err := githubutils.GetRepoReleasesWithPredicateAndMax(ctx, client, "solo-io", repoName, newLatestPatchForMinorPredicate(versionPrefix), 1) + if err != nil { + return nil, errors.Wrap(err, "unable to get releases") + } + if len(releases) == 0 { + return nil, errors.Errorf("Could not find a recent release with version prefix: %s", versionPrefix) + } + v, err := versionutils.ParseVersion(*releases[0].Name) + if err != nil { + return nil, errors.Wrapf(err, "error parsing release name") + } + return v, nil +}