Backport 1.17 Move glooctl and gloomtls tests to new framework

.github/workflows/pr-kubernetes-tests.yaml

@@ -54,7 +54,7 @@ jobs:
         # May 14th: ~ minutes execution time (see load_balancing_tests.md)
         - cluster-name: 'cluster-one'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestK8sGateway$$/^RouteDelegation$$|^TestK8sGateway$$/^Glooctl$$|^TestK8sGateway$$/^Services$$'
+          go-test-run-regex: '^TestGloomtlsGatewayEdgeGateway$$|^TestK8sGateway$$/^RouteDelegation$$|^TestK8sGateway$$/^Services$$|^TestGlooctlGlooGatewayEdgeGateway$$|^TestGlooctlK8sGateway$$'
 
         # May 14th: ~ minutes execution time (see load_balancing_tests.md)
         - cluster-name: 'cluster-two'

.github/workflows/regression-tests.yaml

@@ -49,6 +49,7 @@ jobs:
         # upgrade tests are run on LTS but not on main branch, for main they are run nightly
         # ingress is deprecated from 1.17. Ref: https://solo-io-corp.slack.com/archives/G01EERAK3KJ/p1716389614777799
         # this is the github action version of ternary op
+        # TODO(npolshak): remove gloomtls, glooctl after 3 days of successful runs of new test suite
         kube-e2e-test-type: [ 'gateway', 'gloo', 'helm', 'gloomtls', 'glooctl', 'upgrade' ]
         kube-version: [ { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' } ]
         image-variant:

changelog/v1.17.1/add-glooctl-tests-new-framework.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6303
+    resolvesIssue: false
+    description: >-
+      Migrate glooctl tests from kube2e to new framework.
+      skipCI-docs-build:true

changelog/v1.17.1/gloomtls-e2e-tests.yaml

@@ -0,0 +1,8 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6303
+    resolvesIssue: false
+    description: >-
+      Migrate gloomtls tests from kube2e to new framework.
+      
+      skipCI-docs-build:true

docs/content/reference/cli/glooctl_check-crds.md

@@ -19,6 +19,7 @@ glooctl check-crds [flags]
 ```
   -h, --help                 help for check-crds
       --local-chart string   check against CRDs in helm chart at path specified by this flag (supersedes --version)
+  -n, --namespace string     namespace for reading or writing resources (default "gloo-system")
       --show-yaml            show full yaml of both CRDs that differ
       --version string       version of gloo's CRDs to check against
 ```

projects/gloo/cli/pkg/cmd/check-crds/root.go

@@ -49,6 +49,7 @@ func RootCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra.
 	flagutils.AddVersionFlag(pflags, &opts.CheckCRD.Version)
 	flagutils.AddLocalChartFlag(pflags, &opts.CheckCRD.LocalChart)
 	flagutils.AddShowYamlFlag(pflags, &opts.CheckCRD.ShowYaml)
+	flagutils.AddNamespaceFlag(pflags, &opts.Metadata.Namespace)
 	cliutils.ApplyOptions(cmd, optionsFunc)
 	return cmd
 }

projects/gloo/cli/pkg/cmd/debug/debug.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/solo-io/gloo/pkg/cliutil/install"
 	installcmd "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install"
@@ -41,7 +42,11 @@ func DebugLogs(opts *options.Options, w io.Writer) error {
 	storageClient := debugutils.NewFileStorageClient(fs)
 
 	// if writing to a non-zipped file, create a channel to collect logs
-	var fileBuf chan string
+	// Need fileBufLock here because eg.Go() can create concurrent goroutines, which can access shared fileBuf concurrently.
+	var (
+		fileBuf     chan string
+		fileBufLock sync.Mutex
+	)
 	if !opts.Top.Zip && opts.Top.File != "" {
 		fileBuf = make(chan string, len(responses))
 	}
@@ -50,6 +55,9 @@ func DebugLogs(opts *options.Options, w io.Writer) error {
 	for _, response := range responses {
 		response := response
 		eg.Go(func() error {
+			fileBufLock.Lock()
+			defer fileBufLock.Unlock()
+
 			defer response.Response.Close()
 			var logs strings.Builder
 			if opts.Top.ErrorsOnly {

test/kube2e/README.md

@@ -73,7 +73,7 @@ The below table contains the environment variables that can be used to configure
 
 | Name             | Default | Description                                                                                                                                                                                                                                        |
 |------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| KUBE2E_TESTS     | gateway | Name of the test suite to be run. Options: `'gateway', 'gloo', 'ingress', 'helm', 'gloomtls', 'glooctl', 'upgrade', 'istio'`                                                                                                                       |
+| KUBE2E_TESTS     | gateway | Name of the test suite to be run. Options: `'gateway', 'gloo', 'ingress', 'helm', 'glooctl', 'upgrade', 'istio'`                                                                                                                                   |
 | DEBUG            | 0       | Set to 1 for debug log output                                                                                                                                                                                                                      |
 | WAIT_ON_FAIL     | 0       | Set to 1 to prevent Ginkgo from cleaning up the Gloo Edge installation in case of failure. Useful to exec into inspect resources created by the test. A command to resume the test run (and thus clean up resources) will be logged to the output. |
 | TEAR_DOWN        | false   | Set to true to uninstall Gloo after the test suite completes                                                                                                                                                                                       |

test/kubernetes/e2e/features/glooctl/check.go

@@ -16,7 +16,7 @@ type checkOutput struct {
 }
 
 var (
-	checkOutputByKey = map[string]checkOutput{
+	checkCommonGlooGatewayOutputByKey = map[string]checkOutput{
 		"deployments": {
 			include: ContainSubstring("Checking Deployments... OK"),
 			exclude: And(
@@ -93,6 +93,9 @@ var (
 			exclude:  gstruct.Ignore(), // We have not had historical tests for this, it would be good to add
 			readOnly: ContainSubstring("Warning: checking proxies with port forwarding is disabled"),
 		},
+	}
+
+	checkK8sGatewayOutputByKey = map[string]checkOutput{
 		"kube-gateway-classes": {
 			include:  ContainSubstring("Checking Kubernetes GatewayClasses... OK"),
 			exclude:  Not(ContainSubstring("Checking Kubernetes GatewayClasses...")),

test/kubernetes/e2e/features/glooctl/check_crds_suite.go

@@ -0,0 +1,50 @@
+package glooctl
+
+import (
+	"context"
+
+	"github.com/solo-io/gloo/test/kubernetes/e2e"
+	"github.com/stretchr/testify/suite"
+)
+
+var _ e2e.NewSuiteFunc = NewDebugSuite
+
+// checkCrdsSuite contains the set of tests to validate the behavior of `glooctl check-crds`
+type checkCrdsSuite struct {
+	suite.Suite
+
+	ctx              context.Context
+	testInstallation *e2e.TestInstallation
+}
+
+func NewCheckCrdsSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
+	return &checkCrdsSuite{
+		ctx:              ctx,
+		testInstallation: testInst,
+	}
+}
+
+func (s *checkCrdsSuite) TestValidatesCorrectCrds() {
+	if s.testInstallation.Metadata.ReleasedVersion != "" {
+		err := s.testInstallation.Actions.Glooctl().CheckCrds(s.ctx,
+			"-n", s.testInstallation.Metadata.InstallNamespace,
+			"--kube-context", s.testInstallation.ClusterContext.KubeContext,
+			"--version", s.testInstallation.Metadata.ChartVersion)
+		s.NoError(err)
+	} else {
+		err := s.testInstallation.Actions.Glooctl().CheckCrds(s.ctx,
+			"-n", s.testInstallation.Metadata.InstallNamespace,
+			"--kube-context", s.testInstallation.ClusterContext.KubeContext,
+			"--local-chart", s.testInstallation.Metadata.ChartUri)
+		s.NoError(err)
+	}
+}
+
+func (s *checkCrdsSuite) TestCrdMismatch() {
+	err := s.testInstallation.Actions.Glooctl().CheckCrds(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace,
+		"--kube-context", s.testInstallation.ClusterContext.KubeContext,
+		"--version", "1.9.0")
+	s.Error(err)
+	s.Contains(err.Error(), "One or more CRDs are out of date")
+}

test/kubernetes/e2e/features/glooctl/check_suite.go

@@ -3,11 +3,13 @@ package glooctl
 import (
 	"context"
 	"fmt"
+	"os"
+	"time"
 
 	"github.com/onsi/gomega"
-
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	"github.com/stretchr/testify/suite"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 var _ e2e.NewSuiteFunc = NewCheckSuite
@@ -33,31 +35,55 @@ func (s *checkSuite) TestCheck() {
 		"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics")
 	s.NoError(err)
 
-	for _, expectedOutput := range checkOutputByKey {
+	for _, expectedOutput := range checkCommonGlooGatewayOutputByKey {
 		gomega.Expect(output).To(expectedOutput.include)
 	}
+
+	if s.testInstallation.Metadata.K8sGatewayEnabled {
+		for _, expectedOutput := range checkK8sGatewayOutputByKey {
+			gomega.Expect(output).To(expectedOutput.include)
+		}
+	}
 }
 
 func (s *checkSuite) TestCheckExclude() {
-	for excludeKey, expectedOutput := range checkOutputByKey {
+	for excludeKey, expectedOutput := range checkCommonGlooGatewayOutputByKey {
 		output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
 			"-n", s.testInstallation.Metadata.InstallNamespace, "-x", fmt.Sprintf("xds-metrics,%s", excludeKey))
 		s.NoError(err)
 		gomega.Expect(output).To(expectedOutput.exclude)
 	}
+
+	if s.testInstallation.Metadata.K8sGatewayEnabled {
+		for excludeKey, expectedOutput := range checkK8sGatewayOutputByKey {
+			output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
+				"-n", s.testInstallation.Metadata.InstallNamespace, "-x", fmt.Sprintf("xds-metrics,%s", excludeKey))
+			s.NoError(err)
+			gomega.Expect(output).To(expectedOutput.exclude)
+		}
+	}
 }
 
 func (s *checkSuite) TestCheckReadOnly() {
 	output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
 		"-n", s.testInstallation.Metadata.InstallNamespace, "--read-only")
 	s.NoError(err)
 
-	for _, expectedOutput := range checkOutputByKey {
+	for _, expectedOutput := range checkCommonGlooGatewayOutputByKey {
 		gomega.Expect(output).To(gomega.And(
 			expectedOutput.include,
 			expectedOutput.readOnly,
 		))
 	}
+
+	if s.testInstallation.Metadata.K8sGatewayEnabled {
+		for _, expectedOutput := range checkK8sGatewayOutputByKey {
+			gomega.Expect(output).To(gomega.And(
+				expectedOutput.include,
+				expectedOutput.readOnly,
+			))
+		}
+	}
 }
 
 func (s *checkSuite) TestCheckKubeContext() {
@@ -72,3 +98,163 @@ func (s *checkSuite) TestCheckKubeContext() {
 		"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext)
 	s.NoError(err)
 }
+
+func (s *checkSuite) TestCheckTimeout() {
+	// When passing short timeout, check will fail
+	shortTimeoutValues, err := os.CreateTemp("", "*.yaml")
+	s.NoError(err)
+	_, err = shortTimeoutValues.Write([]byte(`checkTimeoutSeconds: 1ns`))
+	s.NoError(err)
+
+	_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace,
+		"-c", shortTimeoutValues.Name())
+	s.Error(err)
+	s.Contains(err.Error(), "context deadline exceeded")
+
+	// When passing valid timeout, check should succeed
+	normalTimeoutValues, err := os.CreateTemp("", "*.yaml")
+	s.NoError(err)
+	_, err = normalTimeoutValues.Write([]byte(`checkTimeoutSeconds: 300s`))
+	s.NoError(err)
+
+	_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace,
+		"-c", normalTimeoutValues.Name())
+	s.NoError(err)
+}
+
+func (s *checkSuite) TestCheckNamespace() {
+	// namespace does not exist
+	output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system")
+	s.Error(err)
+	s.Contains(output, "Could not communicate with kubernetes cluster: namespaces \"not-gloo-system\" not found")
+
+	// gloo not in namespace
+	output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default")
+	s.Error(err)
+	s.Contains(output, "Warning: The provided label selector (gloo) applies to no pods")
+
+	// pod does not exist
+	output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace,
+		"-p", "not-gloo")
+	s.NoError(err)
+	s.Contains(output, "Warning: The provided label selector (not-gloo) applies to no pods")
+	s.Contains(output, "No problems detected.")
+
+	// resource namespace does not exist
+	output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace,
+		"-r", "not-gloo-system")
+	s.Error(err)
+	s.Contains(output, fmt.Sprintf("No namespaces specified are currently being watched (defaulting to '%s' namespace)", s.testInstallation.Metadata.InstallNamespace))
+}
+
+func (s *checkSuite) TestNoGateways() {
+	s.T().Cleanup(func() {
+		// Scale gateways back
+		err := s.testInstallation.ClusterContext.Cli.Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deploy/gateway-proxy", 1)
+		s.NoError(err)
+		s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, metav1.ObjectMeta{
+			Name:      "gateway-proxy",
+			Namespace: s.testInstallation.Metadata.InstallNamespace,
+		}, gomega.Equal(1))
+
+		err = s.testInstallation.ClusterContext.Cli.Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deploy/public-gw", 2)
+		s.NoError(err)
+		s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, metav1.ObjectMeta{
+			Name:      "public-gw",
+			Namespace: s.testInstallation.Metadata.InstallNamespace,
+		}, gomega.Equal(2)) // helm has replicas=2 defined
+	})
+
+	// Scale gateways down
+	err := s.testInstallation.ClusterContext.Cli.Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deploy/gateway-proxy", 0)
+	s.NoError(err)
+	s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, metav1.ObjectMeta{
+		Name:      "gateway-proxy",
+		Namespace: s.testInstallation.Metadata.InstallNamespace,
+	}, gomega.Equal(0))
+
+	// public-gw is defined in the helm chart
+	err = s.testInstallation.ClusterContext.Cli.Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deploy/public-gw", 0)
+	s.NoError(err)
+	s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, metav1.ObjectMeta{
+		Name:      "public-gw",
+		Namespace: s.testInstallation.Metadata.InstallNamespace,
+	}, gomega.Equal(0))
+
+	_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
+	s.Error(err)
+	s.Contains(err.Error(), "Gloo installation is incomplete: no active gateway-proxy pods exist in cluster")
+}
+
+func (s *checkSuite) TestEdgeGatewayScaled() {
+	s.T().Cleanup(func() {
+		// Scale gateway back
+		err := s.testInstallation.ClusterContext.Cli.Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deploy/gateway-proxy", 1)
+		s.NoError(err)
+		s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, metav1.ObjectMeta{
+			Name:      "gateway-proxy",
+			Namespace: s.testInstallation.Metadata.InstallNamespace,
+		}, gomega.Equal(1))
+	})
+
+	// Scale gateway down
+	err := s.testInstallation.ClusterContext.Cli.Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deploy/gateway-proxy", 0)
+	s.NoError(err)
+
+	// Wait for gateway to be gone
+	s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, metav1.ObjectMeta{
+		Name:      "gateway-proxy",
+		Namespace: s.testInstallation.Metadata.InstallNamespace,
+	}, gomega.Equal(0))
+
+	output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
+	s.NoError(err)
+	s.Contains(output, fmt.Sprintf("Warning: %s:gateway-proxy has zero replicas", s.testInstallation.Metadata.InstallNamespace))
+	s.Contains(output, "No problems detected.")
+	// Check healthy output
+	for _, expectedOutput := range checkCommonGlooGatewayOutputByKey {
+		gomega.Expect(output).To(expectedOutput.include)
+	}
+
+	if s.testInstallation.Metadata.K8sGatewayEnabled {
+		for _, expectedOutput := range checkK8sGatewayOutputByKey {
+			gomega.Expect(output).To(expectedOutput.include)
+		}
+	}
+}
+
+func (s *checkSuite) TestEdgeResourceError() {
+	s.T().Cleanup(func() {
+		// Delete invalid config
+		err := s.testInstallation.ClusterContext.Cli.DeleteFileSafe(s.ctx, invalidVSKubeDest)
+		s.NoError(err)
+		err = s.testInstallation.ClusterContext.Cli.DeleteFileSafe(s.ctx, invalidVSUpstreamDest, "-n", s.testInstallation.Metadata.InstallNamespace)
+		s.NoError(err)
+	})
+
+	// Apply invalid config
+	err := s.testInstallation.ClusterContext.Cli.ApplyFile(s.ctx, invalidVSKubeDest)
+	s.NoError(err)
+	err = s.testInstallation.ClusterContext.Cli.ApplyFile(s.ctx, invalidVSUpstreamDest, "-n", s.testInstallation.Metadata.InstallNamespace)
+	s.NoError(err)
+
+	// Run check. This needs to run in eventually to get all errors reported
+	gomega.Eventually(func() error {
+		_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
+			"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext)
+		return err
+	}, time.Minute*2, time.Second*10).Should(gomega.SatisfyAll(
+		gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* Found rejected virtual service by '%s': default reject-me-too (Reason: 2 errors occurred:", s.testInstallation.Metadata.InstallNamespace))),
+		gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* domain conflict: other virtual services that belong to the same Gateway as this one don't specify a domain (and thus default to '*'): [%s.reject-me]", s.testInstallation.Metadata.InstallNamespace))),
+		gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* VirtualHost Error: DomainsNotUniqueError. Reason: domain * is shared by the following virtual hosts: [default.reject-me-too %s.reject-me]", s.testInstallation.Metadata.InstallNamespace))),
+		gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* Found rejected virtual service by '%s': %s reject-me (Reason: 2 errors occurred:", s.testInstallation.Metadata.InstallNamespace, s.testInstallation.Metadata.InstallNamespace))),
+		gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* domain conflict: other virtual services that belong to the same Gateway as this one don't specify a domain (and thus default to '*'): [default.reject-me-too]"))),
+		gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* VirtualHost Error: DomainsNotUniqueError. Reason: domain * is shared by the following virtual hosts: [default.reject-me-too %s.reject-me]", s.testInstallation.Metadata.InstallNamespace))),
+	))
+}