Move glooctl tests to new framework (#9718)

* initial move for glooctl tests

* combine glooctl helm manifest and edge gw manifest

* fix check crds

* move test helper values in glooContext

* fix crd check for not gloo-system ns

* passing tests

* passing eventually error test

* changelog, cleanup

* update references to old glooctl tests

* regen docs

* remove old glooctl test from workflow

* Adding changelog file to new location

* Deleting changelog file from old location

* fix race debug

* remove old glooctl tests from nightlys

* Adding changelog file to new location

* Deleting changelog file from old location

* Add back nightlys for 1.15 and 1.16

* add glooctl tests back

* remove tmp dir for check-crds

* Adding changelog file to new location

* Deleting changelog file from old location

* istio feedback from prev pr

* helm manifest typo

* chart uri

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>

.github/workflows/nightly-tests.yaml

@@ -238,7 +238,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # TODO(npolshak): remove gloomtls after 3 days of successful runs of new test suite
+        # TODO(npolshak): remove gloomtls, glooctl after 3 days of successful runs of new test suite
         kube-e2e-test-type: ['gateway', 'gloo', 'helm', 'gloomtls', 'glooctl', 'upgrade']
         kube-version: [ { node: 'v1.25.16@sha256:5da57dfc290ac3599e775e63b8b6c49c0c85d3fec771cd7d55b45fae14b38d3b', kubectl: 'v1.25.16', kind: 'v0.20.0', helm: 'v3.13.2' },
                         { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' }]
@@ -268,7 +268,7 @@ jobs:
         #   We should extend the support/usage of those .env files to these other jobs.
         #   The tests are currently in flux, and some of these regression tests are being migrated, so we decided
         #   to limit the scope (and potentially unnecessary work) for now
-        # TODO(npolshak): remove gloomtls after 3 days of successful runs of new test suite
+        # TODO(npolshak): remove gloomtls, glooctl after 3 days of successful runs of new test suite
         kube-e2e-test-type: ['gateway', 'gloo', 'helm', 'gloomtls', 'glooctl', 'upgrade']
         kube-version: [ { node: 'v1.25.16@sha256:5da57dfc290ac3599e775e63b8b6c49c0c85d3fec771cd7d55b45fae14b38d3b', kubectl: 'v1.25.16', kind: 'v0.20.0', helm: 'v3.13.2' },
                         { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' }]
@@ -289,7 +289,7 @@ jobs:
       fail-fast: false
       matrix:
         # ingress are deprecated from 1.17. Ref: https://solo-io-corp.slack.com/archives/G01EERAK3KJ/p1716389614777799
-        # TODO(npolshak): remove gloomtls after 3 days of successful runs of new test suite
+        # TODO(npolshak): remove gloomtls, glooctl after 3 days of successful runs of new test suite
         kube-e2e-test-type: [ 'gateway', 'gloo', 'helm', 'gloomtls', 'glooctl', 'upgrade' ]
         kube-version: [ { node: 'v1.25.16@sha256:5da57dfc290ac3599e775e63b8b6c49c0c85d3fec771cd7d55b45fae14b38d3b', kubectl: 'v1.25.16', kind: 'v0.20.0', helm: 'v3.13.2' },
                         { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' } ]

.github/workflows/pr-kubernetes-tests.yaml

@@ -54,7 +54,7 @@ jobs:
         # May 14th: ~ minutes execution time (see load_balancing_tests.md)
         - cluster-name: 'cluster-one'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestGloomtlsGatewayEdgeGateway$$|^TestK8sGateway$$/^RouteDelegation$$|^TestK8sGateway$$/^Glooctl$$|^TestK8sGateway$$/^Services$$'
+          go-test-run-regex: '^TestGloomtlsGatewayEdgeGateway$$|^TestK8sGateway$$/^RouteDelegation$$|^TestK8sGateway$$/^Services$$|^TestGlooctlGlooGatewayEdgeGateway$$|^TestGlooctlK8sGateway$$'
 
         # May 14th: ~ minutes execution time (see load_balancing_tests.md)
         - cluster-name: 'cluster-two'

.github/workflows/regression-tests.yaml

@@ -49,7 +49,7 @@ jobs:
         # upgrade tests are run on LTS but not on main branch, for main they are run nightly
         # ingress will be deprecated from 1.17. Ref: https://solo-io-corp.slack.com/archives/G01EERAK3KJ/p1716389614777799
         # this is the github action version of ternary op
-        # TODO(npolshak): remove gloomtls after 3 days of successful runs of new test suite
+        # TODO(npolshak): remove gloomtls, glooctl after 3 days of successful runs of new test suite
         kube-e2e-test-type: [ 'gateway', 'gloo', 'helm', 'gloomtls', 'glooctl', 'upgrade' ]
         kube-version: [ { node: 'v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245', kubectl: 'v1.29.2', kind: 'v0.20.0', helm: 'v3.14.4' } ]
         image-variant:

changelog/v1.18.0-beta8/add-glooctl-tests-new-framework.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6303
+    resolvesIssue: false
+    description: >-
+      Migrate glooctl tests from kube2e to new framework.
+      skipCI-docs-build:true

docs/content/reference/cli/glooctl_check-crds.md

@@ -19,6 +19,7 @@ glooctl check-crds [flags]
 ```
   -h, --help                 help for check-crds
       --local-chart string   check against CRDs in helm chart at path specified by this flag (supersedes --version)
+  -n, --namespace string     namespace for reading or writing resources (default "gloo-system")
       --show-yaml            show full yaml of both CRDs that differ
       --version string       version of gloo's CRDs to check against
 ```

projects/gloo/cli/pkg/cmd/check-crds/root.go

@@ -49,6 +49,7 @@ func RootCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra.
 	flagutils.AddVersionFlag(pflags, &opts.CheckCRD.Version)
 	flagutils.AddLocalChartFlag(pflags, &opts.CheckCRD.LocalChart)
 	flagutils.AddShowYamlFlag(pflags, &opts.CheckCRD.ShowYaml)
+	flagutils.AddNamespaceFlag(pflags, &opts.Metadata.Namespace)
 	cliutils.ApplyOptions(cmd, optionsFunc)
 	return cmd
 }

projects/gloo/cli/pkg/cmd/debug/debug.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/solo-io/gloo/pkg/cliutil/install"
 	installcmd "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install"
@@ -41,7 +42,11 @@ func DebugLogs(opts *options.Options, w io.Writer) error {
 	storageClient := debugutils.NewFileStorageClient(fs)
 
 	// if writing to a non-zipped file, create a channel to collect logs
-	var fileBuf chan string
+	// Need fileBufLock here because eg.Go() can create concurrent goroutines, which can access shared fileBuf concurrently.
+	var (
+		fileBuf     chan string
+		fileBufLock sync.Mutex
+	)
 	if !opts.Top.Zip && opts.Top.File != "" {
 		fileBuf = make(chan string, len(responses))
 	}
@@ -50,6 +55,9 @@ func DebugLogs(opts *options.Options, w io.Writer) error {
 	for _, response := range responses {
 		response := response
 		eg.Go(func() error {
+			fileBufLock.Lock()
+			defer fileBufLock.Unlock()
+
 			defer response.Response.Close()
 			var logs strings.Builder
 			if opts.Top.ErrorsOnly {

test/kubernetes/e2e/features/glooctl/check.go

@@ -16,7 +16,7 @@ type checkOutput struct {
 }
 
 var (
-	checkOutputByKey = map[string]checkOutput{
+	checkCommonGlooGatewayOutputByKey = map[string]checkOutput{
 		"deployments": {
 			include: ContainSubstring("Checking Deployments... OK"),
 			exclude: And(
@@ -93,6 +93,9 @@ var (
 			exclude:  gstruct.Ignore(), // We have not had historical tests for this, it would be good to add
 			readOnly: ContainSubstring("Warning: checking proxies with port forwarding is disabled"),
 		},
+	}
+
+	checkK8sGatewayOutputByKey = map[string]checkOutput{
 		"kube-gateway-classes": {
 			include:  ContainSubstring("Checking Kubernetes GatewayClasses... OK"),
 			exclude:  Not(ContainSubstring("Checking Kubernetes GatewayClasses...")),

test/kubernetes/e2e/features/glooctl/check_crds_suite.go

@@ -0,0 +1,50 @@
+package glooctl
+
+import (
+	"context"
+
+	"github.com/solo-io/gloo/test/kubernetes/e2e"
+	"github.com/stretchr/testify/suite"
+)
+
+var _ e2e.NewSuiteFunc = NewDebugSuite
+
+// checkCrdsSuite contains the set of tests to validate the behavior of `glooctl check-crds`
+type checkCrdsSuite struct {
+	suite.Suite
+
+	ctx              context.Context
+	testInstallation *e2e.TestInstallation
+}
+
+func NewCheckCrdsSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
+	return &checkCrdsSuite{
+		ctx:              ctx,
+		testInstallation: testInst,
+	}
+}
+
+func (s *checkCrdsSuite) TestValidatesCorrectCrds() {
+	if s.testInstallation.Metadata.ReleasedVersion != "" {
+		err := s.testInstallation.Actions.Glooctl().CheckCrds(s.ctx,
+			"-n", s.testInstallation.Metadata.InstallNamespace,
+			"--kube-context", s.testInstallation.ClusterContext.KubeContext,
+			"--version", s.testInstallation.Metadata.ChartVersion)
+		s.NoError(err)
+	} else {
+		err := s.testInstallation.Actions.Glooctl().CheckCrds(s.ctx,
+			"-n", s.testInstallation.Metadata.InstallNamespace,
+			"--kube-context", s.testInstallation.ClusterContext.KubeContext,
+			"--local-chart", s.testInstallation.Metadata.ChartUri)
+		s.NoError(err)
+	}
+}
+
+func (s *checkCrdsSuite) TestCrdMismatch() {
+	err := s.testInstallation.Actions.Glooctl().CheckCrds(s.ctx,
+		"-n", s.testInstallation.Metadata.InstallNamespace,
+		"--kube-context", s.testInstallation.ClusterContext.KubeContext,
+		"--version", "1.9.0")
+	s.Error(err)
+	s.Contains(err.Error(), "One or more CRDs are out of date")
+}