[1.17] Ensure namespace override consistency (#9900)

* Ensure namespace override consistency (#9888)

* update to test structure

* add consistency to namespace handling in test setup

* remove ref to env pkg

* move kuberentes e2e test util tests to gha unit tests

* update timeout for nightlies

* add check to action for tests all being run under TestGlooGateway

* add namespace code to example test

* fix regex

* remove summaries change

* update changelog

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>

* bring util test over

* make IsEnvTruthy use IsTruthyValue

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>

.github/workflows/nightly-tests.yaml

@@ -50,7 +50,7 @@ jobs:
     name: End-to-End (branch=${{ github.ref_name }}, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
     if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'workflow_initiating_branch' }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 90
+    timeout-minutes: 150
     strategy:
       # Since we are running these on a schedule, there is no value in failing fast
       # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
@@ -60,7 +60,7 @@ jobs:
           # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
           # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
           - cluster-name: 'cluster-one'
-            go-test-args: '-v -timeout=90m'
+            go-test-args: '-v -timeout=120m'
             go-test-run-regex: ${{ inputs.kubernetes-end-to-end-run-regex }}
         # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
@@ -110,7 +110,7 @@ jobs:
     name: End-to-End (branch=main, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} )
     if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'main') || github.event.schedule == '0 5 * * 1-5' }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 90
+    timeout-minutes: 150
     strategy:
       # Since we are running these on a schedule, there is no value in failing fast
       # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking
@@ -120,7 +120,7 @@ jobs:
           # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel.
           # As a result, we increase the threshold for the tests, since they all run serially on a single cluster
           - cluster-name: 'cluster-one'
-            go-test-args: '-v -timeout=90m'
+            go-test-args: '-v -timeout=120m'
             go-test-run-regex: ""
         # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support
         # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
@@ -367,4 +367,4 @@ jobs:
           PREAMBLE: ${{ steps.compute-preamble.outputs.preamble }}  # text to hyperlink at start of slack message
           SLACKBOT_BEARER: ${{ secrets.SLACKBOT_BEARER }}
         run: |
-          go run .github/workflows/helpers/notify/slack.go '${{ toJSON(needs) }}'
+          go run .github/workflows/helpers/notify/slack.go '${{ toJSON(needs) }}'

.github/workflows/pr-unit-tests.yaml

@@ -36,11 +36,11 @@ jobs:
     - name: Run Tests
       shell: bash
       env:
-        TEST_PKG: ./projects/gateway2/...
+        TEST_PKG: "./projects/gateway2/... ./test/kubernetes/testutils/helper"
       run: make go-test-with-coverage
     - name: Validate Test Coverage
       shell: bash
       # The make will error if test coverage drops below a certain threshold
       # We intentionally ignore the errors while we build out our test coverage, to establish a good baseline
       # However, we should strive to establish a baseline, and then make it required on PRs
-      run: make --always-make --ignore-errors validate-test-coverage
+      run: make --always-make --ignore-errors validate-test-coverage

Makefile

@@ -254,7 +254,7 @@ test-with-coverage: test
 	go tool cover -html $(OUTPUT_DIR)/coverage.cov
 
 .PHONY: run-tests
-run-tests: GINKGO_FLAGS += -skip-package=e2e,gateway2 ## Run all non E2E tests, or only run the test package at {TEST_PKG} if it is specified
+run-tests: GINKGO_FLAGS += -skip-package=e2e,gateway2,test/kubernetes/testutils/helper ## Run all non E2E tests, or only run the test package at {TEST_PKG} if it is specified
 run-tests: GINKGO_FLAGS += --label-filter="!end-to-end && !performance"
 run-tests: test
 

changelog/v1.17.4/test-summaries.yaml

@@ -0,0 +1,6 @@
+changelog:
+  - type: NON_USER_FACING
+    description: >-
+      Update all tests to use INSTALL_NAMESPACE defined by the TestInstallation setup.
+      Move test utils unit tests to GHA because it requires an authorized token.
+      Update nightlies timeout.

pkg/utils/envutils/env.go

@@ -8,12 +8,38 @@ import (
 // IsEnvTruthy returns true if a given environment variable has a truthy value
 // Examples of truthy values are: "1", "t", "T", "true", "TRUE", "True". Anything else is considered false.
 func IsEnvTruthy(envVarName string) bool {
-	envValue, _ := strconv.ParseBool(os.Getenv(envVarName))
-	return envValue
+	return IsTruthyValue(os.Getenv(envVarName))
 }
 
 // IsEnvDefined returns true if a given environment variable has any value
 func IsEnvDefined(envVarName string) bool {
 	envValue := os.Getenv(envVarName)
 	return len(envValue) > 0
 }
+
+// IsTruthyValue returns true if a given value is "truthy".
+// Examples of truthy values are: "1", "t", "T", "true", "TRUE", "True". Anything else is considered false.
+func IsTruthyValue(value string) bool {
+	envValue, _ := strconv.ParseBool(value)
+	return envValue
+}
+
+// GetOrDefault returns the value of the environment variable for the given key,
+// or the default value if the environment variable is not set.
+func GetOrDefault(key, fallback string) string {
+	if value, ok := os.LookupEnv(key); ok {
+		return value
+	}
+	return fallback
+}
+
+// LookupOrDefault returns the value of the environment variable for the given key,
+// or the default value if the environment variable is not set. Also returns whether
+// the value existed.
+func LookupOrDefault(key, fallback string) (string, bool) {
+	if value, ok := os.LookupEnv(key); ok {
+		return value, ok
+	} else {
+		return fallback, ok
+	}
+}

test/kubernetes/e2e/example/debug_logging_test.go

@@ -4,11 +4,14 @@ package example
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
+	"github.com/solo-io/gloo/test/testutils"
 
 	"github.com/solo-io/skv2/codegen/util"
 	"github.com/stretchr/testify/suite"
@@ -21,19 +24,28 @@ import (
 // TestInstallationWithDebugLogLevel is the function which executes a series of tests against a given installation
 func TestInstallationWithDebugLogLevel(t *testing.T) {
 	ctx := context.Background()
+	installNs, overrodeNs := envutils.LookupOrDefault(testutils.InstallNamespace, "automtls-istio-edge-api-test")
 	testInstallation := e2e.CreateTestInstallation(
 		t,
 		&gloogateway.Context{
-			InstallNamespace:   "debug-example",
+			InstallNamespace:   installNs,
 			ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "debug-example.yaml"),
 		},
 	)
 
 	testHelper := e2e.MustTestHelper(ctx, testInstallation)
 
+	// Set the env to the install namespace if it is not already set
+	if os.Getenv(testutils.InstallNamespace) == "" {
+		os.Setenv(testutils.InstallNamespace, installNs)
+	}
+
 	// We register the cleanup function _before_ we actually perform the installation.
 	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
 	t.Cleanup(func() {
+		if overrodeNs {
+			os.Unsetenv(testutils.InstallNamespace)
+		}
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 		}

test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go

@@ -2,11 +2,14 @@ package tests_test
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
+	"github.com/solo-io/gloo/test/testutils"
 
 	"github.com/solo-io/skv2/codegen/util"
 
@@ -19,10 +22,11 @@ import (
 // the k8s Gateway controller is disabled
 func TestAutomtlsIstioEdgeApisGateway(t *testing.T) {
 	ctx := context.Background()
+	installNs, overrodeNs := envutils.LookupOrDefault(testutils.InstallNamespace, "automtls-istio-edge-api-test")
 	testInstallation := e2e.CreateTestInstallation(
 		t,
 		&gloogateway.Context{
-			InstallNamespace:   "automtls-istio-edge-api-test",
+			InstallNamespace:   installNs,
 			ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "istio-automtls-edge-gateway-test-helm.yaml"),
 		},
 	)
@@ -33,9 +37,17 @@ func TestAutomtlsIstioEdgeApisGateway(t *testing.T) {
 		t.Fatalf("failed to get istioctl: %v", err)
 	}
 
+	// Set the env to the install namespace if it is not already set
+	if os.Getenv(testutils.InstallNamespace) == "" {
+		os.Setenv(testutils.InstallNamespace, installNs)
+	}
+
 	// We register the cleanup function _before_ we actually perform the installation.
 	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
 	t.Cleanup(func() {
+		if overrodeNs {
+			os.Unsetenv(testutils.InstallNamespace)
+		}
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 

test/kubernetes/e2e/tests/automtls_istio_test.go

@@ -2,24 +2,28 @@ package tests_test
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	. "github.com/solo-io/gloo/test/kubernetes/e2e/tests"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
+	"github.com/solo-io/gloo/test/testutils"
 	"github.com/solo-io/skv2/codegen/util"
 )
 
 // TestK8sGatewayIstioAutoMtls is the function which executes a series of tests against a given installation
 func TestK8sGatewayIstioAutoMtls(t *testing.T) {
 	ctx := context.Background()
+	installNs, overrodeNs := envutils.LookupOrDefault(testutils.InstallNamespace, "automtls-istio-k8s-gw-test")
 	testInstallation := e2e.CreateTestInstallation(
 		t,
 		&gloogateway.Context{
-			InstallNamespace:   "automtls-istio-k8s-gw-test",
+			InstallNamespace:   installNs,
 			ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "istio-automtls-k8s-gateway-test-helm.yaml"),
 		},
 	)
@@ -30,9 +34,17 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) {
 		t.Errorf("failed to add istioctl: %v\n", err)
 	}
 
+	// Set the env to the install namespace if it is not already set
+	if os.Getenv(testutils.InstallNamespace) == "" {
+		os.Setenv(testutils.InstallNamespace, installNs)
+	}
+
 	// We register the cleanup function _before_ we actually perform the installation.
 	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
 	t.Cleanup(func() {
+		if overrodeNs {
+			os.Unsetenv(testutils.InstallNamespace)
+		}
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 

test/kubernetes/e2e/tests/disabled_transformation_validation_test.go

@@ -2,34 +2,46 @@ package tests_test
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	. "github.com/solo-io/gloo/test/kubernetes/e2e/tests"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
+	"github.com/solo-io/gloo/test/testutils"
 	"github.com/solo-io/skv2/codegen/util"
 )
 
 // TestTransformationValidationDisabled is the function which executes a series of tests against a given
 // installation where validation has disableTransformationValidation=true
 func TestTransformationValidationDisabled(t *testing.T) {
 	ctx := context.Background()
+	installNs, overrodeNs := envutils.LookupOrDefault(testutils.InstallNamespace, "disable-transformation-validation-test")
 	testInstallation := e2e.CreateTestInstallation(
 		t,
 		&gloogateway.Context{
-			InstallNamespace:   "disable-transformation-validation-test",
+			InstallNamespace:   installNs,
 			ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "disable-transformation-validation-helm.yaml"),
 		},
 	)
 
 	testHelper := e2e.MustTestHelper(ctx, testInstallation)
 
+	// Set the env to the install namespace if it is not already set
+	if os.Getenv(testutils.InstallNamespace) == "" {
+		os.Setenv(testutils.InstallNamespace, installNs)
+	}
+
 	// We register the cleanup function _before_ we actually perform the installation.
 	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
 	t.Cleanup(func() {
+		if overrodeNs {
+			os.Unsetenv(testutils.InstallNamespace)
+		}
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 		}

test/kubernetes/e2e/tests/edge_gw_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/solo-io/gloo/pkg/utils/env"
+	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	. "github.com/solo-io/gloo/test/kubernetes/e2e/tests"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway"
@@ -20,7 +20,7 @@ import (
 // the k8s Gateway controller is disabled
 func TestGlooGatewayEdgeGateway(t *testing.T) {
 	ctx := context.Background()
-	installNs := env.GetOrDefault(testutils.InstallNamespace, "gloo-gateway-edge-test")
+	installNs, overrodeNs := envutils.LookupOrDefault(testutils.InstallNamespace, "gloo-gateway-edge-test")
 	testInstallation := e2e.CreateTestInstallation(
 		t,
 		&gloogateway.Context{
@@ -38,6 +38,9 @@ func TestGlooGatewayEdgeGateway(t *testing.T) {
 	// We register the cleanup function _before_ we actually perform the installation.
 	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
 	t.Cleanup(func() {
+		if overrodeNs {
+			os.Unsetenv(testutils.InstallNamespace)
+		}
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 		}

test/kubernetes/e2e/tests/glooctl_edge_gw_test.go

@@ -2,34 +2,46 @@ package tests_test
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	. "github.com/solo-io/gloo/test/kubernetes/e2e/tests"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway"
 	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
+	"github.com/solo-io/gloo/test/testutils"
 	"github.com/solo-io/skv2/codegen/util"
 )
 
 // TestGlooctlGlooGatewayEdgeGateway is the function which executes a series of glooctl tests against a given
 // installation where the k8s Gateway controller is disabled.
 func TestGlooctlGlooGatewayEdgeGateway(t *testing.T) {
 	ctx := context.Background()
+	installNs, overrodeNs := envutils.LookupOrDefault(testutils.InstallNamespace, "glooctl-gloo-gateway-edge-test")
 	testInstallation := e2e.CreateTestInstallation(
 		t,
 		&gloogateway.Context{
-			InstallNamespace:   "glooctl-gloo-gateway-edge-test",
+			InstallNamespace:   installNs,
 			ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "glooctl-edge-gateway-test-helm.yaml"),
 		},
 	)
 
 	testHelper := e2e.MustTestHelper(ctx, testInstallation)
 
+	// Set the env to the install namespace if it is not already set
+	if os.Getenv(testutils.InstallNamespace) == "" {
+		os.Setenv(testutils.InstallNamespace, installNs)
+	}
+
 	// We register the cleanup function _before_ we actually perform the installation.
 	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
 	t.Cleanup(func() {
+		if overrodeNs {
+			os.Unsetenv(testutils.InstallNamespace)
+		}
 		if t.Failed() {
 			testInstallation.PreFailHandler(ctx)
 		}