Skip to content

Commit

Permalink
e2e: cluster details getter layer
Browse files Browse the repository at this point in the history
Abstract from a specific way of getting information about a cluster.
Provide two ways of getting e.g. clusterplatform type.
Either by reading infrastructure.config.io/cluster object.
Or, be reading corresponding environment variables.
  • Loading branch information
ingvagabund committed Feb 15, 2023
1 parent 7f660fc commit 31b76a6
Show file tree
Hide file tree
Showing 71 changed files with 3,964 additions and 257 deletions.
24 changes: 13 additions & 11 deletions test/extended/apiserver/api_requests.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ var _ = g.Describe("[sig-arch][Late]", func() {

oc := exutil.NewCLIWithoutNamespace("api-requests")

g.It("clients should not use APIs that are removed in upcoming releases [apigroup:config.openshift.io]", func() {
g.It("clients should not use APIs that are removed in upcoming releases [apigroup:apiserver.openshift.io]", func() {
ctx := context.Background()
apirequestCountClient, err := apiserverclientv1.NewForConfig(oc.AdminConfig())
o.Expect(err).NotTo(o.HaveOccurred())
Expand Down Expand Up @@ -91,12 +91,14 @@ var _ = g.Describe("[sig-arch][Late]", func() {
}
})

g.It("operators should not create watch channels very often [apigroup:config.openshift.io]", func() {
g.It("operators should not create watch channels very often [apigroup:apiserver.openshift.io]", func() {
ctx := context.Background()
apirequestCountClient, err := apiserverclientv1.NewForConfig(oc.AdminConfig())
o.Expect(err).NotTo(o.HaveOccurred())

infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
platformType, err := exutil.ClusterDetails(oc.AdminConfigClient()).PlatformType()
o.Expect(err).NotTo(o.HaveOccurred())
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

type platformUpperBound map[string]int64
Expand Down Expand Up @@ -320,16 +322,16 @@ var _ = g.Describe("[sig-arch][Late]", func() {

var upperBound platformUpperBound

if infra.Status.ControlPlaneTopology == configv1.SingleReplicaTopologyMode {
if _, exists := upperBoundsSingleNode[infra.Spec.PlatformSpec.Type]; !exists {
e2eskipper.Skipf("Unsupported single node platform type: %v", infra.Spec.PlatformSpec.Type)
if *controlPlaneTopology == configv1.SingleReplicaTopologyMode {
if _, exists := upperBoundsSingleNode[platformType]; !exists {
e2eskipper.Skipf("Unsupported single node platform type: %v", platformType)
}
upperBound = upperBoundsSingleNode[infra.Spec.PlatformSpec.Type]
upperBound = upperBoundsSingleNode[platformType]
} else {
if _, exists := upperBounds[infra.Spec.PlatformSpec.Type]; !exists {
e2eskipper.Skipf("Unsupported platform type: %v", infra.Spec.PlatformSpec.Type)
if _, exists := upperBounds[platformType]; !exists {
e2eskipper.Skipf("Unsupported platform type: %v", platformType)
}
upperBound = upperBounds[infra.Spec.PlatformSpec.Type]
upperBound = upperBounds[platformType]
}

apiRequestCounts, err := apirequestCountClient.APIRequestCounts().List(ctx, metav1.ListOptions{})
Expand Down Expand Up @@ -429,7 +431,7 @@ var _ = g.Describe("[sig-arch][Late]", func() {
allowedCount, exists := upperBound[operator]

if !exists {
framework.Logf("Operator %v not found in upper bounds for %v", operator, infra.Spec.PlatformSpec.Type)
framework.Logf("Operator %v not found in upper bounds for %v", operator, platformType)
framework.Logf("operator=%v, watchrequestcount=%v", item.operator, item.count)
continue
}
Expand Down
4 changes: 2 additions & 2 deletions test/extended/apiserver/graceful_termination.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,8 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Late]", func() {
}
})

g.It("API LBs follow /readyz of kube-apiserver and stop sending requests before server shutdowns for external clients [apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It("API LBs follow /readyz of kube-apiserver and stop sending requests before server shutdowns for external clients", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

// set up
Expand Down
4 changes: 2 additions & 2 deletions test/extended/apiserver/kubeconfigs.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local ku
"localhost-recovery.kubeconfig",
} {
kubeconfig := kc
g.It(fmt.Sprintf("%q should be present on all masters and work [apigroup:config.openshift.io]", kubeconfig), func() {
g.It(fmt.Sprintf("%q should be present on all masters and work", kubeconfig), func() {
// external controlplane topology doesn't have master nodes
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())
if *controlPlaneTopology == configv1.ExternalTopologyMode {
g.Skip("ExternalControlPlaneTopology doesn't have master node kubeconfigs")
Expand Down
4 changes: 2 additions & 2 deletions test/extended/authentication/front_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ var _ = g.Describe("[sig-auth][Feature:Authentication] ", func() {
oc := exutil.NewCLI("project-api")

g.Describe("TestFrontProxy", func() {
g.It(fmt.Sprintf("should succeed [apigroup:config.openshift.io]"), func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It(fmt.Sprintf("should succeed"), func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

if *controlPlaneTopology == configv1.ExternalTopologyMode {
Expand Down
8 changes: 4 additions & 4 deletions test/extended/authorization/authorization.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ func (test resourceAccessReviewTest) run() {
failMessage := ""
var err error

g.By("resourceAccessReviewTest - "+test.description+" [apigroup:config.openshift.io]", func() {
g.By("resourceAccessReviewTest - "+test.description, func() {
// keep trying the test until you get a success or you timeout. Every time you have a failure, set the fail message
// so that if you never have a success, we can call t.Errorf with a reasonable message
// exiting the poll with `failMessage=""` indicates success.
Expand All @@ -254,7 +254,7 @@ func (test resourceAccessReviewTest) run() {
}
}

controlPlaneTopology, err := exutil.GetControlPlaneTopology(test.ocClient)
controlPlaneTopology, err := exutil.ClusterDetails(test.ocClient.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

// many operators installed in openshift- namespaces are allowed to read all namespaces. We simply suppress all those users from the user slice
Expand Down Expand Up @@ -314,7 +314,7 @@ func (test localResourceAccessReviewTest) run() {
failMessage := ""
var err error

g.By("localResourceAccessReviewTest - "+test.description+" [apigroup:authorization.openshift.io][apigroup:config.openshift.io]", func() {
g.By("localResourceAccessReviewTest - "+test.description+" [apigroup:authorization.openshift.io]", func() {
// keep trying the test until you get a success or you timeout. Every time you have a failure, set the fail message
// so that if you never have a success, we can call t.Errorf with a reasonable message
// exiting the poll with `failMessage=""` indicates success.
Expand Down Expand Up @@ -344,7 +344,7 @@ func (test localResourceAccessReviewTest) run() {
return false, nil
}

controlPlaneTopology, err := exutil.GetControlPlaneTopology(test.ocClient)
controlPlaneTopology, err := exutil.ClusterDetails(test.ocClient.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

// many operators installed in openshift- namespaces are allowed to read all namespaces. We simply suppress all those users from the user slice
Expand Down
9 changes: 4 additions & 5 deletions test/extended/baremetal/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,9 @@ import (
func skipIfNotBaremetal(oc *exutil.CLI) {
g.By("checking platform type")

infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
platformType, err := exutil.ClusterDetails(oc.AdminConfigClient()).PlatformType()
o.Expect(err).NotTo(o.HaveOccurred())

if infra.Status.PlatformStatus.Type != configv1.BareMetalPlatformType {
if platformType != configv1.BareMetalPlatformType {
e2eskipper.Skipf("No baremetal platform detected")
}
}
Expand All @@ -35,10 +34,10 @@ func skipIfNotBaremetal(oc *exutil.CLI) {
func skipIfUnsupportedPlatformOrConfig(oc *exutil.CLI, dc dynamic.Interface) {
g.By("checking supported platforms")

infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
platformType, err := exutil.ClusterDetails(oc.AdminConfigClient()).PlatformType()
o.Expect(err).NotTo(o.HaveOccurred())

switch infra.Status.PlatformStatus.Type {
switch platformType {
case configv1.BareMetalPlatformType:
return
case configv1.OpenStackPlatformType:
Expand Down
6 changes: 3 additions & 3 deletions test/extended/baremetal/hosts.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = g.Describe("[sig-installer][Feature:baremetal] Baremetal/OpenStack/vSphere/None/AWS/Azure/GCP platforms [apigroup:config.openshift.io]", func() {
var _ = g.Describe("[sig-installer][Feature:baremetal] Baremetal/OpenStack/vSphere/None/AWS/Azure/GCP platforms", func() {
defer g.GinkgoRecover()

var (
Expand All @@ -37,7 +37,7 @@ var _ = g.Describe("[sig-installer][Feature:baremetal] Baremetal/OpenStack/vSphe
})
})

var _ = g.Describe("[sig-installer][Feature:baremetal] Baremetal platform should [apigroup:config.openshift.io]", func() {
var _ = g.Describe("[sig-installer][Feature:baremetal] Baremetal platform should", func() {
defer g.GinkgoRecover()

oc := exutil.NewCLI("baremetal")
Expand Down Expand Up @@ -147,7 +147,7 @@ var _ = g.Describe("[sig-installer][Feature:baremetal] Baremetal platform should

// This block must be used for the serial tests. Any eventual extra worker deployed will be
// automatically deleted during the AfterEach
var _ = g.Describe("[sig-installer][Feature:baremetal][Serial] Baremetal platform should [apigroup:config.openshift.io]", func() {
var _ = g.Describe("[sig-installer][Feature:baremetal][Serial] Baremetal platform should", func() {
defer g.GinkgoRecover()

var (
Expand Down
17 changes: 8 additions & 9 deletions test/extended/ci/job_names.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,18 @@ var _ = g.Describe("[sig-ci] [Early] prow job name", func() {
}
isPeriodic := strings.HasPrefix(jobName, "periodic-")

g.It("should match platform type [apigroup:config.openshift.io]", func() {
g.It("should match platform type", func() {
if jobName == "" {
e2eskipper.Skipf("JOB_NAME env var not set, skipping")
} else if strings.Contains(jobName, "agnostic") {
e2eskipper.Skipf("JOB_NAME contains agnostic, not expecting platform in name")
}

infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
platformType, err := exutil.ClusterDetails(oc.AdminConfigClient()).PlatformType()
o.Expect(err).NotTo(o.HaveOccurred())

hasPlatform := true
platform := infra.Status.PlatformStatus.Type
switch platform {
switch platformType {
case v1.AlibabaCloudPlatformType:
if !strings.Contains(jobName, "alibaba") {
hasPlatform = false
Expand All @@ -55,25 +54,25 @@ var _ = g.Describe("[sig-ci] [Early] prow job name", func() {
hasPlatform = false
}
default:
if !strings.Contains(jobName, strings.ToLower(string(platform))) {
if !strings.Contains(jobName, strings.ToLower(string(platformType))) {
hasPlatform = false
}
}

if !hasPlatform {
result.Flakef("job name %q does not contain platform type in name (%s)", originalJobName, platform)
result.Flakef("job name %q does not contain platform type in name (%s)", originalJobName, platformType)
}

})

g.It("should match network type [apigroup:config.openshift.io]", func() {
g.It("should match network type", func() {
if jobName == "" {
e2eskipper.Skipf("JOB_NAME env var not set, skipping")
}

network, err := oc.AdminConfigClient().ConfigV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{})
networkType, err := exutil.ClusterDetails(oc.AdminConfigClient()).NetworkType()
o.Expect(err).NotTo(o.HaveOccurred())
switch network.Status.NetworkType {
switch networkType {
case "OpenShiftSDN":
if strings.Contains(jobName, "ovn") {
e2e.Failf("job name %q has mismatched network type in name (expected `sdn`, found `ovn`)", jobName)
Expand Down
4 changes: 2 additions & 2 deletions test/extended/cli/explain.go
Original file line number Diff line number Diff line change
Expand Up @@ -569,10 +569,10 @@ var _ = g.Describe("[sig-cli] oc explain", func() {
})
}

g.It("should contain proper spec+status for CRDs [apigroup:config.openshift.io]", func() {
g.It("should contain proper spec+status for CRDs", func() {
crdClient := apiextensionsclientset.NewForConfigOrDie(oc.AdminConfig())
crdTypesTest := getCrdTypes(oc)
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())
// External clusters are not expected to have 'autoscaling' or
// 'machine' Types, just the 'base' Types
Expand Down
14 changes: 7 additions & 7 deletions test/extended/cli/mustgather.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ var _ = g.Describe("[sig-cli] oc adm must-gather", func() {
o.Expect(err).NotTo(o.HaveOccurred())
})

g.It("runs successfully [apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It("runs successfully", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())
if *controlPlaneTopology == configv1.ExternalTopologyMode {
g.Skip("oc must-gather doesn't currently support external controlPlaneTopology")
Expand Down Expand Up @@ -101,8 +101,8 @@ var _ = g.Describe("[sig-cli] oc adm must-gather", func() {
}
})

g.It("runs successfully with options [apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It("runs successfully with options", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())
if *controlPlaneTopology == configv1.ExternalTopologyMode {
g.Skip("oc must-gather doesn't currently support external controlPlaneTopology")
Expand All @@ -128,7 +128,7 @@ var _ = g.Describe("[sig-cli] oc adm must-gather", func() {

g.It("runs successfully for audit logs [apigroup:config.openshift.io][apigroup:oauth.openshift.io]", func() {
// On External clusters, events will not be part of the output, since audit logs do not include control plane logs.
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

if *controlPlaneTopology == configv1.ExternalTopologyMode {
Expand Down Expand Up @@ -301,8 +301,8 @@ var _ = g.Describe("[sig-cli] oc adm must-gather", func() {

g.When("looking at the audit logs", func() {
g.Describe("[sig-node] kubelet", func() {
g.It("runs apiserver processes strictly sequentially in order to not risk audit log corruption [apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It("runs apiserver processes strictly sequentially in order to not risk audit log corruption", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

// On External clusters, events will not be part of the output, since audit logs do not include control plane logs.
Expand Down
6 changes: 3 additions & 3 deletions test/extended/cloud_controller_manager/ccm.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ var _ = g.Describe("[sig-cloud-provider][Feature:OpenShiftCloudControllerManager
defer g.GinkgoRecover()
oc := exutil.NewCLI("ccm")

g.It("Deploy an external cloud provider [apigroup:config.openshift.io][apigroup:machineconfiguration.openshift.io]", func() {
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
g.It("Deploy an external cloud provider [apigroup:machineconfiguration.openshift.io]", func() {
platformType, err := exutil.ClusterDetails(oc.AdminConfigClient()).PlatformType()
o.Expect(err).NotTo(o.HaveOccurred())

if infra.Status.PlatformStatus.Type != configv1.OpenStackPlatformType {
if platformType != configv1.OpenStackPlatformType {
g.Skip("Platform does not use external cloud provider")
}

Expand Down
6 changes: 3 additions & 3 deletions test/extended/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ var _ = g.Describe("[sig-cli][Feature:LegacyCommandTests][Disruptive][Serial] te
continue
}

g.It("test/cmd/"+currFilename+" [apigroup:image.openshift.io][apigroup:config.openshift.io]", func() {
g.It("test/cmd/"+currFilename+" [apigroup:image.openshift.io]", func() {
testsDir := exutil.FixturePath("testdata", "cmd", "test", "cmd")
oc.AddExplicitResourceToDelete(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"}, "", "cmd-"+currFilename[0:len(currFilename)-3])

Expand Down Expand Up @@ -71,7 +71,7 @@ var _ = g.Describe("[sig-cli][Feature:LegacyCommandTests][Disruptive][Serial] te
}
}

infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
APIServerURL, err := exutil.ClusterDetails(oc.AdminConfigClient()).APIServerURL()
o.Expect(err).NotTo(o.HaveOccurred())

log, errs := exutil.RunOneShotCommandPod(oc, "test-cmd", cliImageRef, "/var/tests/test/cmd/"+currFilename,
Expand Down Expand Up @@ -99,7 +99,7 @@ var _ = g.Describe("[sig-cli][Feature:LegacyCommandTests][Disruptive][Serial] te
},
[]corev1.EnvVar{
{Name: "KUBECONFIG_TESTS", Value: "/var/tests/kubeconfig/kubeconfig"},
{Name: "KUBERNETES_MASTER", Value: infra.Status.APIServerURL},
{Name: "KUBERNETES_MASTER", Value: APIServerURL},
{Name: "USER_TOKEN", Value: oc.UserConfig().BearerToken},
{Name: "TESTS_DIR", Value: "/var/tests/test/cmd"},
{Name: "TEST_NAME", Value: currFilename[0 : len(currFilename)-3]},
Expand Down
12 changes: 6 additions & 6 deletions test/extended/csrapprover/csrapprover.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ var _ = g.Describe("[sig-cluster-lifecycle]", func() {
oc := exutil.NewCLIWithPodSecurityLevel("cluster-client-cert", admissionapi.LevelBaseline)
defer g.GinkgoRecover()

g.It("Pods cannot access the /config/master API endpoint [apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It("Pods cannot access the /config/master API endpoint", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

// https://issues.redhat.com/browse/CO-760
Expand All @@ -54,10 +54,10 @@ var _ = g.Describe("[sig-cluster-lifecycle]", func() {
o.Expect(err).NotTo(o.HaveOccurred())

// get the API server URL, mutate to internal API (use infra.Status.APIServerURLInternal) once API is bumped
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
APIServerURL, err := exutil.ClusterDetails(oc.AdminConfigClient()).APIServerURL()
o.Expect(err).NotTo(o.HaveOccurred())

internalAPI, err := url.Parse(infra.Status.APIServerURL)
internalAPI, err := url.Parse(APIServerURL)
o.Expect(err).NotTo(o.HaveOccurred())
internalAPI.Host = strings.Replace(internalAPI.Host, "api.", "api-int.", 1)

Expand All @@ -73,8 +73,8 @@ var _ = g.Describe("[sig-cluster-lifecycle]", func() {
o.Expect(curlOutput).To(o.ContainSubstring("Connection refused"))
})

g.It("CSRs from machines that are not recognized by the cloud provider are not approved [apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
g.It("CSRs from machines that are not recognized by the cloud provider are not approved", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

if *controlPlaneTopology == configv1.ExternalTopologyMode {
Expand Down
4 changes: 2 additions & 2 deletions test/extended/etcd/etcd_test_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ var _ = g.Describe("[sig-api-machinery] API data in etcd", func() {

oc := exutil.NewCLIWithPodSecurityLevel("etcd-storage-path", psapi.LevelBaseline).AsAdmin()

_ = g.It("should be stored at the correct location and version for all resources [Serial][apigroup:config.openshift.io]", func() {
controlPlaneTopology, err := exutil.GetControlPlaneTopology(oc)
_ = g.It("should be stored at the correct location and version for all resources [Serial]", func() {
controlPlaneTopology, err := exutil.ClusterDetails(oc.AdminConfigClient()).ControlPlaneTopology()
o.Expect(err).NotTo(o.HaveOccurred())

if *controlPlaneTopology == configv1.ExternalTopologyMode {
Expand Down
Loading

0 comments on commit 31b76a6

Please sign in to comment.