From e7a50238f2e22faaca9699ef779dd4bc6cd6a3bb Mon Sep 17 00:00:00 2001 From: Xue Li Date: Tue, 19 Mar 2024 14:27:43 +0800 Subject: [PATCH] OCM-6314 | ci: Refactor day1 creation function for test Fix comments Fix comment Remove the redunant func OCM-6314 | ci: Refactor day1 creation function for test Fix name prefix cannot be set issue --- .gitignore | 1 + tests/ci/config/config.go | 87 +++ tests/ci/data/profiles/external.yaml | 2 +- tests/ci/data/profiles/rosa-classic.yaml | 42 +- tests/ci/data/profiles/rosa-hcp.yaml | 4 +- tests/e2e/dummy_test.go | 28 +- tests/e2e/e2e_setup_test.go | 24 + tests/utils/common/constants/cluster.go | 34 ++ tests/utils/common/file.go | 19 +- tests/utils/common/helper.go | 10 + tests/utils/common/string.go | 33 + tests/utils/config/cluster.go | 93 ++- tests/utils/exec/rosacli/cluster_service.go | 27 +- tests/utils/exec/rosacli/cmd_parser.go | 2 +- tests/utils/exec/rosacli/cmd_runner.go | 11 +- .../exec/rosacli/ocm_resource_service.go | 28 + tests/utils/exec/rosacli/version_service.go | 89 ++- .../utils/profilehandler/data_preparation.go | 208 +++++++ tests/utils/profilehandler/interface.go | 78 +++ tests/utils/profilehandler/parse_yaml.go | 51 ++ tests/utils/profilehandler/profile_handler.go | 565 ++++++++++++++++++ 21 files changed, 1373 insertions(+), 63 deletions(-) create mode 100644 tests/ci/config/config.go create mode 100644 tests/e2e/e2e_setup_test.go create mode 100644 tests/utils/common/helper.go create mode 100644 tests/utils/profilehandler/data_preparation.go create mode 100644 tests/utils/profilehandler/interface.go create mode 100644 tests/utils/profilehandler/parse_yaml.go create mode 100644 tests/utils/profilehandler/profile_handler.go diff --git a/.gitignore b/.gitignore index c75d5264d6..df5d0394b2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ docs/ /rosa-linux-arm64 /rosa-linux-arm64.sha256 /*policy.json +/tests/output .envrc .env cover.out diff --git a/tests/ci/config/config.go b/tests/ci/config/config.go new file mode 100644 index 0000000000..a6346455a9 --- /dev/null +++ b/tests/ci/config/config.go @@ -0,0 +1,87 @@ +package config + +import ( + "fmt" + "os" + "path" + "strconv" + "strings" + + "github.com/openshift/rosa/tests/utils/common" + . "github.com/openshift/rosa/tests/utils/log" +) + +var Test *TestConfig + +// TestConfig contains platforms info for the rosacli testing +type TestConfig struct { + // Env is the OpenShift Cluster Management environment used to provision clusters. + ENV string `env:"OCM_LOGIN_ENV" default:""` + TestProfile string `env:"TEST_PROFILE" default:""` + OutputDir string `env:"OUTPUT_DIR" default:""` + YAMLProfilesDir string `env:"TEST_PROFILE_DIR" default:""` + RootDir string `env:"WORKSPACE" default:""` + ClusterConfigFile string + ArtifactDir string `env:"ARTIFACT_DIR" default:""` + UserDataFile string + ClusterIDFile string // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + CreateCommandFile string + APIURLFile string // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + ClusterNameFile string // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + ClusterTypeFile string // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + ConsoleUrlFile string // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + InfraIDFile string // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + ClusterDetailFile string + ClusterInstallLogArtifactFile string + ClusterAdminFile string + GlobalENV *GlobalENVVariables +} +type GlobalENVVariables struct { + ChannelGroup string `env:"CHANNEL_GROUP" default:""` + Version string `env:"VERSION" default:""` + Region string `env:"REGION" default:""` + ProvisionShard string `env:"PROVISION_SHARD" default:""` + NamePrefix string `env:"NAME_PREFIX"` + ClusterWaitingTime int `env:"CLUSTER_TIMEOUT" default:"60"` +} + +func init() { + Test = new(TestConfig) + currentDir, _ := os.Getwd() + project := "rosa" + + Test.TestProfile = common.ReadENVWithDefaultValue("TEST_PROFILE", "") + Test.RootDir = common.ReadENVWithDefaultValue("WORKSPACE", strings.SplitAfter(currentDir, project)[0]) + Test.YAMLProfilesDir = common.ReadENVWithDefaultValue("TEST_PROFILE_DIR", path.Join(Test.RootDir, "tests", "ci", "data", "profiles")) + Test.OutputDir = common.ReadENVWithDefaultValue("SHARED_DIR", path.Join(Test.RootDir, "tests", "output", Test.TestProfile)) + Test.ArtifactDir = common.ReadENVWithDefaultValue("ARTIFACT_DIR", Test.OutputDir) + err := os.MkdirAll(Test.OutputDir, 0777) + if err != nil { + Logger.Errorf("Meet error %s when create output dirs", err.Error()) + } + Test.ClusterConfigFile = path.Join(Test.OutputDir, "cluster-config") + Test.UserDataFile = path.Join(Test.OutputDir, "resources.json") + Test.APIURLFile = path.Join(Test.OutputDir, "api.url") + Test.ClusterIDFile = path.Join(Test.OutputDir, "cluster-id") // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + Test.ClusterNameFile = path.Join(Test.OutputDir, "cluster-name") // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + Test.ClusterTypeFile = path.Join(Test.OutputDir, "cluster-type") // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + Test.ConsoleUrlFile = path.Join(Test.OutputDir, "console.url") // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + Test.InfraIDFile = path.Join(Test.OutputDir, "infra_id") // Temporary file to compatible to current CI jobs. Will remove once all CI jobs migration finished + Test.ClusterDetailFile = path.Join(Test.OutputDir, "cluster-detail.json") + Test.ClusterInstallLogArtifactFile = path.Join(Test.ArtifactDir, ".install.log") + Test.ClusterAdminFile = path.Join(Test.ArtifactDir, ".admin") + + waitingTime, err := strconv.Atoi(common.ReadENVWithDefaultValue("CLUSTER_TIMEOUT", "60")) + if err != nil { + panic(fmt.Errorf("env variable CLUSTER_TIMEOUT must be set to an integer")) + } + Test.GlobalENV = &GlobalENVVariables{ + ChannelGroup: os.Getenv("CHANNEL_GROUP"), + Version: os.Getenv("VERSION"), + Region: os.Getenv("REGION"), + ProvisionShard: os.Getenv("PROVISION_SHARD"), + NamePrefix: os.Getenv("NAME_PREFIX"), + ClusterWaitingTime: waitingTime, + } + +} diff --git a/tests/ci/data/profiles/external.yaml b/tests/ci/data/profiles/external.yaml index cb046c1fa8..39f9f1167b 100644 --- a/tests/ci/data/profiles/external.yaml +++ b/tests/ci/data/profiles/external.yaml @@ -31,4 +31,4 @@ profiles: additional_sg_number: 3 account-role: path: "" - permission_boundary: "" \ No newline at end of file + permission_boundary: "" diff --git a/tests/ci/data/profiles/rosa-classic.yaml b/tests/ci/data/profiles/rosa-classic.yaml index f92d5b7e4e..583e0e086d 100644 --- a/tests/ci/data/profiles/rosa-classic.yaml +++ b/tests/ci/data/profiles/rosa-classic.yaml @@ -1,7 +1,7 @@ profiles: - as: rosa-advanced - version: latest - channel_group: candidate + version: "latest" + channel_group: "candidate" region: "us-east-1" cluster: multi_az: true @@ -28,11 +28,47 @@ profiles: volume_size: 150 autoscaler_enabled: true disable_uwm: true - long_name: true + name_length: 54 + domain_prefix_enabled: true additional_sg_number: 3 account-role: path: "/test/" permission_boundary: "arn:aws:iam::aws:policy/AdministratorAccess" +- as: rosa-private-link + version: "latest" + channel_group: "candidate" + region: "us-east-1" + cluster: + multi_az: true + instance_type: "" + hcp: false + sts: true + byo_vpc: true + private_link: true + private: true + etcd_encryption: false + fips: false + autoscale: true + kms_key: true + networking: false + proxy_enabled: false + label_enabled: false + tag_enabled: false + zones: "" + imdsv2: "" + shared_vpc: false + ingress_customized: false + oidc_config: managed + admin_enabled: false + volume_size: 150 + autoscaler_enabled: false + disable_uwm: false + name_length: 15 + domain_prefix_enabled: false + additional_sg_number: 0 + account-role: + path: "/test/" + permission_boundary: "arn:aws:iam::aws:policy/AdministratorAccess" - as: rosa-shared-vpc version: latest channel_group: candidate diff --git a/tests/ci/data/profiles/rosa-hcp.yaml b/tests/ci/data/profiles/rosa-hcp.yaml index ad73421022..61d914c650 100644 --- a/tests/ci/data/profiles/rosa-hcp.yaml +++ b/tests/ci/data/profiles/rosa-hcp.yaml @@ -24,7 +24,6 @@ profiles: oidc_config: "managed" auditlog_forward: true admin_enabled: false - managed_policies: true disable_uwm: true autoscaler_enabled: true account-role: @@ -81,6 +80,8 @@ profiles: imdsv2: "" ingress_customized: true auditlog_forward: false + name_length: 54 + customize_domain_prefix: true account-role: path: "" permission_boundary: "" @@ -139,7 +140,6 @@ profiles: shared_vpc: false auditlog_forward: true admin_enabled: false - managed_policies: true disable_uwm: true network_type: other external_auth_config: true diff --git a/tests/e2e/dummy_test.go b/tests/e2e/dummy_test.go index d0e773bd44..315e29c207 100644 --- a/tests/e2e/dummy_test.go +++ b/tests/e2e/dummy_test.go @@ -1,10 +1,15 @@ package e2e import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/openshift/rosa/tests/utils/log" + TC "github.com/openshift/rosa/tests/ci/config" + "github.com/openshift/rosa/tests/utils/exec/rosacli" + "github.com/openshift/rosa/tests/utils/log" + PH "github.com/openshift/rosa/tests/utils/profilehandler" ) var _ = Describe("ROSA CLI Test", func() { @@ -12,7 +17,26 @@ var _ = Describe("ROSA CLI Test", func() { It("Dummy", func() { str := "dummy string" Expect(str).ToNot(BeEmpty()) - Logger.Infof("This is a dummy test to check everything is fine by executing jobs. Please remove me once other tests are added") + log.Logger.Infof("This is a dummy test to check everything is fine by executing jobs. Please remove me once other tests are added") + }) + }) + Describe("Profile test", func() { + It("ProfileParserTest", func() { + profile := PH.LoadProfileYamlFileByENV() + log.Logger.Infof("Got configured profile prefix: %v", *profile) + log.Logger.Infof("Got configured profile: %v", profile.NamePrefix) + log.Logger.Infof("Got configured cluster profile: %v", *profile.ClusterConfig) + log.Logger.Infof("Got configured account role profile: %v", *profile.AccountRoleConfig) + }) + It("TestENVSetup", func() { + log.Logger.Infof("Got dir of out: %v", TC.Test.OutputDir) + }) + It("TestPrepareClusterByProfile", func() { + client := rosacli.NewClient() + profile := PH.LoadProfileYamlFileByENV() + cluster, err := PH.CreateClusterByProfile(profile, client, true) + Expect(err).ToNot(HaveOccurred()) + fmt.Println(cluster.ID) }) }) }) diff --git a/tests/e2e/e2e_setup_test.go b/tests/e2e/e2e_setup_test.go new file mode 100644 index 0000000000..3864894e82 --- /dev/null +++ b/tests/e2e/e2e_setup_test.go @@ -0,0 +1,24 @@ +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openshift/rosa/tests/ci/labels" + "github.com/openshift/rosa/tests/utils/exec/rosacli" + "github.com/openshift/rosa/tests/utils/log" + PH "github.com/openshift/rosa/tests/utils/profilehandler" +) + +var _ = Describe("ROSA CLI Test", func() { + It("PrepareClusterByProfile", + labels.Critical, + labels.Day1Prepare, + func() { + client := rosacli.NewClient() + profile := PH.LoadProfileYamlFileByENV() + cluster, err := PH.CreateClusterByProfile(profile, client, true) + Expect(err).ToNot(HaveOccurred()) + log.Logger.Infof("Cluster prepared successfully with id %s", cluster.ID) + }) +}) diff --git a/tests/utils/common/constants/cluster.go b/tests/utils/common/constants/cluster.go index 7e0c16130a..56aec889f5 100644 --- a/tests/utils/common/constants/cluster.go +++ b/tests/utils/common/constants/cluster.go @@ -1,6 +1,40 @@ package constants +import ( + "regexp" +) + const ( ClusterDescriptionComputeDesired = "Compute (desired)" ClusterDescriptionComputeAutoscaled = "Compute (autoscaled)" ) + +// role and OIDC config +const ( + MaxRolePrefixLength = 32 + MaxOIDCConfigPrefixLength = 15 +) + +// profile +const ( + DefaultNamePrefix = "rosacli-ci" +) + +// cluster status +const ( + Ready = "ready" + Installing = "installing" + Waiting = "waiting" + Pending = "pending" + Error = "error" + Uninstalling = "uninstalling" + Validating = "validating" +) + +// version pattern supported for the CI +var ( + VersionLatestPattern = regexp.MustCompile("latest") + VersionMajorMinorPattern = regexp.MustCompile(`^[0-9]+\.[0-9]+$`) + VersionRawPattern = regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+-?[0-9a-z\.-]*`) + VersionFlexyPattern = regexp.MustCompile(`[xy]{1}-[1-3]{1}`) +) diff --git a/tests/utils/common/file.go b/tests/utils/common/file.go index 870b90374b..96fdfb35e6 100644 --- a/tests/utils/common/file.go +++ b/tests/utils/common/file.go @@ -1,6 +1,7 @@ package common import ( + "encoding/json" "os" "strings" @@ -20,8 +21,22 @@ func CreateTempFileWithPrefixAndContent(prefix string, fileContent string) (stri } // Write string to a file -func CreateFileWithContent(fileAbsPath string, content string) (string, error) { - err := os.WriteFile(fileAbsPath, []byte(content), 0644) +func CreateFileWithContent(fileAbsPath string, content interface{}) (string, error) { + var err error + switch content := content.(type) { + case string: + err = os.WriteFile(fileAbsPath, []byte(content), 0644) + case []byte: + err = os.WriteFile(fileAbsPath, content, 0644) + case interface{}: + var marshedContent []byte + marshedContent, err = json.Marshal(content) + if err != nil { + return fileAbsPath, err + } + err = os.WriteFile(fileAbsPath, marshedContent, 0644) + } + if err != nil { Logger.Errorf("Failed to write to file: %s", err) return "", err diff --git a/tests/utils/common/helper.go b/tests/utils/common/helper.go new file mode 100644 index 0000000000..4af4c64446 --- /dev/null +++ b/tests/utils/common/helper.go @@ -0,0 +1,10 @@ +package common + +import "os" + +func ReadENVWithDefaultValue(envName string, fallback string) string { + if os.Getenv(envName) != "" { + return os.Getenv(envName) + } + return fallback +} diff --git a/tests/utils/common/string.go b/tests/utils/common/string.go index 68bf7697da..89046f037b 100644 --- a/tests/utils/common/string.go +++ b/tests/utils/common/string.go @@ -1,6 +1,8 @@ package common import ( + r "crypto/rand" + "encoding/base64" "fmt" "math/rand" "strings" @@ -28,6 +30,22 @@ func ParseCommaSeparatedStrings(input string) (output []string) { } return } +func GenerateRandomStringWithSymbols(length int) string { + b := make([]byte, length) + _, err := r.Read(b) + if err != nil { + panic(err) + } + randomString := base64.StdEncoding.EncodeToString(b)[:length] + f := func(r rune) bool { + return r < 'A' || r > 'z' + } + // Verify that the string contains special character or number + if strings.IndexFunc(randomString, f) == -1 { + randomString = randomString[:len(randomString)-1] + "!" + } + return randomString +} // Generate random string func GenerateRandomString(n int) string { @@ -44,3 +62,18 @@ func GenerateRandomString(n int) string { func GenerateRandomName(prefix string, n int) string { return fmt.Sprintf("%s-%s", prefix, strings.ToLower(GenerateRandomString(n))) } + +func TrimNameByLength(name string, length int) string { + if len(name) <= length { + return name + } + return name[0:length] +} + +func SplitMajorVersion(openshiftVersion string) string { + splited := strings.Split(openshiftVersion, ".") + if len(splited) < 2 { + return openshiftVersion + } + return strings.Join(splited[0:2], ".") +} diff --git a/tests/utils/config/cluster.go b/tests/utils/config/cluster.go index d63803850c..d926b748fd 100644 --- a/tests/utils/config/cluster.go +++ b/tests/utils/config/cluster.go @@ -57,34 +57,73 @@ type Nodes struct { type Autoscaling struct { Enabled bool `json:"enabled,omitempty"` } +type Autoscaler struct { + AutoscalerBalanceSimilarNodeGroups bool `json:"autoscaler_balance_similar_node_groups,omitempty"` + AutoscalerSkipNodesWithLocalStorage bool `json:"autoscaler_skip_nodes_with_local_storage,omitempty"` + AutoscalerLogVerbosity string `json:"autoscaler_log_verbosity,omitempty"` + AutoscalerMaxPodGracePeriod string `json:"autoscaler_max_pod_grace_period,omitempty"` + AutoscalerPodPriorityThreshold string `json:"autoscaler_pod_priority_threshold,omitempty"` + AutoscalerIgnoreDaemonsetsUtilization bool `json:"autoscaler_ignore_daemonsets_utilization,omitempty"` + AutoscalerMaxNodeProvisionTime string `json:"autoscaler_max_node_provision_time,omitempty"` + AutoscalerBalancingIgnoredLabels string `json:"autoscaler_balancing_ignored_labels,omitempty"` + AutoscalerMaxNodesTotal string `json:"autoscaler_max_nodes_total,omitempty"` + AutoscalerMinCores string `json:"autoscaler_min_cores,omitempty"` + AutoscalerMaxCores string `json:"autoscaler_max_cores,omitempty"` + AutoscalerMinMemory string `json:"autoscaler_min_memory,omitempty"` + AutoscalerMaxMemory string `json:"autoscaler_max_memory,omitempty"` + AutoscalerGpuLimit string `json:"autoscaler_gpu_limit,omitempty"` + AutoscalerScaleDownEnabled bool `json:"autoscaler_scale_down_enabled,omitempty"` + AutoscalerScaleDownUnneededTime string `json:"autoscaler_scale_down_unneeded_time,omitempty"` + AutoscalerScaleDownUtilizationThreshold string `json:"autoscaler_scale_down_utilization_threshold,omitempty"` + AutoscalerScaleDownDelayAfterAdd string `json:"autoscaler_scale_down_delay_after_add,omitempty"` + AutoscalerScaleDownDelayAfterDelete string `json:"autoscaler_scale_down_delay_after_delete,omitempty"` + AutoscalerScaleDownDelayAfterFailure string `json:"autoscaler_scale_down_delay_after_failure,omitempty"` +} +type IngressConfig struct { + DefaultIngressRouteSelector string `json:"default_ingress_route_sector,omitempty"` + DefaultIngressExcludedNamespaces string `json:"default_ingress_excluded_namespaces,omitempty"` + DefaultIngressWildcardPolicy string `json:"default_ingress_wildcard_policy,omitempty"` + DefaultIngressNamespaceOwnershipPolicy string `json:"default_ingress_namespace_ownership_policy,omitempty"` +} +type Networking struct { + MachineCIDR string `json:"machine_cidr,omitempty"` + ServiceCIDR string `json:"service_cidr,omitempty"` + PodCIDR string `json:"pod_cidr,omitempty"` + HostPrefix string `json:"host_prefix,omitempty"` +} type ClusterConfig struct { - DisableScpChecks bool `json:"disable_scp_checks,omitempty"` - DisableWorkloadMonitoring bool `json:"disable_workload_monitoring,omitempty"` - EnableCustomerManagedKey bool `json:"enable_customer_managed_key,omitempty"` - EtcdEncryption bool `json:"etcd_encryption,omitempty"` - Fips bool `json:"fips,omitempty"` - Hypershift bool `json:"hypershift,omitempty"` - MultiAZ bool `json:"multi_az,omitempty"` - Private bool `json:"private,omitempty"` - PrivateLink bool `json:"private_link,omitempty"` - Sts bool `json:"sts,omitempty"` - AuditLogArn string `json:"audit_log_arn,omitempty"` - AvailabilityZones string `json:"availability_zones,omitempty"` - DefaultMpLabels string `json:"default_mp_labels,omitempty"` - Ec2MetadataHttpTokens string `json:"ec2_metadata_http_tokens,omitempty"` - Name string `json:"name,omitempty"` - Region string `json:"region,omitempty"` - Tags string `json:"tags,omitempty"` - WorkerDiskSize string `json:"worker_disk_size,omitempty"` - Autoscaling Autoscaling `json:"autoscaling,omitempty"` - Aws AWS `json:"aws,omitempty"` - Encryption Encryption `json:"encryption,omitempty"` - Nodes Nodes `json:"nodes,omitempty"` - Properties Properties `json:"properties,omitempty"` - Proxy Proxy `json:"proxy,omitempty"` - Subnets Subnets `json:"subnets,omitempty"` - Version Version `json:"version,omitempty"` + DisableScpChecks bool `json:"disable_scp_checks,omitempty"` + DisableWorkloadMonitoring bool `json:"disable_workload_monitoring,omitempty"` + EnableCustomerManagedKey bool `json:"enable_customer_managed_key,omitempty"` + EtcdEncryption bool `json:"etcd_encryption,omitempty"` + Fips bool `json:"fips,omitempty"` + Hypershift bool `json:"hypershift,omitempty"` + MultiAZ bool `json:"multi_az,omitempty"` + Private bool `json:"private,omitempty"` + PrivateLink bool `json:"private_link,omitempty"` + Sts bool `json:"sts,omitempty"` + AuditLogArn string `json:"audit_log_arn,omitempty"` + AvailabilityZones string `json:"availability_zones,omitempty"` + DefaultMpLabels string `json:"default_mp_labels,omitempty"` + Ec2MetadataHttpTokens string `json:"ec2_metadata_http_tokens,omitempty"` + Name string `json:"name,omitempty"` + Region string `json:"region,omitempty"` + Tags string `json:"tags,omitempty"` + WorkerDiskSize string `json:"worker_disk_size,omitempty"` + DomainPrefix string `json:"domain_prefix,omitempty"` + BillingAccount string `json:"billing_account,omitempty"` + Autoscaling *Autoscaling `json:"autoscaling,omitempty"` + Aws *AWS `json:"aws,omitempty"` + Encryption *Encryption `json:"encryption,omitempty"` + Nodes *Nodes `json:"nodes,omitempty"` + Properties *Properties `json:"properties,omitempty"` + Proxy *Proxy `json:"proxy,omitempty"` + Subnets *Subnets `json:"subnets,omitempty"` + Version *Version `json:"version,omitempty"` + Autoscaler *Autoscaler `json:"autoscaler,omitempty"` + IngressConfig *IngressConfig `json:"ingress_config,omitempty"` + Networking *Networking `json:"networking,omitempty"` } func ParseClusterProfile() (*ClusterConfig, error) { @@ -129,7 +168,7 @@ func GetClusterID() (clusterID string) { // Get the cluster config file, for jean chen func getClusterIDFile() string { sharedDir := os.Getenv("SHARED_DIR") - return path.Join(sharedDir, "cluster-id") + return path.Join(sharedDir, "cluster_id") } // Get the clusterID env. diff --git a/tests/utils/exec/rosacli/cluster_service.go b/tests/utils/exec/rosacli/cluster_service.go index 3ced87b101..23b974b578 100644 --- a/tests/utils/exec/rosacli/cluster_service.go +++ b/tests/utils/exec/rosacli/cluster_service.go @@ -19,9 +19,12 @@ type ClusterService interface { ReflectClusterDescription(result bytes.Buffer) (*ClusterDescription, error) DescribeClusterAndReflect(clusterID string) (*ClusterDescription, error) List() (bytes.Buffer, error) + Create(clusterName string, flags ...string) (bytes.Buffer, error, string) CreateDryRun(clusterName string, flags ...string) (bytes.Buffer, error) EditCluster(clusterID string, flags ...string) (bytes.Buffer, error) DeleteUpgrade(flags ...string) (bytes.Buffer, error) + InstallLog(clusterID string, flags ...string) (bytes.Buffer, error) + UnInstallLog(clusterID string, flags ...string) (bytes.Buffer, error) IsHostedCPCluster(clusterID string) (bool, error) IsSTSCluster(clusterID string) (bool, error) @@ -144,6 +147,14 @@ func (c *clusterService) CreateDryRun(clusterName string, flags ...string) (byte CmdFlags(combflags...) return createDryRun.Run() } +func (c *clusterService) Create(clusterName string, flags ...string) (bytes.Buffer, error, string) { + combflags := append([]string{"-c", clusterName}, flags...) + createCommand := c.client.Runner. + Cmd("create", "cluster"). + CmdFlags(combflags...) + output, err := createCommand.Run() + return output, err, createCommand.CMDString() +} func (c *clusterService) EditCluster(clusterID string, flags ...string) (bytes.Buffer, error) { combflags := append([]string{"-c", clusterID}, flags...) @@ -159,6 +170,18 @@ func (c *clusterService) DeleteUpgrade(flags ...string) (bytes.Buffer, error) { CmdFlags(flags...) return DeleteUpgrade.Run() } +func (c *clusterService) InstallLog(clusterID string, flags ...string) (bytes.Buffer, error) { + installLog := c.client.Runner. + Cmd("logs", "install", "-c", clusterID). + CmdFlags(flags...) + return installLog.Run() +} +func (c *clusterService) UnInstallLog(clusterID string, flags ...string) (bytes.Buffer, error) { + UnInstallLog := c.client.Runner. + Cmd("logs", "uninstall", "-c", clusterID). + CmdFlags(flags...) + return UnInstallLog.Run() +} func (c *clusterService) CleanResources(clusterID string) (errors []error) { Logger.Debugf("Nothing releated to cluster was done there") @@ -210,7 +233,7 @@ func (c *clusterService) GetClusterVersion(clusterID string) (clusterVersion con } if clusterConfig.Version.RawID != "" { - clusterVersion = clusterConfig.Version + clusterVersion = *clusterConfig.Version } else { // Else retrieve from cluster description var jsonData *jsonData @@ -254,7 +277,7 @@ func RetrieveDesiredComputeNodes(clusterDescription *ClusterDescription) (nodesN var isInt bool nodesNb, isInt = clusterDescription.Nodes[0]["Compute (desired)"].(int) if !isInt { - err = fmt.Errorf("'%v' is not an integer value") + err = fmt.Errorf("'%v' is not an integer value", isInt) } } else { // Try autoscale one diff --git a/tests/utils/exec/rosacli/cmd_parser.go b/tests/utils/exec/rosacli/cmd_parser.go index ca484571a4..4a09b035b8 100644 --- a/tests/utils/exec/rosacli/cmd_parser.go +++ b/tests/utils/exec/rosacli/cmd_parser.go @@ -264,7 +264,7 @@ func (tab *tableData) Parse() *tableData { } func (jd *jsonData) Parse() *jsonData { - var object map[string]interface{} + var object interface{} err := json.Unmarshal(jd.input.Bytes(), &object) if err != nil { Logger.Errorf(" error in Parse is %v", err) diff --git a/tests/utils/exec/rosacli/cmd_runner.go b/tests/utils/exec/rosacli/cmd_runner.go index 80d1add4a4..0b3c52b34e 100644 --- a/tests/utils/exec/rosacli/cmd_runner.go +++ b/tests/utils/exec/rosacli/cmd_runner.go @@ -160,14 +160,21 @@ func (rc *runnerConfig) GenerateCmdFlags() (flags []string) { return } -func (r *runner) Run() (bytes.Buffer, error) { - rosacmd := "rosa" +func (r *runner) CmdElements() []string { cmdElements := r.cmds if len(r.cmdArgs) > 0 { cmdElements = append(cmdElements, r.cmdArgs...) } cmdElements = append(cmdElements, r.runnerCfg.GenerateCmdFlags()...) + return cmdElements +} +func (r *runner) CMDString() string { + return fmt.Sprintf("rosa %s", strings.Join(r.CmdElements(), " ")) +} +func (r *runner) Run() (bytes.Buffer, error) { + rosacmd := "rosa" + cmdElements := r.CmdElements() var output bytes.Buffer var err error retry := 0 diff --git a/tests/utils/exec/rosacli/ocm_resource_service.go b/tests/utils/exec/rosacli/ocm_resource_service.go index e23c19e500..3317b61d21 100644 --- a/tests/utils/exec/rosacli/ocm_resource_service.go +++ b/tests/utils/exec/rosacli/ocm_resource_service.go @@ -16,6 +16,13 @@ var RoleTypeSuffixMap = map[string]string{ "Worker": "Worker-Role", } +type AccountRolesUnit struct { + InstallerRole string `json:"Installer,omitempty"` + SupportRole string `json:"Support,omitempty"` + WorkerRole string `json:"Worker,omitempty"` + ControlPlaneRole string `json:"Control plane,omitempty"` +} + type OCMResourceService interface { ResourcesCleaner @@ -299,6 +306,27 @@ func (arl AccountRoleList) AccountRoles(prefix string) (accountRoles []*AccountR return } +// Get all specified account roles by prefix and classic +func (arl AccountRoleList) DigAccountRoles(prefix string, hcp bool) *AccountRolesUnit { + var accRoles *AccountRolesUnit = new(AccountRolesUnit) + roleMap := map[string]interface{}{} + matchedAccountRoles := arl.AccountRoles(prefix) + for _, role := range matchedAccountRoles { + if hcp && strings.Contains(role.RoleName, "HCP") { + roleMap[role.RoleType] = role.RoleArn + continue + } + + if !strings.Contains(role.RoleName, "HCP") { + roleMap[role.RoleType] = role.RoleArn + continue + } + + } + MapStructure(roleMap, accRoles) + return accRoles +} + // Get specified account role by the arn func (arl AccountRoleList) AccountRole(arn string) (accountRole *AccountRole) { for _, roleItem := range arl.AccountRoleList { diff --git a/tests/utils/exec/rosacli/version_service.go b/tests/utils/exec/rosacli/version_service.go index d30699d9f4..bdbae05196 100644 --- a/tests/utils/exec/rosacli/version_service.go +++ b/tests/utils/exec/rosacli/version_service.go @@ -11,13 +11,14 @@ import ( const VersionChannelGroupStable = "stable" const VersionChannelGroupNightly = "nightly" +const VersionChannelGroupCandidate = "candidate" type VersionService interface { ResourcesCleaner - ReflectVersions(result bytes.Buffer) (*OpenShiftVersionList, error) + ReflectVersions(result bytes.Buffer) (*OpenShiftVersionTableList, error) ListVersions(channelGroup string, hostedCP bool, flags ...string) (bytes.Buffer, error) - ListAndReflectVersions(channelGroup string, hostedCP bool, flags ...string) (*OpenShiftVersionList, error) + ListAndReflectVersions(channelGroup string, hostedCP bool, flags ...string) (*OpenShiftVersionTableList, error) } type versionService struct { @@ -32,22 +33,32 @@ func NewVersionService(client *Client) VersionService { } } -type OpenShiftVersion struct { +type OpenShiftVersionTableOutput struct { Version string `json:"VERSION,omitempty"` Default string `json:"DEFAULT,omitempty"` AvailableUpgrades string `json:"AVAILABLE UPGRADES,omitempty"` } -type OpenShiftVersionList struct { - OpenShiftVersions []*OpenShiftVersion `json:"OpenShiftVersions,omitempty"` +type OpenShiftVersionJsonOutput struct { + ID string `json:"VERSION,omitempty"` + RAWID string `json:"raw_id,omitempty"` + ChannelGroup string `json:"channel_group,omitempty"` + HCPDefault bool `json:"hosted_control_plane_default,omitempty"` + HCPEnabled bool `json:"hosted_control_plane_enabled,omitempty"` + Default bool `json:"default,omitempty"` + AvailableUpgrades []string `json:"available_upgrades,omitempty"` +} + +type OpenShiftVersionTableList struct { + OpenShiftVersions []*OpenShiftVersionTableOutput `json:"OpenShiftVersions,omitempty"` } // Reflect versions -func (v *versionService) ReflectVersions(result bytes.Buffer) (versionList *OpenShiftVersionList, err error) { - versionList = &OpenShiftVersionList{} +func (v *versionService) ReflectVersions(result bytes.Buffer) (versionList *OpenShiftVersionTableList, err error) { + versionList = &OpenShiftVersionTableList{} theMap := v.client.Parser.TableData.Input(result).Parse().Output() for _, item := range theMap { - version := &OpenShiftVersion{} + version := &OpenShiftVersionTableOutput{} err = MapStructure(item, version) if err != nil { return versionList, err @@ -57,6 +68,23 @@ func (v *versionService) ReflectVersions(result bytes.Buffer) (versionList *Open return versionList, err } +// Reflect versions json +func (v *versionService) ReflectJsonVersions(result bytes.Buffer) (versionList []*OpenShiftVersionJsonOutput, err error) { + versionList = []*OpenShiftVersionJsonOutput{} + parser := v.client.Parser.JsonData.Input(result).Parse() + theMap := parser.Output().([]interface{}) + for index, _ := range theMap { + version := &OpenShiftVersionJsonOutput{} + vDetail := parser.DigObject(index).(map[string]interface{}) + err := MapStructure(vDetail, version) + if err != nil { + return versionList, err + } + versionList = append(versionList, version) + } + return versionList, err +} + // list version `rosa list version` or `rosa list version --hosted-cp` func (v *versionService) ListVersions(channelGroup string, hostedCP bool, flags ...string) (bytes.Buffer, error) { listVersion := v.client.Runner. @@ -74,7 +102,17 @@ func (v *versionService) ListVersions(channelGroup string, hostedCP bool, flags return listVersion.Run() } -func (v *versionService) ListAndReflectVersions(channelGroup string, hostedCP bool, flags ...string) (versionList *OpenShiftVersionList, err error) { +func (v *versionService) ListAndReflectVersions(channelGroup string, hostedCP bool, flags ...string) (versionList *OpenShiftVersionTableList, err error) { + var output bytes.Buffer + output, err = v.ListVersions(channelGroup, hostedCP, flags...) + if err != nil { + return versionList, err + } + + versionList, err = v.ReflectVersions(output) + return versionList, err +} +func (v *versionService) ListAndReflectJsonVersions(channelGroup string, hostedCP bool, flags ...string) (versionList *OpenShiftVersionTableList, err error) { var output bytes.Buffer output, err = v.ListVersions(channelGroup, hostedCP, flags...) if err != nil { @@ -92,7 +130,7 @@ func (v *versionService) CleanResources(clusterID string) (errors []error) { // This function will find the nearest lower OCP version which version is under `Major.{minor-sub}`. // `strict` will find only the `Major.{minor-sub}` ones -func (vl *OpenShiftVersionList) FindNearestBackwardMinorVersion(version string, minorSub int64, strict bool) (vs *OpenShiftVersion, err error) { +func (vl *OpenShiftVersionTableList) FindNearestBackwardMinorVersion(version string, minorSub int64, strict bool) (vs *OpenShiftVersionTableOutput, err error) { var baseVersionSemVer *semver.Version baseVersionSemVer, err = semver.NewVersion(version) if err != nil { @@ -110,8 +148,8 @@ func (vl *OpenShiftVersionList) FindNearestBackwardMinorVersion(version string, } // Sort sort the version list from lower to higher (or reverse) -func (vl *OpenShiftVersionList) Sort(reverse bool) (nvl *OpenShiftVersionList, err error) { - versionListIndexMap := make(map[string]*OpenShiftVersion) +func (vl *OpenShiftVersionTableList) Sort(reverse bool) (nvl *OpenShiftVersionTableList, err error) { + versionListIndexMap := make(map[string]*OpenShiftVersionTableOutput) var semVerList []*semver.Version var vSemVer *semver.Version for _, version := range vl.OpenShiftVersions { @@ -129,12 +167,12 @@ func (vl *OpenShiftVersionList) Sort(reverse bool) (nvl *OpenShiftVersionList, e sort.Sort(semver.Collection(semVerList)) } - var sortedImageVersionList []*OpenShiftVersion + var sortedImageVersionList []*OpenShiftVersionTableOutput for _, semverVersion := range semVerList { sortedImageVersionList = append(sortedImageVersionList, versionListIndexMap[semverVersion.Original()]) } - nvl = &OpenShiftVersionList{ + nvl = &OpenShiftVersionTableList{ OpenShiftVersions: sortedImageVersionList, } @@ -143,8 +181,8 @@ func (vl *OpenShiftVersionList) Sort(reverse bool) (nvl *OpenShiftVersionList, e // FilterVersionsByMajorMinor filter the version list for all major/minor corresponding and returns a new `OpenShiftVersionList` struct // `strict` will find only the `Major.minor` ones -func (vl *OpenShiftVersionList) FilterVersionsSameMajorAndEqualOrLowerThanMinor(major int64, minor int64, strict bool) (nvl *OpenShiftVersionList, err error) { - var filteredVersions []*OpenShiftVersion +func (vl *OpenShiftVersionTableList) FilterVersionsSameMajorAndEqualOrLowerThanMinor(major int64, minor int64, strict bool) (nvl *OpenShiftVersionTableList, err error) { + var filteredVersions []*OpenShiftVersionTableOutput var semverVersion *semver.Version for _, version := range vl.OpenShiftVersions { if semverVersion, err = semver.NewVersion(version.Version); err != nil { @@ -155,7 +193,7 @@ func (vl *OpenShiftVersionList) FilterVersionsSameMajorAndEqualOrLowerThanMinor( } } - nvl = &OpenShiftVersionList{ + nvl = &OpenShiftVersionTableList{ OpenShiftVersions: filteredVersions, } @@ -163,11 +201,11 @@ func (vl *OpenShiftVersionList) FilterVersionsSameMajorAndEqualOrLowerThanMinor( } // FilterVersionsByMajorMinor filter the version list for all lower versions than the given one -func (vl *OpenShiftVersionList) FilterVersionsLowerThan(version string) (nvl *OpenShiftVersionList, err error) { +func (vl *OpenShiftVersionTableList) FilterVersionsLowerThan(version string) (nvl *OpenShiftVersionTableList, err error) { var givenSemVer *semver.Version givenSemVer, err = semver.NewVersion(version) - var filteredVersions []*OpenShiftVersion + var filteredVersions []*OpenShiftVersionTableOutput var semverVersion *semver.Version for _, version := range vl.OpenShiftVersions { if semverVersion, err = semver.NewVersion(version.Version); err != nil { @@ -177,13 +215,22 @@ func (vl *OpenShiftVersionList) FilterVersionsLowerThan(version string) (nvl *Op } } - nvl = &OpenShiftVersionList{ + nvl = &OpenShiftVersionTableList{ OpenShiftVersions: filteredVersions, } return } -func (vl *OpenShiftVersionList) Len() int { +func (vl *OpenShiftVersionTableList) Len() int { return len(vl.OpenShiftVersions) } + +func (vl *OpenShiftVersionTableList) Latest() (*OpenShiftVersionTableOutput, error) { + vl, err := vl.Sort(true) + if err != nil { + return nil, err + } + return vl.OpenShiftVersions[0], err + +} diff --git a/tests/utils/profilehandler/data_preparation.go b/tests/utils/profilehandler/data_preparation.go new file mode 100644 index 0000000000..3f266ce130 --- /dev/null +++ b/tests/utils/profilehandler/data_preparation.go @@ -0,0 +1,208 @@ +package profilehandler + +import ( + "bytes" + "fmt" + + "github.com/openshift/rosa/pkg/ocm" + "github.com/openshift/rosa/tests/utils/common" + con "github.com/openshift/rosa/tests/utils/common/constants" + "github.com/openshift/rosa/tests/utils/exec/rosacli" +) + +func PrepareVersion(client *rosacli.Client, versionRequirement string, channelGroup string, hcp bool) ( + *rosacli.OpenShiftVersionTableOutput, error) { + versionList, err := client.Version.ListAndReflectVersions(channelGroup, hcp) + if err != nil { + return nil, err + } + + if con.VersionLatestPattern.MatchString(versionRequirement) { + return versionList.Latest() + } else if con.VersionMajorMinorPattern.MatchString(versionRequirement) { + version, err := versionList.FindNearestBackwardMinorVersion(versionRequirement, 0, true) + return version, err + } else if con.VersionRawPattern.MatchString(versionRequirement) { + return &rosacli.OpenShiftVersionTableOutput{ + Version: versionRequirement, + }, nil + } else if con.VersionFlexyPattern.MatchString(versionRequirement) { + return nil, nil // TODO + } + return nil, fmt.Errorf("not supported version requirement: %s", versionRequirement) +} + +// PrepareNames will generate the name for cluster creation +// if longname is set, it will generate the long name with con.DefaultLongClusterNamelength +func PreparePrefix(profilePrefix string, nameLength int) string { + if nameLength > ocm.MaxClusterNameLength { + panic(fmt.Errorf("name length %d is longer than allowed max name length %d", nameLength, ocm.MaxClusterNameLength)) + } + return common.GenerateRandomName(profilePrefix, nameLength-len(profilePrefix)-1) +} + +func PrepareSubnetsDummy(vpcID string, region string, zones string) map[string][]string { + return map[string][]string{} +} + +func PrepareProxysDummy(vpcID string, region string, zones string) map[string]string { + return map[string]string{ + "https_proxy": "", + "http_proxy": "", + "no_proxy": "", + "ca_file_path": "", + } +} + +func PrepareKMSKeyDummy(region string) string { + return "" +} + +func PrepareSecurityGroupsDummy(vpcID string, region string, securityGroupCount int) []string { + return []string{} +} + +// PrepareAccountRoles will prepare account roles according to the parameters +// openshiftVersion must follow 4.15.2-x format +func PrepareAccountRoles(client *rosacli.Client, + namePrefix string, + hcp bool, + openshiftVersion string, + channelGroup string, + path string, + permissionsBoundary string) ( + accRoles *rosacli.AccountRolesUnit, err error) { + flags := GenerateAccountRoleCreationFlag(client, + namePrefix, + hcp, + openshiftVersion, + channelGroup, + path, + permissionsBoundary, + ) + + output, err := client.OCMResource.CreateAccountRole( + flags..., + ) + if err != nil { + err = fmt.Errorf("error happens when create account-roles, %s", output.String()) + return + } + accRoleList, output, err := client.OCMResource.ListAccountRole() + if err != nil { + err = fmt.Errorf("error happens when list account-roles, %s", output.String()) + return + } + roleDig := accRoleList.DigAccountRoles(namePrefix, hcp) + + return roleDig, nil + +} + +// PrepareOperatorRolesByOIDCConfig will prepare operator roles with OIDC config ID +// When sharedVPCRoleArn is not empty it will be set to the flag +func PrepareOperatorRolesByOIDCConfig(client *rosacli.Client, + namePrefix string, + oidcConfigID string, + roleArn string, + sharedVPCRoleArn string, + hcp bool) error { + flags := []string{ + "-y", + "--mode", "auto", + "--prefix", namePrefix, + "--role-arn", roleArn, + "--oidc-config-id", oidcConfigID, + } + if hcp { + flags = append(flags, "--hosted-cp") + } + if sharedVPCRoleArn != "" { + flags = append(flags, "--shared-vpc-role-arn", sharedVPCRoleArn) + } + _, err := client.OCMResource.CreateOperatorRoles( + flags..., + ) + return err +} + +func PrepareAdminUser() (string, string) { + userName := common.GenerateRandomString(10) + password := common.GenerateRandomStringWithSymbols(14) + return userName, password +} + +func PrepareAuditlogDummy() string { + return "" +} + +func PrepareOperatorRolesByCluster(client *rosacli.Client, cluster string) error { + flags := []string{ + "-y", + "--mode", "auto", + "--cluster", cluster, + } + _, err := client.OCMResource.CreateOperatorRoles( + flags..., + ) + return err +} + +// PrepareOIDCConfig will prepare the oidc config for the cluster, if the oidcConfigType="managed", roleArn and prefix won't be set +func PrepareOIDCConfig(client *rosacli.Client, + oidcConfigType string, + region string, + roleArn string, + prefix string) (string, error) { + var oidcConfigID string + var output bytes.Buffer + var err error + switch oidcConfigType { + case "managed": + output, err = client.OCMResource.CreateOIDCConfig( + "-o", "json", + "--mode", "auto", + "--region", region, + "--managed", + "-y", + ) + case "unmanaged": + output, err = client.OCMResource.CreateOIDCConfig( + "-o", "json", + "--mode", "auto", + "--prefix", prefix, + "--region", region, + "--role-arn", roleArn, + "--managed=false", + "-y", + ) + + default: + return "", fmt.Errorf("only 'managed' or 'unmanaged' oidc config is allowed") + } + if err != nil { + return oidcConfigID, err + } + parser := rosacli.NewParser() + oidcConfigID = parser.JsonData.Input(output).Parse().DigString("id") + return oidcConfigID, nil +} + +func PrepareOIDCProvider(client *rosacli.Client, oidcConfigID string) error { + _, err := client.OCMResource.CreateOIDCProvider( + "--mode", "auto", + "-y", + "--oidc-config-id", oidcConfigID, + ) + return err +} +func PrepareOIDCProviderByCluster(client *rosacli.Client, cluster string) error { + _, err := client.OCMResource.CreateOIDCProvider( + "--mode", "auto", + "-y", + "--cluster", cluster, + ) + return err +} + +func PrepareExternalAuthConfigDummy() {} diff --git a/tests/utils/profilehandler/interface.go b/tests/utils/profilehandler/interface.go new file mode 100644 index 0000000000..7e3c438097 --- /dev/null +++ b/tests/utils/profilehandler/interface.go @@ -0,0 +1,78 @@ +package profilehandler + +// Profile will map the profile settings from the profile yaml file +type Profile struct { + ChannelGroup string `yaml:"channel_group,omitempty"` + Name string `yaml:"as,omitempty"` + NamePrefix string `yaml:"name_prefix,omitempty"` + Region string `yaml:"region,omitempty"` + Version string `yaml:"version,omitempty"` + AccountRoleConfig *AccountRoleConfig `yaml:"account-role,omitempty"` + ClusterConfig *ClusterConfig `yaml:"cluster,omitempty"` +} + +// AccountRoleConfig will map the configuration of account roles from profile settings +type AccountRoleConfig struct { + Path string `yaml:"path,omitempty"` + PermissionBoundary string `yaml:"permission_boundary,omitempty"` +} + +// ClusterConfig will map the clsuter configuration from profile settings +type ClusterConfig struct { + BillingAccount string `yaml:"billing_account,omitempty" json:"billing_account,omitempty"` + Ec2MetadataHttpTokens string `yaml:"imdsv2,omitempty" json:"imdsv2,omitempty"` + InstanceType string `yaml:"instance_type,omitempty" json:"instance_type,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + OIDCConfig string `yaml:"oidc_config,omitempty" json:"oidc_config,omitempty"` + ProvisionShard string `yaml:"provision_shard,omitempty" json:"provision_shard,omitempty"` + Zones string `yaml:"zones,omitempty" json:"zones,omitempty"` + AdditionalSGNumber int `yaml:"additional_sg_number,omitempty" json:"additional_sg_number,omitempty"` + ExpirationTime int `yaml:"expiration_time,omitempty" json:"expiration_time,omitempty"` + NameLegnth int `default:"15" yaml:"name_length,omitempty" json:"name_length,omitempty"` + VolumeSize int `yaml:"volume_size,omitempty" json:"volume_size,omitempty"` + WorkerPoolReplicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"` + AdminEnabled bool `yaml:"admin_enabled,omitempty" json:"admin_enabled,omitempty"` + AuditLogForward bool `yaml:"auditlog_forward,omitempty" json:"auditlog_forward,omitempty"` + Autoscale bool `yaml:"autoscale,omitempty" json:"autoscale,omitempty"` + AutoscalerEnabled bool `yaml:"autoscaler_enabled,omitempty" json:"autoscaler_enabled,omitempty"` + BYOVPC bool `yaml:"byo_vpc,omitempty" json:"byo_vpc,omitempty"` + DomainPrefixEnabled bool `yaml:"domain_prefix_enabled,omitempty" json:"domain_prefix_enabled,omitempty"` + DisableUserWorKloadMonitoring bool `yaml:"disable_uwm,omitempty" json:"disable_uwm,omitempty"` + DisableSCPChecks bool `yaml:"disable_scp_checks,omitempty" json:"disable_scp_checks,omitempty"` + ExternalAuthConfig bool `yaml:"external_auth_config,omitempty" json:"external_auth_config,omitempty"` + EtcdEncryption bool `yaml:"etcd_encryption,omitempty" json:"etcd_encryption,omitempty"` + ETCDKMS bool `yaml:"etcd_kms,omitempty" json:"etcd_kms,omitempty"` + FIPS bool `yaml:"fips,omitempty" json:"fips,omitempty"` + HCP bool `yaml:"hcp,omitempty" json:"hypershift,omitempty"` + IngressCustomized bool `yaml:"ingress_customized,omitempty" json:"ingress_customized,omitempty"` + KMSKey bool `yaml:"kms_key,omitempty" json:"kms_key,omitempty"` + LabelEnabled bool `yaml:"label_enabled,omitempty" json:"label_enabled,omitempty"` + MultiAZ bool `yaml:"multi_az,omitempty" json:"multi_az,omitempty"` + NetWorkingSet bool `yaml:"networking,omitempty" json:"networking,omitempty"` + PrivateLink bool `yaml:"private_link,omitempty" json:"private_link,omitempty"` + Private bool `yaml:"private,omitempty" json:"private,omitempty"` + ProxyEnabled bool `yaml:"proxy_enabled,omitempty" json:"proxy_enabled,omitempty"` + STS bool `yaml:"sts,omitempty" json:"sts,omitempty"` + SharedVPC bool `yaml:"shared_vpc,omitempty" json:"shared_vpc,omitempty"` + TagEnabled bool `yaml:"tag_enabled,omitempty" json:"tag_enabled,omitempty"` +} + +// UserData will record the user data prepared for resource clean up +type UserData struct { + AccountRolesPrefix string `json:"account_roles_prefix,omitempty"` + AuditLogArn string `json:"audit_log,omitempty"` + KMSKey string `json:"kms_key,omitempty"` + OperatorRolesPrefix string `json:"operator_roles_prefix,omitempty"` + OIDCConfigID string `json:"oidc_config_id,omitempty"` + VpcID string `json:"vpc_id,omitempty"` +} + +// ClusterDetail will record basic cluster info to support other team's testing +type ClusterDetail struct { + APIURL string `json:"api_url,omitempty"` + ClusterID string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + ClusterType string `json:"cluster_type,omitempty"` + ConsoleURL string `json:"console_url,omitempty"` + InfraID string `json:"infra_id,omitempty"` +} diff --git a/tests/utils/profilehandler/parse_yaml.go b/tests/utils/profilehandler/parse_yaml.go new file mode 100644 index 0000000000..c307b377c2 --- /dev/null +++ b/tests/utils/profilehandler/parse_yaml.go @@ -0,0 +1,51 @@ +package profilehandler + +import ( + "log" + "os" + "path" + + "gopkg.in/yaml.v3" +) + +type profiles struct { + Profiles []*Profile `yaml:"profiles,omitempty"` +} + +func ParseProfiles(profilesDir string) map[string]*Profile { + files, err := os.ReadDir(profilesDir) + if err != nil { + log.Fatal(err) + } + + profileMap := make(map[string]*Profile) + for _, file := range files { + yfile, err := os.ReadFile(path.Join(profilesDir, file.Name())) + if err != nil { + log.Fatal(err) + } + + p := new(profiles) + err = yaml.Unmarshal(yfile, &p) + if err != nil { + log.Fatal(err) + } + + for _, theProfile := range p.Profiles { + profileMap[theProfile.Name] = theProfile + } + + } + + return profileMap +} + +func GetProfile(profileName string, profilesDir string) *Profile { + profileMap := ParseProfiles(profilesDir) + + if _, exist := profileMap[profileName]; !exist { + log.Fatalf("Can not find the profile %s in %s\n", profileName, profilesDir) + } + + return profileMap[profileName] +} diff --git a/tests/utils/profilehandler/profile_handler.go b/tests/utils/profilehandler/profile_handler.go new file mode 100644 index 0000000000..4564a8fd2f --- /dev/null +++ b/tests/utils/profilehandler/profile_handler.go @@ -0,0 +1,565 @@ +package profilehandler + +import ( + "fmt" + "strings" + "time" + + "github.com/openshift/rosa/pkg/ocm" + "github.com/openshift/rosa/tests/ci/config" + "github.com/openshift/rosa/tests/utils/common" + CON "github.com/openshift/rosa/tests/utils/common/constants" + ClusterConfigure "github.com/openshift/rosa/tests/utils/config" + "github.com/openshift/rosa/tests/utils/exec/rosacli" + "github.com/openshift/rosa/tests/utils/log" +) + +var client rosacli.Client + +func init() { + client = *rosacli.NewClient() +} + +func GetYAMLProfilesDir() string { + return config.Test.YAMLProfilesDir +} +func LoadProfileYamlFile(profileName string) *Profile { + p := GetProfile(profileName, GetYAMLProfilesDir()) + log.Logger.Infof("Loaded cluster profile configuration from origional profile %s : %v", profileName, *p) + log.Logger.Infof("Loaded cluster profile configuration from origional cluster %s : %v", profileName, *p.ClusterConfig) + log.Logger.Infof("Loaded cluster profile configuration from origional account-roles %s : %v", profileName, *p.AccountRoleConfig) + if p.NamePrefix == "" { + p.NamePrefix = CON.DefaultNamePrefix + } + return p +} + +func LoadProfileYamlFileByENV() *Profile { + if config.Test.TestProfile == "" { + panic(fmt.Errorf("ENV Variable TEST_PROFILE is empty, please make sure you set the env value")) + } + profile := LoadProfileYamlFile(config.Test.TestProfile) + + // Supporting global env setting to overrite profile settings + if config.Test.GlobalENV.ChannelGroup != "" { + log.Logger.Infof("Got global env settings for CHANNEL_GROUP, overwritten the profile setting with value %s", + config.Test.GlobalENV.ChannelGroup) + profile.ChannelGroup = config.Test.GlobalENV.ChannelGroup + } + if config.Test.GlobalENV.Version != "" { + log.Logger.Infof("Got global env settings for VERSION, overwritten the profile setting with value %s", + config.Test.GlobalENV.Version) + profile.Version = config.Test.GlobalENV.Version + } + if config.Test.GlobalENV.Region != "" { + log.Logger.Infof("Got global env settings for REGION, overwritten the profile setting with value %s", + config.Test.GlobalENV.Region) + profile.Region = config.Test.GlobalENV.Region + } + if config.Test.GlobalENV.ProvisionShard != "" { + log.Logger.Infof("Got global env settings for PROVISION_SHARD, overwritten the profile setting with value %s", + config.Test.GlobalENV.ProvisionShard) + profile.ClusterConfig.ProvisionShard = config.Test.GlobalENV.ProvisionShard + } + if config.Test.GlobalENV.NamePrefix != "" { + log.Logger.Infof("Got global env settings for NAME_PREFIX, overwritten the profile setting with value %s", + config.Test.GlobalENV.NamePrefix) + profile.NamePrefix = config.Test.GlobalENV.NamePrefix + } + + return profile +} + +// GenerateAccountRoleCreationFlag will generate account role creation flags +func GenerateAccountRoleCreationFlag(client *rosacli.Client, + namePrefix string, + hcp bool, + openshiftVersion string, + channelGroup string, + path string, + permissionsBoundary string) []string { + flags := []string{ + "--prefix", namePrefix, + "--mode", "auto", + "-y", + } + if openshiftVersion != "" { + majorVersion := common.SplitMajorVersion(openshiftVersion) + flags = append(flags, "--version", majorVersion) + } + if channelGroup != "" { + flags = append(flags, "--channel-group", channelGroup) + } + if hcp { + flags = append(flags, "--hosted-cp") + } else { + flags = append(flags, "--classic") + } + if path != "" { + flags = append(flags, "--path", path) + } + if permissionsBoundary != "" { + flags = append(flags, "--permissions-boundary", permissionsBoundary) + } + return flags + +} + +// GenerateClusterCreateFlags will generate cluster creation flags +func GenerateClusterCreateFlags(profile *Profile, client *rosacli.Client) ([]string, error) { + clusterName := PreparePrefix(profile.NamePrefix, profile.ClusterConfig.NameLegnth) + profile.ClusterConfig.Name = clusterName + var clusterConfiguration = new(ClusterConfigure.ClusterConfig) + var userData = new(UserData) + defer func() { + + // Record userdata + _, err := common.CreateFileWithContent(config.Test.UserDataFile, userData) + if err != nil { + log.Logger.Errorf("Cannot record user data: %s", err.Error()) + panic(fmt.Errorf("cannot record user data: %s", err.Error())) + } + + // Record cluster configuration + + _, err = common.CreateFileWithContent(config.Test.ClusterConfigFile, clusterConfiguration) + if err != nil { + log.Logger.Errorf("Cannot record cluster configuration: %s", err.Error()) + panic(fmt.Errorf("cannot record cluster configuration: %s", err.Error())) + } + }() + flags := []string{} + clusterConfiguration.Name = clusterName + + if profile.Version != "" { + version, err := PrepareVersion(client, profile.Version, profile.ChannelGroup, profile.ClusterConfig.HCP) + if err != nil { + return flags, err + } + profile.Version = version.Version + flags = append(flags, "--version", version.Version) + + clusterConfiguration.Version = &ClusterConfigure.Version{ + ChannelGroup: profile.ChannelGroup, + RawID: version.Version, + } + } + if profile.ChannelGroup != "" { + flags = append(flags, "--channel-group", profile.ChannelGroup) + if clusterConfiguration.Version == nil { + clusterConfiguration.Version = &ClusterConfigure.Version{} + } + clusterConfiguration.Version.ChannelGroup = profile.ChannelGroup + } + if profile.Region != "" { + flags = append(flags, "--region", profile.Region) + clusterConfiguration.Region = profile.Region + } + if profile.ClusterConfig.DomainPrefixEnabled { + flags = append(flags, + "--domain-prefix", common.TrimNameByLength(clusterName, ocm.MaxClusterDomainPrefixLength), + ) + + } + if profile.ClusterConfig.STS { + var accRoles *rosacli.AccountRolesUnit + accountRolePrefix := common.TrimNameByLength(clusterName, CON.MaxRolePrefixLength) + log.Logger.Infof("Got sts set to true. Going to prepare Account roles with prefix %s", accountRolePrefix) + accRoles, err := PrepareAccountRoles( + client, accountRolePrefix, + profile.ClusterConfig.HCP, + profile.Version, + profile.ChannelGroup, + profile.AccountRoleConfig.Path, + profile.AccountRoleConfig.PermissionBoundary, + ) + if err != nil { + log.Logger.Errorf("Got error happens when prepare account roles: %s", err.Error()) + return flags, err + } + userData.AccountRolesPrefix = accountRolePrefix + flags = append(flags, + "--role-arn", accRoles.InstallerRole, + "--support-role-arn", accRoles.SupportRole, + "--worker-iam-role", accRoles.WorkerRole, + ) + clusterConfiguration.Sts = true + clusterConfiguration.Aws = &ClusterConfigure.AWS{ + Sts: ClusterConfigure.Sts{ + RoleArn: accRoles.InstallerRole, + SupportRoleArn: accRoles.SupportRole, + WorkerRoleArn: accRoles.WorkerRole, + }, + } + if !profile.ClusterConfig.HCP { + flags = append(flags, + "--controlplane-iam-role", accRoles.ControlPlaneRole, + ) + clusterConfiguration.Aws.Sts.ControlPlaneRoleArn = accRoles.ControlPlaneRole + } + operatorRolePrefix := accountRolePrefix + if profile.ClusterConfig.OIDCConfig != "" { + oidcConfigPrefix := common.TrimNameByLength(clusterName, CON.MaxOIDCConfigPrefixLength) + log.Logger.Infof("Got oidc config setting, going to prepare the %s oidc with prefix %s", + profile.ClusterConfig.OIDCConfig, oidcConfigPrefix) + var oidcConfigID string + oidcConfigID, err = PrepareOIDCConfig(client, profile.ClusterConfig.OIDCConfig, + profile.Region, accRoles.InstallerRole, oidcConfigPrefix) + if err != nil { + return flags, err + } + err = PrepareOIDCProvider(client, oidcConfigID) + if err != nil { + return flags, err + } + err = PrepareOperatorRolesByOIDCConfig(client, operatorRolePrefix, + oidcConfigID, accRoles.InstallerRole, "", profile.ClusterConfig.HCP) + if err != nil { + return flags, err + } + flags = append(flags, "--oidc-config-id", oidcConfigID) + clusterConfiguration.Aws.Sts.OidcConfigID = oidcConfigID + userData.OIDCConfigID = oidcConfigID + } + + flags = append(flags, "--operator-roles-prefix", operatorRolePrefix) + clusterConfiguration.Aws.Sts.OperatorRolesPrefix = operatorRolePrefix + userData.OperatorRolesPrefix = operatorRolePrefix + } + if profile.ClusterConfig.AdditionalSGNumber != 0 { + PrepareSecurityGroupsDummy("", profile.Region, profile.ClusterConfig.AdditionalSGNumber) + } + if profile.ClusterConfig.AdminEnabled { + // Comment below part due to OCM-7112 + log.Logger.Infof("Day1 admin is enabled. Going to generate the admin user and password and record in %s", + config.Test.ClusterAdminFile) + _, password := PrepareAdminUser() // Unuse cluster-admin right now + userName := "cluster-admin" + + flags = append(flags, + "--create-admin-user", + "--cluster-admin-password", password, + // "--cluster-admin-user", userName, + ) + common.CreateFileWithContent(config.Test.ClusterAdminFile, fmt.Sprintf("%s:%s", userName, password)) + } + if profile.ClusterConfig.AuditLogForward { + PrepareAuditlogDummy() + clusterConfiguration.AuditLogArn = "" + } + if profile.ClusterConfig.Autoscale { + minReplicas := "3" + maxRelicas := "6" + flags = append(flags, + "--enable-autoscaling", + "--min-replicas", minReplicas, + "--max-replicas", maxRelicas, + ) + clusterConfiguration.Autoscaling = &ClusterConfigure.Autoscaling{ + Enabled: true, + } + clusterConfiguration.Nodes = &ClusterConfigure.Nodes{ + MinReplicas: minReplicas, + MaxReplicas: maxRelicas, + } + } + if profile.ClusterConfig.WorkerPoolReplicas != 0 { + flags = append(flags, "--replicas", fmt.Sprintf("%v", profile.ClusterConfig.WorkerPoolReplicas)) + clusterConfiguration.Nodes = &ClusterConfigure.Nodes{ + Replicas: fmt.Sprintf("%v", profile.ClusterConfig.WorkerPoolReplicas), + } + } + + if profile.ClusterConfig.IngressCustomized { + clusterConfiguration.IngressConfig = &ClusterConfigure.IngressConfig{ + DefaultIngressRouteSelector: "app1=test1,app2=test2", + DefaultIngressExcludedNamespaces: "test-ns1,test-ns2", + DefaultIngressWildcardPolicy: "WildcardsDisallowed", + DefaultIngressNamespaceOwnershipPolicy: "Strict", + } + flags = append(flags, + "--default-ingress-route-selector", clusterConfiguration.IngressConfig.DefaultIngressRouteSelector, + "--default-ingress-excluded-namespaces", clusterConfiguration.IngressConfig.DefaultIngressExcludedNamespaces, + "--default-ingress-wildcard-policy", clusterConfiguration.IngressConfig.DefaultIngressWildcardPolicy, + "--default-ingress-namespace-ownership-policy", clusterConfiguration.IngressConfig.DefaultIngressNamespaceOwnershipPolicy, + ) + } + if profile.ClusterConfig.AutoscalerEnabled { + autoscaler := &ClusterConfigure.Autoscaler{ + AutoscalerBalanceSimilarNodeGroups: true, + AutoscalerSkipNodesWithLocalStorage: true, + AutoscalerLogVerbosity: "4", + AutoscalerMaxPodGracePeriod: "0", + AutoscalerPodPriorityThreshold: "0", + AutoscalerIgnoreDaemonsetsUtilization: true, + AutoscalerMaxNodeProvisionTime: "10m", + AutoscalerBalancingIgnoredLabels: "aaa", + AutoscalerMaxNodesTotal: "1000", + AutoscalerMinCores: "0", + AutoscalerMaxCores: "100", + AutoscalerMinMemory: "0", + AutoscalerMaxMemory: "4096", + // AutoscalerGpuLimit: "1", + AutoscalerScaleDownEnabled: true, + AutoscalerScaleDownUtilizationThreshold: "0.5", + AutoscalerScaleDownDelayAfterAdd: "10s", + AutoscalerScaleDownDelayAfterDelete: "10s", + AutoscalerScaleDownDelayAfterFailure: "10s", + // AutoscalerScaleDownUnneededTime: "3m", + } + flags = append(flags, + "--autoscaler-balance-similar-node-groups", + "--autoscaler-skip-nodes-with-local-storage", + "--autoscaler-log-verbosity", autoscaler.AutoscalerLogVerbosity, + "--autoscaler-max-pod-grace-period", autoscaler.AutoscalerMaxPodGracePeriod, + "--autoscaler-pod-priority-threshold", autoscaler.AutoscalerPodPriorityThreshold, + "--autoscaler-ignore-daemonsets-utilization", + "--autoscaler-max-node-provision-time", autoscaler.AutoscalerMaxNodeProvisionTime, + "--autoscaler-balancing-ignored-labels", autoscaler.AutoscalerBalancingIgnoredLabels, + "--autoscaler-max-nodes-total", autoscaler.AutoscalerMaxNodesTotal, + "--autoscaler-min-cores", autoscaler.AutoscalerMinCores, + "--autoscaler-max-cores", autoscaler.AutoscalerMaxCores, + "--autoscaler-min-memory", autoscaler.AutoscalerMinMemory, + "--autoscaler-max-memory", autoscaler.AutoscalerMaxMemory, + // "--autoscaler-gpu-limit", autoscaler.AutoscalerGpuLimit, + "--autoscaler-scale-down-enabled", + // "--autoscaler-scale-down-unneeded-time", autoscaler.AutoscalerScaleDownUnneededTime, + "--autoscaler-scale-down-utilization-threshold", autoscaler.AutoscalerScaleDownUtilizationThreshold, + "--autoscaler-scale-down-delay-after-add", autoscaler.AutoscalerScaleDownDelayAfterAdd, + "--autoscaler-scale-down-delay-after-delete", autoscaler.AutoscalerScaleDownDelayAfterDelete, + "--autoscaler-scale-down-delay-after-failure", autoscaler.AutoscalerScaleDownDelayAfterFailure, + ) + + clusterConfiguration.Autoscaler = autoscaler + } + if profile.ClusterConfig.BYOVPC { + PrepareSubnetsDummy("", profile.Region, "") + } + if profile.ClusterConfig.BillingAccount != "" { + flags = append(flags, " --billing-account", profile.ClusterConfig.BillingAccount) + clusterConfiguration.BillingAccount = profile.ClusterConfig.BillingAccount + } + if profile.ClusterConfig.DisableSCPChecks { + flags = append(flags, "--disable-scp-checks") + } + if profile.ClusterConfig.DisableUserWorKloadMonitoring { + flags = append(flags, "--disable-workload-monitoring") + } + if profile.ClusterConfig.ETCDKMS { + PrepareKMSKeyDummy(profile.Region) + } + if profile.ClusterConfig.Ec2MetadataHttpTokens != "" { + flags = append(flags, "--ec2-metadata-http-tokens", profile.ClusterConfig.Ec2MetadataHttpTokens) + clusterConfiguration.Ec2MetadataHttpTokens = profile.ClusterConfig.Ec2MetadataHttpTokens + } + if profile.ClusterConfig.EtcdEncryption { + flags = append(flags, "--etcd-encryption") + clusterConfiguration.EtcdEncryption = profile.ClusterConfig.EtcdEncryption + + } + if profile.ClusterConfig.ExternalAuthConfig { + PrepareExternalAuthConfigDummy() + } + + if profile.ClusterConfig.FIPS { + flags = append(flags, "--fips") + } + if profile.ClusterConfig.HCP { + flags = append(flags, "--hosted-cp") + } + if profile.ClusterConfig.InstanceType != "" { + flags = append(flags, "--compute-machine-type", profile.ClusterConfig.InstanceType) + } + if profile.ClusterConfig.KMSKey { + PrepareKMSKeyDummy(profile.Region) + clusterConfiguration.Encryption = &ClusterConfigure.Encryption{ + KmsKeyArn: "", // placeHolder + } + } + if profile.ClusterConfig.LabelEnabled { + dmpLabel := "test-label/openshift.io=,test-label=testvalue" + flags = append(flags, "--worker-mp-labels", dmpLabel) + clusterConfiguration.DefaultMpLabels = dmpLabel + } + if profile.ClusterConfig.MultiAZ { + flags = append(flags, "--multi-az") + clusterConfiguration.MultiAZ = profile.ClusterConfig.MultiAZ + } + if profile.ClusterConfig.NetWorkingSet { + networking := &ClusterConfigure.Networking{ + MachineCIDR: "10.2.0.0/16", + PodCIDR: "192.168.0.0/18", + ServiceCIDR: "172.31.0.0/24", + HostPrefix: "25", + } + flags = append(flags, + "--machine-cidr", networking.MachineCIDR, // Placeholder, it should be vpc CIDR + "--service-cidr", networking.ServiceCIDR, + "--pod-cidr", networking.PodCIDR, + "--host-prefix", networking.HostPrefix, + ) + clusterConfiguration.Networking = networking + } + if profile.ClusterConfig.Private { + flags = append(flags, "--private") + clusterConfiguration.Private = profile.ClusterConfig.Private + } + if profile.ClusterConfig.PrivateLink { + flags = append(flags, "--private-link") + clusterConfiguration.PrivateLink = profile.ClusterConfig.PrivateLink + } + if profile.ClusterConfig.ProvisionShard != "" { + flags = append(flags, "--properties", fmt.Sprintf("provision_shard_id:%s", profile.ClusterConfig.ProvisionShard)) + clusterConfiguration.Properties = &ClusterConfigure.Properties{ + ProvisionShardID: profile.ClusterConfig.ProvisionShard, + } + } + if profile.ClusterConfig.ProxyEnabled { + PrepareProxysDummy("", profile.Region, "") + + clusterConfiguration.Proxy = &ClusterConfigure.Proxy{ + Enabled: profile.ClusterConfig.ProxyEnabled, + } + + } + if profile.ClusterConfig.SharedVPC { + //Placeholder for shared vpc, need to research what to be set here + } + if profile.ClusterConfig.TagEnabled { + tags := "test-tag:tagvalue,qe-managed:true" + flags = append(flags, "--tags", tags) + clusterConfiguration.Tags = tags + } + if profile.ClusterConfig.VolumeSize != 0 { + diskSize := fmt.Sprintf("%dGiB", profile.ClusterConfig.VolumeSize) + flags = append(flags, "--worker-disk-size", diskSize) + clusterConfiguration.WorkerDiskSize = diskSize + } + if profile.ClusterConfig.Zones != "" && !profile.ClusterConfig.BYOVPC { + flags = append(flags, " --availability-zones", profile.ClusterConfig.Zones) + clusterConfiguration.AvailabilityZones = profile.ClusterConfig.Zones + } + + return flags, nil +} +func WaitForClusterReady(client *rosacli.Client, cluster string, timeoutMin int) error { + + endTime := time.Now().Add(time.Duration(timeoutMin) * time.Minute) + sleepTime := 0 + for time.Now().Before(endTime) { + output, err := client.Cluster.DescribeClusterAndReflect(cluster) + if err != nil { + return err + } + switch output.State { + case CON.Ready: + log.Logger.Infof("Cluster %s is ready now.", cluster) + return nil + case CON.Uninstalling: + return fmt.Errorf("cluster %s is %s now. Cannot wait for it ready", + cluster, CON.Uninstalling) + default: + if strings.Contains(output.State, CON.Error) { + log.Logger.Errorf("Cluster is in %s status now. Recording the installation log", CON.Error) + RecordClusterInstallationLog(client, cluster) + return fmt.Errorf("cluster %s is in %s state with reason: %s", + cluster, CON.Error, output.State) + } + if strings.Contains(output.State, CON.Pending) || + strings.Contains(output.State, CON.Installing) || + strings.Contains(output.State, CON.Validating) { + time.Sleep(2 * time.Minute) + continue + } + if strings.Contains(output.State, CON.Waiting) { + log.Logger.Infof("Cluster is in status of %v, wait for ready", CON.Waiting) + if sleepTime >= 6 { + return fmt.Errorf("cluster stuck to %s status for more than 6 mins. Check the user data preparation for roles", output.State) + } + sleepTime += 2 + time.Sleep(2 * time.Minute) + continue + } + return fmt.Errorf("unknown cluster state %s", output.State) + } + + } + + return fmt.Errorf("timeout for cluster ready waiting after %d mins", timeoutMin) +} + +func ReverifyClusterNetwork(client *rosacli.Client, clusterID string) error { + log.Logger.Infof("verify network of cluster %s ", clusterID) + _, err := client.NetworkVerifier.CreateNetworkVerifierWithCluster(clusterID) + return err +} + +func RecordClusterInstallationLog(client *rosacli.Client, cluster string) error { + output, err := client.Cluster.InstallLog(cluster) + if err != nil { + return err + } + _, err = common.CreateFileWithContent(config.Test.ClusterInstallLogArtifactFile, output.String()) + return err +} + +func CreateClusterByProfile(profile *Profile, client *rosacli.Client, waitForClusterReady bool) (*rosacli.ClusterDescription, error) { + clusterDetail := new(ClusterDetail) + + flags, err := GenerateClusterCreateFlags(profile, client) + if err != nil { + log.Logger.Errorf("Error happened when generate flags: %s", err.Error()) + return nil, err + } + log.Logger.Infof("User data and flags preparation finished") + _, err, createCMD := client.Cluster.Create(profile.ClusterConfig.Name, flags...) + if err != nil { + return nil, err + } + common.CreateFileWithContent(config.Test.CreateCommandFile, createCMD) + log.Logger.Info("Cluster created succesfully") + description, err := client.Cluster.DescribeClusterAndReflect(profile.ClusterConfig.Name) + if err != nil { + return nil, err + } + defer func() { + log.Logger.Info("Going to record the necessary information") + common.CreateFileWithContent(config.Test.ClusterDetailFile, clusterDetail) + common.CreateFileWithContent(config.Test.ClusterIDFile, description.ID) // Temporary recoding file to make it compatible to existing jobs + common.CreateFileWithContent(config.Test.ClusterNameFile, description.Name) // Temporary recoding file to make it compatible to existing jobs + common.CreateFileWithContent(config.Test.APIURLFile, description.APIURL) // Temporary recoding file to make it compatible to existing jobs + common.CreateFileWithContent(config.Test.ConsoleUrlFile, description.ConsoleURL) // Temporary recoding file to make it compatible to existing jobs + common.CreateFileWithContent(config.Test.InfraIDFile, description.InfraID) // Temporary recoding file to make it compatible to existing jobs + common.CreateFileWithContent(config.Test.ClusterTypeFile, "rosa") // Temporary recoding file to make it compatible to existing jobs + }() + clusterDetail.ClusterID = description.ID + clusterDetail.ClusterName = description.Name + clusterDetail.ClusterType = "rosa" + + // Need to do the post step when cluster has no oidcconfig enabled + if profile.ClusterConfig.OIDCConfig == "" { + err = PrepareOIDCProviderByCluster(client, description.ID) + if err != nil { + return description, err + } + err = PrepareOperatorRolesByCluster(client, description.ID) + if err != nil { + return description, err + } + + } + // if profile.ClusterConfig.BYOVPC { + // log.Logger.Infof("Reverify the network for the cluster %s to make sure it can be parsed", description.ID) + // ReverifyClusterNetwork(client, description.ID) + // } + if waitForClusterReady { + log.Logger.Infof("Waiting for the cluster %s to ready", description.ID) + err = WaitForClusterReady(client, description.ID, config.Test.GlobalENV.ClusterWaitingTime) + if err != nil { + return description, err + } + description, err = client.Cluster.DescribeClusterAndReflect(profile.ClusterConfig.Name) + } + + return description, err +}