diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml index 867a382e..b31436c5 100644 --- a/roles/config/cluster/common/defaults/main.yml +++ b/roles/config/cluster/common/defaults/main.yml @@ -37,6 +37,12 @@ default_cluster_kts: KEYTRUSTEE_PASSIVE_SERVER: {} DB_PASSIVE: {} +default_cluster_ecs: + name: ECS + services: [DOCKER, ECS] + +ecs_databases: [ALERTS, CLASSIC_CLUSTERS, CLUSTER_ACCESS_MANAGER, CLUSTER_PROXY, DEX, DWX, ENV, LIFTIE, MLX, RESOURCEPOOL_MANAGER, UMS] + default_cluster_compute: base_cluster: data_context: SDX diff --git a/roles/config/cluster/ecs/meta/main.yml b/roles/config/cluster/ecs/meta/main.yml new file mode 100644 index 00000000..cf20f5e5 --- /dev/null +++ b/roles/config/cluster/ecs/meta/main.yml @@ -0,0 +1,21 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +dependencies: + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.config.cluster.common + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/config/cluster/ecs/tasks/main.yml b/roles/config/cluster/ecs/tasks/main.yml new file mode 100644 index 00000000..61854262 --- /dev/null +++ b/roles/config/cluster/ecs/tasks/main.yml @@ -0,0 +1,44 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +# This variable is used by other roles +# please take care when changing it +- set_fact: + databases: "{{ database_defaults | combine(cluster.databases | default({}), recursive=True) }}" + +- name: Retrieve repository metadata + include_role: + name: cloudera.cluster.deployment.repometa + vars: + repositories: "{{ cluster.repositories | default({}) }}" + +- name: Reset custom configuration dictionary + set_fact: + custom_configs: {} + +- name: Generate custom configurations + set_fact: + custom_configs: "{{ custom_configs | combine(lookup('template', custom_config_template.template) | from_yaml, recursive=True) }}" + loop: "{{ custom_config_templates }}" + loop_control: + loop_var: custom_config_template + when: custom_config_template.condition | default(True) + +# This variable is used by other roles +# please take care when changing it +- name: Merge custom configurations + set_fact: + merged_configs: "{{ custom_configs | combine(cluster.configs | default({}), recursive=True) }}" diff --git a/roles/config/cluster/ecs/templates/configs/ecs.j2 b/roles/config/cluster/ecs/templates/configs/ecs.j2 new file mode 100644 index 00000000..f8ae90a0 --- /dev/null +++ b/roles/config/cluster/ecs/templates/configs/ecs.j2 @@ -0,0 +1,12 @@ +--- +DOCKER: + SERVICEWIDE: + docker_images_destination_registry_user: registry-user +ECS: + SERVICEWIDE: + k8s_webui_secret_admin_token: ecs-k8s_webui_secret_admin_token + cp_prometheus_ingress_user: cloudera-manager + infra_prometheus_ingress_user: cloudera-manager + longhorn_replication: 1 + lsoDataPath: /ecs/storage + app_domain: "{{ cluster.application_domain }}" \ No newline at end of file diff --git a/roles/config/cluster/ecs/vars/main.yml b/roles/config/cluster/ecs/vars/main.yml new file mode 100644 index 00000000..0590e30d --- /dev/null +++ b/roles/config/cluster/ecs/vars/main.yml @@ -0,0 +1,19 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +custom_config_templates: +# Custom configurations for ECS + - template: configs/ecs.j2 diff --git a/roles/deployment/cluster/defaults/main.yml b/roles/deployment/cluster/defaults/main.yml new file mode 100644 index 00000000..e69de29b diff --git a/roles/deployment/cluster/meta/main.yml b/roles/deployment/cluster/meta/main.yml index 468f7203..6794e8de 100644 --- a/roles/deployment/cluster/meta/main.yml +++ b/roles/deployment/cluster/meta/main.yml @@ -15,6 +15,7 @@ --- dependencies: - role: cloudera.cluster.cloudera_manager.api_hosts + - role: cloudera.cluster.deployment.credential - role: cloudera.cluster.infrastructure.ca_common - role: cloudera.cluster.prereqs.local_accounts_common diff --git a/roles/deployment/cluster/tasks/create_ecs.yml b/roles/deployment/cluster/tasks/create_ecs.yml new file mode 100644 index 00000000..218193bb --- /dev/null +++ b/roles/deployment/cluster/tasks/create_ecs.yml @@ -0,0 +1,139 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Generate complete ecs cluster configs + include_role: + name: cloudera.cluster.config.cluster.ecs + +- name: Create databases and users + include_role: + name: cloudera.cluster.deployment.databases + vars: + services: "{{ cluster.ecs_databases | default(ecs_databases) }}" + +# Add PVC parcel repo - assume that this has been set via REMOTE_PARCEL_REPO_URLS in the cm config. We can add this in later if needed + +- name: Create ECS cluster + cloudera.cluster.cm_api: + endpoint: /clusters + method: POST + body: "{{ lookup('template', 'cluster_template/ecs/clusters.j2', convert_data=False) }}" + register: cluster_creation_result + failed_when: '"Status code was 400" in cluster_creation_result.msg' + when: not cluster_template_dry_run + +- name: Add hosts to ECS cluster + cloudera.cluster.cm_api: + endpoint: /clusters/{{ cluster.name | urlencode() }}/hosts + method: POST + body: "{{ lookup('template', 'cluster_template/ecs/hosts.j2', convert_data=False) }}" + register: cluster_hosts_result + failed_when: '"Status code was 400" in cluster_hosts_result.msg' + when: not cluster_template_dry_run + +- name: Add services to ECS cluster + cloudera.cluster.cm_api: + endpoint: /clusters/{{ cluster.name | urlencode() }}/services + method: POST + body: "{{ lookup('template', 'cluster_template/ecs/services.j2', convert_data=False) }}" + register: cluster_services_result + failed_when: '"Status code was 400" in cluster_services_result.msg' + when: not cluster_template_dry_run + +- name: Get CM config + cloudera.cluster.cm_api: + endpoint: "cm/config" + method: GET + register: cm_config + +- set_fact: + parcel_repos: "{{ (cm_config.json | json_query('items[?name==`REMOTE_PARCEL_REPO_URLS`].value') | default(['']))[0].split(',') | default([]) | union(cluster.repositories) }}" + +- name: Update parcelrepos + cloudera.cluster.cm_api: + endpoint: "cm/config" + body: + items: + - name: REMOTE_PARCEL_REPO_URLS + value: "{{ parcel_repos | join(', ') }}" + method: PUT + +- name: Refresh parcel repos command + cloudera.cluster.cm_api: + endpoint: "/cm/commands/refreshParcelRepos" + method: POST + +- set_fact: + new_parcel_version: "{{ products | json_query('[?product==`ECS`].version') | first }}" + +- name: Start Parcel Download + cloudera.cluster.cm_api: + endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/startDownload" + method: POST + +- name: Wait for parcels to download + cloudera.cluster.cm_api: + endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }} + register: parcels_response + until: parcels_response.json.stage in ("DOWNLOADED", "DISTRIBUTED", "ACTIVATED") + retries: "{{ parcel_poll_max_retries | default(30) }}" + delay: "{{ parcel_poll_duration | default(60) }}" + +- name: Start Parcel Distribution + cloudera.cluster.cm_api: + endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/startDistribution" + method: POST + +- name: Wait for parcels to become distributed to all hosts + cloudera.cluster.cm_api: + endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }} + register: parcels_response + until: parcels_response.json.stage in ("DISTRIBUTED", "ACTIVATED") + retries: "{{ parcel_poll_max_retries | default(30) }}" + delay: "{{ parcel_poll_duration | default(60) }}" + +- name: Activate Parcels + cloudera.cluster.cm_api: + endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/activate" + method: POST + +- name: Wait for parcels to become distributed to all hosts + cloudera.cluster.cm_api: + endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }} + register: parcels_response + until: parcels_response.json.stage in ("ACTIVATED") + retries: "{{ parcel_poll_max_retries | default(30) }}" + delay: "{{ parcel_poll_duration | default(60) }}" + +- name: Generate custom values + set_fact: + custom_values: "{{ lookup('template', 'cluster_template/ecs/controlPlaneValues.j2') | from_yaml | combine(cluster.controlplane_config, recursive=True) }}" + +- name: Show custom values + debug: + var: custom_values + +- name: Install ECS cluster + cloudera.cluster.cm_api: + endpoint: /controlPlanes/commands/installEmbeddedControlPlane + method: POST + body: + remoteRepoUrl: "{{ cluster.pvc_repository }}" + experienceClusterName: "{{ cluster.name }}" + datalakeClusterName: "{{ cluster.datalake }}" + valuesYaml: "{{ custom_values | to_yaml }}" + register: controlplane_response + diff --git a/roles/deployment/cluster/tasks/main.yml b/roles/deployment/cluster/tasks/main.yml index 914e9e99..42ee4f6e 100644 --- a/roles/deployment/cluster/tasks/main.yml +++ b/roles/deployment/cluster/tasks/main.yml @@ -114,6 +114,18 @@ - cluster.type | default(default_cluster_type) == 'compute' - cluster.name not in existing_clusters +- name: Create ECS clusters + include_tasks: create_ecs.yml + loop: "{{ definition.clusters }}" + loop_control: + loop_var: _cluster + vars: + cluster: "{{ default_cluster_ecs | combine(_cluster) }}" + when: + - cluster.type | default(default_cluster_type) == 'ecs' + - cluster.name not in existing_clusters + + - name: Restart Cloudera Management Service cloudera.cluster.cm_api: endpoint: /cm/service/commands/restart diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 new file mode 100644 index 00000000..b438561e --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2 @@ -0,0 +1,12 @@ +{ +"items": +[ + { + "name" : "{{ cluster.name }}", + "displayName": "{{ cluster.name }}", + "version" : "EXPERIENCE1", + "fullVersion": "{{ products | cloudera.cluster.get_product_version('ECS') }}", + "clusterType": "EXPERIENCE_CLUSTER" + } +] +} \ No newline at end of file diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/cm_api.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/cm_api.j2 new file mode 100644 index 00000000..88ab1447 --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/ecs/cm_api.j2 @@ -0,0 +1,90 @@ +{% macro ApiConfig(k, v, force_uppercase_keys=False) -%} + {%- set config = {} -%} + {%- if force_uppercase_keys -%} + {{ config.update({"name": k | upper}) }} + {%- else -%} + {{ config.update({"name": k}) }} + {%- endif -%} + {# the extra to_json filter renders a dict into properly escaped JSON string for the CM API call #} + {%- if v is mapping -%} + {{ config.update({"value": v | to_json}) }} + {%- else -%} + {{ config.update({"value": v}) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiConfigList(configs, force_uppercase_keys=False) -%} + {%- set config_list = { "items": [] } -%} + {%- if configs is mapping -%} + {%- for (k, v) in configs.items() -%} + {%- set config = ApiConfig(k, v, force_uppercase_keys) -%} + {{ config_list["items"].append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfig(key, service_type, role_type, value, value_type="value") -%} + {%- set config = {} -%} + {{ config.update({"name": key}) }} + {%- if value_type == "ref" or value_type == "value" -%} + {{ config.update({ value_type: value }) }} + {%- else -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {{ config.update({ "variable": var_name | upper }) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfigList(service_merged_configs, service_type, role_type) -%} + {%- set config_list = [] -%} + {%- if service_type in service_merged_configs -%} + {%- if service_merged_configs[service_type][role_type] is mapping -%} + {%- for (k, v) in service_merged_configs[service_type][role_type].items() -%} + {%- if k.endswith("_service") and not "suppression" in k -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "ref") -%} + {%- else -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%} + {%- endif -%} + {{ config_list.append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiService(service) -%} +{} +{%- endmacro %} + +{% macro ApiServiceList(services) -%} + {%- set api_service_list = { "items": [] } -%} + {%- if services -%} + {%- for service in services -%} + {%- set api_service = ApiService(service) -%} + {{ api_service_list["items"].append(api_service | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ api_service_list | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMapping(auth_type, user_mapping) %} + {%- set external_user_mapping = {'name': user_mapping.group, 'type': auth_type, 'authRoles': []} -%} + {%- for role in user_mapping.roles -%} + {{ external_user_mapping['authRoles'].append(ApiAuthRoleRef(role) | from_json) }} + {%- endfor -%} + {{ external_user_mapping | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMappingList(auth_type, auth_user_mappings) %} + {%- set external_user_mapping_list = { "items": [] } -%} + {%- for user_mapping in auth_user_mappings -%} + {{ external_user_mapping_list["items"].append(ApiExternalUserMapping(auth_type, user_mapping) | from_json) }} + {%- endfor -%} + {{ external_user_mapping_list | to_json }} +{%- endmacro %} + +{% macro ApiAuthRoleRef(name) -%} + { "uuid": {{ auth_role_uuids[name] | to_json }} } +{%- endmacro %} diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 new file mode 100644 index 00000000..5f5fbe01 --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/ecs/controlPlaneValues.j2 @@ -0,0 +1,199 @@ +Services: + thunderheadenvironment: + Config: + database: + name: {{ databases.ENV.name }} + mode: {{ databases.ENV.mode | default(ecs_database_mode) }} + host: {{ databases.ENV.host }} + port: {{ databases.ENV.port }} + username: {{ databases.ENV.user }} + password: {{ databases.ENV.password }} + clusterproxy: + Config: + database: + name: {{ databases.CLUSTER_PROXY.name }} + mode: {{ databases.CLUSTER_PROXY.mode | default(ecs_database_mode) }} + host: {{ databases.CLUSTER_PROXY.host }} + port: {{ databases.CLUSTER_PROXY.port }} + username: {{ databases.CLUSTER_PROXY.user }} + password: {{ databases.CLUSTER_PROXY.password }} + classicclusters: + Config: + database: + name: {{ databases.CLASSIC_CLUSTERS.name }} + mode: {{ databases.CLASSIC_CLUSTERS.mode | default(ecs_database_mode) }} + host: {{ databases.CLASSIC_CLUSTERS.host }} + port: {{ databases.CLASSIC_CLUSTERS.port }} + username: {{ databases.CLASSIC_CLUSTERS.user }} + password: {{ databases.CLASSIC_CLUSTERS.password }} + mlxcontrolplaneapp: + Config: + database: + name: {{ databases.MLX.name }} + mode: {{ databases.MLX.mode | default(ecs_database_mode) }} + host: {{ databases.MLX.host }} + port: {{ databases.MLX.port }} + username: {{ databases.MLX.user }} + password: {{ databases.MLX.password }} + cpxliftie: + Config: + database: + name: {{ databases.LIFTIE.name }} + mode: {{ databases.LIFTIE.mode | default(ecs_database_mode) }} + host: {{ databases.LIFTIE.host }} + port: {{ databases.LIFTIE.port }} + username: {{ databases.LIFTIE.user }} + password: {{ databases.LIFTIE.password }} + monitoringapp: + Config: + database: + name: {{ databases.ALERTS.name }} + mode: {{ databases.ALERTS.mode | default(ecs_database_mode) }} + host: {{ databases.ALERTS.host }} + port: {{ databases.ALERTS.port }} + username: {{ databases.ALERTS.user }} + password: {{ databases.ALERTS.password }} + prometheus: + ingressForCM: + enabled: false + # The control plane prometheus ingress hostname will be .apps. + subdomain: {{ cluster.prometheus_subdomain | default("prometheus-cp") }} + # This is a username and password hash in htpasswd format encoded in base64 + htpasswdCredentials: {{ cluster.htpasswd }} + dwx: + Config: + database: + name: {{ databases.DWX.name }} + mode: {{ databases.DWX.mode | default(ecs_database_mode) }} + host: {{ databases.DWX.host }} + port: {{ databases.DWX.port }} + username: {{ databases.DWX.user }} + password: {{ databases.DWX.password }} + dex: + Config: + database: + name: {{ databases.DEX.name }} + mode: {{ databases.DEX.mode | default(ecs_database_mode) }} + host: {{ databases.DEX.host }} + port: {{ databases.DEX.port }} + username: {{ databases.DEX.user }} + password: {{ databases.DEX.password }} + resourcepoolmanager: + Config: + database: + name: {{ databases.RESOURCEPOOL_MANAGER.name }} + mode: {{ databases.RESOURCEPOOL_MANAGER.mode | default(ecs_database_mode) }} + host: {{ databases.RESOURCEPOOL_MANAGER.host }} + port: {{ databases.RESOURCEPOOL_MANAGER.port }} + username: {{ databases.RESOURCEPOOL_MANAGER.user }} + password: {{ databases.RESOURCEPOOL_MANAGER.password }} + clusteraccessmanager: + Config: + database: + name: {{ databases.CLUSTER_ACCESS_MANAGER.name }} + mode: {{ databases.CLUSTER_ACCESS_MANAGER.mode | default(ecs_database_mode) }} + host: {{ databases.CLUSTER_ACCESS_MANAGER.host }} + port: {{ databases.CLUSTER_ACCESS_MANAGER.port }} + username: {{ databases.CLUSTER_ACCESS_MANAGER.user }} + password: {{ databases.CLUSTER_ACCESS_MANAGER.password }} + + thunderheadusermanagementprivate: + Config: + database: + name: {{ databases.UMS.name }} + mode: {{ databases.UMS.mode | default(ecs_database_mode) }} + host: {{ databases.UMS.host }} + port: {{ databases.UMS.port }} + username: {{ databases.UMS.user }} + password: {{ databases.UMS.password }} + replicaCount: 2 + + thunderheadiamconsole: + Config: + replicaCount: 2 + + thunderheadiamapi: + Config: + replicaCount: 2 + +ContainerInfo: + ## The mode can be public or custom + Mode: embedded + CopyDocker: true + ## Docker read credentials are typically required. + # Username: + # Password: + ## If the mode is custom, then enter the docker registry + repository here: + ## e.g. registry.example.com/repository_name + # Registry: + +Database: + ## The mode can be either embedded or existing + ## If existing is set, then database host, port, username, password must be + ## set for each database in the services section above. + Mode: embedded + ## The amount of storage in GB given to the embedded database. + EmbeddedDbStorage: 5 + +Vault: + ## The mode can be either embedded or external. + Mode: embedded + ## The amount of storage in GB given to the embedded vault. + EmbeddedStorage: 2 + ## If external is set, then you also need to set the following: + # Address: + # Token: + # CaCert: + # SecretPath: kv + # K8AuthPathPrefix: cloudera + # InClusterAddress: + # VaultK8sAddress: + # SkipVaultTLS: + +OptionalCerts: + MiscCaCerts: "" + +{% set auth_provider = auth_providers[service_auth_provider] %} +## If you want to use LDAP, uncomment the following +{%- if service_auth_provider is defined and service_auth_provider in auth_providers | default({}) -%} +ExternalAuth: + Type: {%- if auth_provider.type == "LDAP" -%}LDAP{%- else -%}ACTIVE_DIRECTORY{%- endif -%} + Config: + ldap_url: "{{ auth_provider.ldap_url }}" + ldap_bind_dn: "{{ auth_provider.ldap_bind_user_dn }}" + ldap_bind_pw: "{{ auth_provider.ldap_bind_password }} + ldap_user_search_base: "{{ auth_provider.ldap_search_base.common | default('') }}" + ldap_user_search_filter: "(objectClass={{ auth_provider.ldap_object_class.user }})" + ldap_group_search_base: "{{ auth_provider.ldap_search_base.group | default('') }}" + ldap_group_search_filter: "(objectClass={{ auth_provider.ldap_object_class.group }})" +{%- endif -%} +## You should fill in the following information. Otherwise, you might not be able to +## communicate with the Cloudera Manager instance or send diagnostics to Cloudera. +Other: + cmLicense: + uuid: {{ cloudera_manager_repo_username }} + owner: "{{ cloudera_manager_license | regex_replace('(.|\n)*\"name\"\\s*:\\s*\"([^\"]*)\"(.|\n)*', '\\2') }}" +# truststoreJKS: +# truststorePEM: +# truststoreChecksum: +## If you want to use a Storage Class other than the default, uncomment the following +# Storage: +# StorageClass: + +## Valid values for Name: openshift/standard +## standard (non-openshift, rancher, etc.) +## Note: values in lowercase +Platform: + Name: standard +## Only filled for Standard platform + WebConsoleUrl: "" + +## For Rancher, set the appropriate application domain name. e.g. rancher-1.vpc.cloudera.com +ApplicationDomain: "{{ cluster.application_domain }}" + + +## Openshift injects a valid fsGroup value for securityContext for its deployments. Since Rke2 doesn't have +## this feature built-in, it causes a permission issue due to which postgresql fails to create the data directory +## for persisting data (OPSX-1633). This flag needs to be set to false for Openshift, true otherwise. +SecurityContext: + Enabled: true \ No newline at end of file diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 new file mode 100644 index 00000000..10fc1de5 --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/ecs/hosts.j2 @@ -0,0 +1,11 @@ +{ +"items" : [ +{%- set host_joiner = joiner(",") -%} +{%- for host in groups['ecs_servers'] -%} + {{ host_joiner() }} + { + "hostId" : "{{ cloudera_manager_api_hosts[host]['id'] }}" + } +{%- endfor -%} +] +} \ No newline at end of file diff --git a/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 b/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 new file mode 100644 index 00000000..39859383 --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/ecs/services.j2 @@ -0,0 +1,56 @@ +{% import 'cm_api.j2' as cm_api with context %} +{ + "items": [ + {%- set service_joiner = joiner(",") -%} + {%- for service in cluster.services -%} + {%- set service_role_types = role_mappings[service] -%} + {{ service_joiner() }} + { + {%- set service_type = service -%} + {%- set service_name = service | lower -%} + "name": "{{ service_name }}", + "displayName": "{{ service_type | replace("_", " ") }}", + "type": "{{ service_type }}", + {% set config_sep = joiner(",") -%} + "config": {{ cm_api.ApiConfigList(merged_configs[service]['SERVICEWIDE']) }}, + "roles": [ + {% set role_sep = joiner(",") -%} + {%- for (template_name, role_mapping) in cluster.host_templates.items() -%} + {%- for (template_service, roles) in role_mapping.items() -%} + {%- if template_service == service -%} + {%- for role in roles -%} + {%- for host in groups[template_name] -%} + {{ role_sep() }} + { + "type": "{{ role }}", + "hostRef" : { + "hostId" : "{{ cloudera_manager_api_hosts[host]['id'] }}" + }, + "roleConfigGroupRef" : { + "roleConfigGroupName" : "{{ service_name }}-{{ role }}-BASE" + } + } + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + ], + "roleConfigGroups": [ + {%- set role_group_sep = joiner(",") -%} + {%- if service_role_types is iterable -%} + {%- for role_type in service_role_types -%} + {{ role_group_sep() }} + { + "name": "{{ service_name }}-{{ role_type }}-BASE", + "roleType": "{{ role_type }}", + {% set config_sep = joiner(",") -%} + "config": {{ cm_api.ApiConfigList(merged_configs[service][role_type]) }}, + "base": true + } + {%- endfor -%} + {%- endif -%} + ] + } + {%- endfor -%} +]} \ No newline at end of file diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml index b2d6ffff..bcae31e6 100644 --- a/roles/deployment/definition/defaults/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -36,6 +36,38 @@ database_defaults: name: amon user: amon password: "{{ database_default_password }}" + ALERTS: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-alerts + user: db-alerts + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" + CLASSIC_CLUSTERS: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: classic-clusters + user: classic-clusters + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" + CLUSTER_ACCESS_MANAGER: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-clusteraccessmanager + user: db-clusteraccessmanager + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" + CLUSTER_PROXY: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: cluster-proxy + user: cluster-proxy + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" DAS: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" @@ -43,6 +75,30 @@ database_defaults: name: das user: das password: "{{ database_default_password }}" + DEX: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-dex + user: db-dex + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" + DWX: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-dwx + user: db-dwx + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" + ENV: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-env + user: db-env + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" HIVE: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" @@ -57,6 +113,14 @@ database_defaults: name: hue user: hue password: "{{ database_default_password }}" + LIFTIE: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-liftie + user: db-liftie + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" NAVIGATOR: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" @@ -71,6 +135,14 @@ database_defaults: name: navms user: navms password: "{{ database_default_password }}" + MLX: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-mlx + user: db-mlx + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" OOZIE: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" @@ -92,6 +164,14 @@ database_defaults: name: rman user: rman password: "{{ database_default_password }}" + RESOURCEPOOL_MANAGER: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-resourcepoolmanager + user: db-resourcepoolmanager + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" SCHEMAREGISTRY: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" @@ -113,3 +193,12 @@ database_defaults: name: sentry user: sentry password: "{{ database_default_password }}" + UMS: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: db-ums + user: db-ums + password: "{{ database_default_password }}" + mode: "{{ cluster.ecs_database_mode | default('existing') }}" + diff --git a/roles/deployment/repometa/templates/role_mappings/ecs.j2 b/roles/deployment/repometa/templates/role_mappings/ecs.j2 new file mode 100644 index 00000000..d72c3bd8 --- /dev/null +++ b/roles/deployment/repometa/templates/role_mappings/ecs.j2 @@ -0,0 +1,5 @@ +DOCKER: + - DOCKER_SERVER +ECS: + - ECS_AGENT + - ECS_SERVER