Pvc experiences (#44)

Include support for PvC experiences.

Things still outstanding:

Teardown
TLS (getting truststores and jks base64 encoded and into the yaml file)
Prometheus ht credentials
Testing on a real cluster
Proposal is to do these under a separate PR, instead get this PR into devel for people to start using

roles/config/cluster/common/defaults/main.yml

@@ -37,6 +37,12 @@ default_cluster_kts:
       KEYTRUSTEE_PASSIVE_SERVER: {}
       DB_PASSIVE: {}
 
+default_cluster_ecs:
+  name: ECS
+  services: [DOCKER, ECS]
+
+ecs_databases: [ALERTS, CLASSIC_CLUSTERS, CLUSTER_ACCESS_MANAGER, CLUSTER_PROXY, DEX, DWX, ENV, LIFTIE, MLX, RESOURCEPOOL_MANAGER, UMS]
+
 default_cluster_compute:
   base_cluster:
     data_context: SDX

roles/config/cluster/ecs/meta/main.yml

@@ -0,0 +1,21 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+dependencies:
+  - role: cloudera.cluster.cloudera_manager.common
+  - role: cloudera.cluster.deployment.definition
+  - role: cloudera.cluster.config.cluster.common
+  - role: cloudera.cluster.infrastructure.ca_common

roles/config/cluster/ecs/tasks/main.yml

@@ -0,0 +1,44 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+# This variable is used by other roles
+# please take care when changing it
+- set_fact:
+    databases: "{{ database_defaults | combine(cluster.databases | default({}), recursive=True) }}"
+
+- name: Retrieve repository metadata
+  include_role:
+    name: cloudera.cluster.deployment.repometa
+  vars:
+    repositories: "{{ cluster.repositories | default({}) }}"
+
+- name: Reset custom configuration dictionary
+  set_fact:
+    custom_configs: {}
+
+- name: Generate custom configurations
+  set_fact:
+    custom_configs: "{{ custom_configs | combine(lookup('template', custom_config_template.template) | from_yaml, recursive=True) }}"
+  loop: "{{ custom_config_templates }}"
+  loop_control:
+    loop_var: custom_config_template
+  when: custom_config_template.condition | default(True)
+
+# This variable is used by other roles
+# please take care when changing it
+- name: Merge custom configurations
+  set_fact:
+    merged_configs: "{{ custom_configs | combine(cluster.configs | default({}), recursive=True) }}"

roles/config/cluster/ecs/templates/configs/ecs.j2

@@ -0,0 +1,12 @@
+---
+DOCKER:
+  SERVICEWIDE:
+    docker_images_destination_registry_user: registry-user
+ECS:
+  SERVICEWIDE:
+    k8s_webui_secret_admin_token: ecs-k8s_webui_secret_admin_token
+    cp_prometheus_ingress_user: cloudera-manager
+    infra_prometheus_ingress_user: cloudera-manager
+    longhorn_replication: 1
+    lsoDataPath: /ecs/storage
+    app_domain: "{{ cluster.application_domain }}"

roles/config/cluster/ecs/vars/main.yml

@@ -0,0 +1,19 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+custom_config_templates:
+# Custom configurations for ECS
+  - template: configs/ecs.j2

roles/deployment/cluster/meta/main.yml

@@ -15,6 +15,7 @@
 ---
 dependencies:
   - role: cloudera.cluster.cloudera_manager.api_hosts
+  - role: cloudera.cluster.deployment.credential
   - role: cloudera.cluster.infrastructure.ca_common
   - role: cloudera.cluster.prereqs.local_accounts_common
 

roles/deployment/cluster/tasks/create_ecs.yml

@@ -0,0 +1,139 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Generate complete ecs cluster configs
+  include_role:
+    name: cloudera.cluster.config.cluster.ecs
+
+- name: Create databases and users
+  include_role:
+    name: cloudera.cluster.deployment.databases
+  vars:
+    services: "{{ cluster.ecs_databases | default(ecs_databases) }}"
+
+# Add PVC parcel repo - assume that this has been set via REMOTE_PARCEL_REPO_URLS in the cm config. We can add this in later if needed
+
+- name: Create ECS cluster
+  cloudera.cluster.cm_api:
+    endpoint: /clusters
+    method: POST
+    body: "{{ lookup('template', 'cluster_template/ecs/clusters.j2', convert_data=False) }}"
+  register: cluster_creation_result
+  failed_when: '"Status code was 400" in cluster_creation_result.msg'
+  when: not cluster_template_dry_run
+
+- name: Add hosts to ECS cluster
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{ cluster.name | urlencode() }}/hosts
+    method: POST
+    body: "{{ lookup('template', 'cluster_template/ecs/hosts.j2', convert_data=False) }}"
+  register: cluster_hosts_result
+  failed_when: '"Status code was 400" in cluster_hosts_result.msg'
+  when: not cluster_template_dry_run
+
+- name: Add services to ECS cluster
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{ cluster.name | urlencode() }}/services
+    method: POST
+    body: "{{ lookup('template', 'cluster_template/ecs/services.j2', convert_data=False) }}"
+  register: cluster_services_result
+  failed_when: '"Status code was 400" in cluster_services_result.msg'
+  when: not cluster_template_dry_run
+
+- name: Get CM config
+  cloudera.cluster.cm_api:
+    endpoint: "cm/config"
+    method: GET
+  register: cm_config
+
+- set_fact:
+    parcel_repos: "{{ (cm_config.json | json_query('items[?name==`REMOTE_PARCEL_REPO_URLS`].value') | default(['']))[0].split(',') | default([]) | union(cluster.repositories) }}"
+
+- name: Update parcelrepos
+  cloudera.cluster.cm_api:
+    endpoint: "cm/config"
+    body:
+      items:
+        - name: REMOTE_PARCEL_REPO_URLS
+          value: "{{ parcel_repos | join(', ') }}"
+    method: PUT
+
+- name: Refresh parcel repos command
+  cloudera.cluster.cm_api:
+    endpoint: "/cm/commands/refreshParcelRepos"
+    method: POST
+
+- set_fact:
+    new_parcel_version: "{{ products | json_query('[?product==`ECS`].version') | first }}"
+
+- name: Start Parcel Download
+  cloudera.cluster.cm_api:
+    endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/startDownload"
+    method: POST
+
+- name: Wait for parcels to download
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}
+  register: parcels_response
+  until: parcels_response.json.stage in ("DOWNLOADED", "DISTRIBUTED", "ACTIVATED")
+  retries: "{{ parcel_poll_max_retries | default(30) }}"
+  delay: "{{ parcel_poll_duration | default(60) }}"
+
+- name: Start Parcel Distribution
+  cloudera.cluster.cm_api:
+    endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/startDistribution"
+    method: POST
+
+- name: Wait for parcels to become distributed to all hosts
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}
+  register: parcels_response
+  until:  parcels_response.json.stage in ("DISTRIBUTED", "ACTIVATED")
+  retries: "{{ parcel_poll_max_retries | default(30) }}"
+  delay: "{{ parcel_poll_duration | default(60) }}"
+
+- name: Activate Parcels
+  cloudera.cluster.cm_api:
+    endpoint: "/clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}/commands/activate"
+    method: POST
+
+- name: Wait for parcels to become distributed to all hosts
+  cloudera.cluster.cm_api:
+    endpoint: /clusters/{{cluster.name | urlencode() }}/parcels/products/ECS/versions/{{ new_parcel_version }}
+  register: parcels_response
+  until:  parcels_response.json.stage in ("ACTIVATED")
+  retries: "{{ parcel_poll_max_retries | default(30) }}"
+  delay: "{{ parcel_poll_duration | default(60) }}"
+
+- name: Generate custom values
+  set_fact:
+    custom_values: "{{ lookup('template', 'cluster_template/ecs/controlPlaneValues.j2') | from_yaml | combine(cluster.controlplane_config, recursive=True) }}"
+
+- name: Show custom values
+  debug:
+    var: custom_values
+
+- name: Install ECS cluster
+  cloudera.cluster.cm_api:
+    endpoint: /controlPlanes/commands/installEmbeddedControlPlane
+    method: POST
+    body:
+      remoteRepoUrl: "{{ cluster.pvc_repository }}"
+      experienceClusterName: "{{ cluster.name }}"
+      datalakeClusterName: "{{ cluster.datalake }}"
+      valuesYaml: "{{ custom_values | to_yaml }}"
+  register: controlplane_response
+

roles/deployment/cluster/tasks/main.yml

@@ -114,6 +114,18 @@
     - cluster.type | default(default_cluster_type) == 'compute'
     - cluster.name not in existing_clusters
 
+- name: Create ECS clusters
+  include_tasks: create_ecs.yml
+  loop: "{{ definition.clusters }}"
+  loop_control:
+    loop_var: _cluster
+  vars:
+    cluster: "{{ default_cluster_ecs | combine(_cluster) }}"
+  when:
+    - cluster.type | default(default_cluster_type) == 'ecs'
+    - cluster.name not in existing_clusters
+
+
 - name: Restart Cloudera Management Service
   cloudera.cluster.cm_api:
     endpoint: /cm/service/commands/restart

roles/deployment/cluster/templates/cluster_template/ecs/clusters.j2

@@ -0,0 +1,12 @@
+{
+"items":
+[
+  {
+    "name" : "{{ cluster.name }}",
+    "displayName": "{{ cluster.name }}",
+    "version" : "EXPERIENCE1",
+    "fullVersion": "{{ products | cloudera.cluster.get_product_version('ECS') }}",
+    "clusterType": "EXPERIENCE_CLUSTER"
+  }
+]
+}

roles/deployment/cluster/templates/cluster_template/ecs/cm_api.j2

@@ -0,0 +1,90 @@
+{% macro ApiConfig(k, v, force_uppercase_keys=False) -%}
+  {%- set config = {} -%}
+  {%- if force_uppercase_keys -%}
+    {{ config.update({"name": k | upper}) }}
+  {%- else -%}
+    {{ config.update({"name": k}) }}
+  {%- endif -%}
+  {# the extra to_json filter renders a dict into properly escaped JSON string for the CM API call #}
+  {%- if v is mapping -%}
+    {{ config.update({"value": v | to_json}) }}
+  {%- else -%}
+    {{ config.update({"value": v}) }}
+  {%- endif -%}
+  {{ config | to_json }}
+{%- endmacro %}
+
+{% macro ApiConfigList(configs, force_uppercase_keys=False) -%}
+  {%- set config_list = { "items": [] } -%}
+  {%- if configs is mapping -%}
+    {%- for (k, v) in configs.items() -%}
+      {%- set config = ApiConfig(k, v, force_uppercase_keys) -%}
+      {{ config_list["items"].append(config | from_json) }}
+    {%- endfor -%}
+  {%- endif -%}
+  {{ config_list | to_json }}
+{%- endmacro %}
+
+{% macro ApiClusterTemplateConfig(key, service_type, role_type, value, value_type="value") -%}
+  {%- set config = {} -%}
+  {{ config.update({"name": key}) }}
+  {%- if value_type == "ref" or value_type == "value" -%}
+    {{ config.update({ value_type: value }) }}
+  {%- else -%}
+    {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%}
+    {{ config.update({ "variable": var_name | upper }) }}
+  {%- endif -%}
+  {{ config | to_json }}
+{%- endmacro %}
+
+{% macro ApiClusterTemplateConfigList(service_merged_configs, service_type, role_type) -%}
+  {%- set config_list = [] -%}
+  {%- if service_type in service_merged_configs -%}
+    {%- if service_merged_configs[service_type][role_type] is mapping -%}
+      {%- for (k, v) in service_merged_configs[service_type][role_type].items() -%}
+        {%- if k.endswith("_service") and not "suppression" in k -%}
+          {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "ref") -%}
+        {%- else -%}
+          {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%}
+        {%- endif -%}
+        {{ config_list.append(config | from_json) }}
+      {%- endfor -%}
+    {%- endif -%}
+  {%- endif -%}
+  {{ config_list | to_json }}
+{%- endmacro %}
+
+{% macro ApiService(service) -%}
+{}
+{%- endmacro %}
+
+{% macro ApiServiceList(services) -%}
+  {%- set api_service_list = { "items": [] } -%}
+  {%- if services -%}
+    {%- for service in services -%}
+      {%- set api_service = ApiService(service) -%}
+      {{ api_service_list["items"].append(api_service | from_json) }}
+    {%- endfor -%}
+  {%- endif -%}
+  {{ api_service_list | to_json }}
+{%- endmacro %}
+
+{% macro ApiExternalUserMapping(auth_type, user_mapping) %}
+  {%- set external_user_mapping = {'name': user_mapping.group, 'type': auth_type, 'authRoles': []} -%}
+  {%- for role in user_mapping.roles -%}
+    {{ external_user_mapping['authRoles'].append(ApiAuthRoleRef(role) | from_json) }}
+  {%- endfor -%}
+  {{ external_user_mapping | to_json }}
+{%- endmacro %}
+
+{% macro ApiExternalUserMappingList(auth_type, auth_user_mappings) %}
+  {%- set external_user_mapping_list = { "items": [] } -%}
+  {%- for user_mapping in auth_user_mappings -%}
+    {{ external_user_mapping_list["items"].append(ApiExternalUserMapping(auth_type, user_mapping) | from_json) }}
+  {%- endfor -%}
+  {{ external_user_mapping_list | to_json }}
+{%- endmacro %}
+
+{% macro ApiAuthRoleRef(name) -%}
+  { "uuid": {{ auth_role_uuids[name] | to_json }} }
+{%- endmacro %}