Updates for 7.1.9 cluster

Signed-off-by: Jim Enright <jenright@cloudera.com>
cloudera-labs · Oct 9, 2023 · 227a06b · 227a06b
1 parent c57f33a
commit 227a06b
Show file tree

Hide file tree

Showing 14 changed files with 78 additions and 50 deletions.
diff --git a/private-cloud/ecs-on-aws/ansible-navigator.yml b/private-cloud/ecs-on-aws/ansible-navigator.yml
@@ -30,11 +30,16 @@ ansible-navigator:
         ANSIBLE_DEPRECATION_WARNINGS: False
         ANSIBLE_HOST_KEY_CHECKING: False
         ANSIBLE_SSH_RETRIES: 10
-    image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections
+        ANSIBLE_COLLECTIONS_PATH: "/mnt/jenright/data/Cloudera/gitRepos/cloudera-labs/ansible_dev/collections:/usr/share/ansible/collections/"
+    # image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections
+    image: ghcr.io/cloudera-labs/cldr-runner:aws-latest
     pull:
       arguments:
         - "--tls-verify=false"
     volume-mounts:
+      - src: "${ANSIBLE_COLLECTIONS_PATH}"
+        dest: "${ANSIBLE_COLLECTIONS_PATH}"
+        options: "Z"    
       - src: "${SSH_PUBLIC_KEY_FILE}"
         dest: "${SSH_PUBLIC_KEY_FILE}"
       - src: "${CDP_LICENSE_FILE}"

diff --git a/private-cloud/ecs-on-aws/cluster.yml b/private-cloud/ecs-on-aws/cluster.yml
@@ -28,7 +28,6 @@ clusters:
           longhorn_replication: 2
           lsoDataPath: "/ecs/local"
           defaultDataPath: "/ecs/longhorn-storage"
-          nfs_provisioned: 800
           nfs_over_provisioning: 800
           # TODO: Create cert and key for ECS ingress controller
           # ssl_certificate: /opt/cloudera/security/pki/ecs.pem
@@ -143,6 +142,11 @@ clusters:
         STORAGE_CONTAINER_MANAGER:
           scm_max_heap_size: 2048      
         OZONE_DATANODE:
+          ozone-conf/ozone-site.xml_role_safety_valve: |
+             <property>
+               <name>hdds.datanode.client.port</name>
+               <value>9874</value>
+             </property>
           ozone_datanode_heap_size: 2048
         OZONE_PROMETHEUS:
           ozone.prometheus.http-port: 19090

diff --git a/private-cloud/ecs-on-aws/definition.yml b/private-cloud/ecs-on-aws/definition.yml
@@ -6,9 +6,9 @@ encryption_activated: True
 # autotls: False
 
 repositories:
-  # 7.1.8 CHF4
-  - https://archive.cloudera.com/p/cdh7/7.1.8.15/parcels/
-cloudera_manager_version: 7.10.1
+  # Offical CDH 7.1.9.0
+  - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/
+cloudera_manager_version: 7.11.3
 
 jdk_version: 11
 
@@ -40,6 +40,8 @@ dns_provider: "freeipa"
 
 # Connect FreeIPA to Knox and Ranger
 freeipa_activated: yes
+# FreeIPA client install on cluster nodes is done during pre-setup
+freeipa_enroll: no 
 
 # FreeIPA realm settings
 freeipa:
@@ -64,12 +66,6 @@ cloudera_manager_options:
   CUSTOM_BANNER_HTML: "1.5.1 - PvC ECS Lab ({{ name_prefix }})"
   SESSION_TIMEOUT: 43200
   PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560
-  # NOTE: Disabled in ChuckL's latest definition
-  # PHONE_HOME: false
-  # PROXYUSER_KNOX_GROUPS: "*"
-  # PROXYUSER_KNOX_HOSTS: "*"
-  # PROXYUSER_KNOX_PRINCIPAL: "knox"
-  # PROXYUSER_KNOX_USERS: "*"
   KRB_AUTH_ENABLE: "true"
 
 # License options (this is due to a hardcoded tmp directory on the target/manager node)
@@ -81,8 +77,9 @@ license_local_tmp_path: /tmp/cloudera_license.txt
 database_type: postgresql
 database_version: 12
 database_tls: true
+database_default_password: "{{ common_password }}"
 
-#########++++
+###############
 
 ## Red Hat FreeIPA
 krb5_kdc_type: Red Hat IPA

diff --git a/private-cloud/ecs-on-aws/execution-environment.yml b/private-cloud/ecs-on-aws/execution-environment.yml
@@ -4,15 +4,13 @@ version: 3
 
 images:
   base_image:
-    name: ghcr.io/wmudge/cldr-runner:aws-tmp-devel-collections
+    name: ghcr.io/cloudera-labs/cldr-runner:aws-latest
 
 dependencies:
   galaxy:
     collections:
-      - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git
+      - name: https://github.com/jimright/cloudera.exe.git
         type: git
-        version: devel
-      - name: freeipa.ansible_freeipa
-        version: 1.11.0
+        version: feature/tmp_pvc
   python:
     - dnspython
diff --git a/private-cloud/ecs-on-aws/pre_setup.yml b/private-cloud/ecs-on-aws/pre_setup.yml
@@ -21,7 +21,7 @@
     - name: Provision the infrastructure resources in the cloud provider
       when: infra_type != 'static'
       ansible.builtin.import_role:
-        name: cloudera.infra.provision
+        name: cloudera.exe.provision
       vars:
         provision_provider: "{{ infra_type }}"
         provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini"
@@ -66,7 +66,7 @@
     - name: Prepare storage volumes
       when: infra_type != "static" and storage_volumes | length > 0
       ansible.builtin.import_role:
-        name: cloudera.infra.mount
+        name: cloudera.exe.mount
       vars:
         mount_volumes: "{{ storage_volumes }}"
         mount_provider: "{{ infra_type }}"
@@ -90,8 +90,7 @@
   tasks:
     - name: Set up the FreeIPA server
       ansible.builtin.import_role:
-        # name: cloudera.exe.freeipa_server
-        name: cloudera.infra.freeipa_server
+        name: cloudera.exe.freeipa_server
       vars:
         ipaserver_hostname: "{{ inventory_hostname }}"
         ipaserver_realm: "{{ freeipa.realm }}"
@@ -214,8 +213,7 @@
     - name: Register host with FreeIPA services
       when: krb5_kdc_type == "Red Hat IPA"
       ansible.builtin.import_role:
-        # name: cloudera.exe.freeipa_client
-        name: cloudera.infra.freeipa_client
+        name: cloudera.exe.freeipa_client
       vars:
         ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}"
         ipaserver_realm: "{{ freeipa.realm }}"

diff --git a/private-cloud/ecs-on-aws/pre_setup_resources.yml b/private-cloud/ecs-on-aws/pre_setup_resources.yml
@@ -13,4 +13,12 @@
           - tree 
           - jq 
           - zip
+          - python38
         state: present
+
+    - name: Install required packages
+      ansible.builtin.pip:
+        name: psycopg2-binary==2.9.5
+        executable: pip3.8
+        state: present
+
diff --git a/private-cloud/ecs-on-aws/pre_teardown.yml b/private-cloud/ecs-on-aws/pre_teardown.yml
@@ -11,7 +11,7 @@
     - name: Deprovision the infrastructure resources in the cloud provider
       when: groups['all'] | length > 0
       ansible.builtin.import_role:
-        name: cloudera.infra.provision
+        name: cloudera.exe.provision
       vars:
         provision_state: absent
         provision_provider: "{{ infra_type }}"

diff --git a/private-cloud/pvc-base-on-aws/ansible-navigator.yml b/private-cloud/pvc-base-on-aws/ansible-navigator.yml
@@ -30,11 +30,16 @@ ansible-navigator:
         ANSIBLE_DEPRECATION_WARNINGS: False
         ANSIBLE_HOST_KEY_CHECKING: False
         ANSIBLE_SSH_RETRIES: 10
-    image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections
+        ANSIBLE_COLLECTIONS_PATH: "/mnt/jenright/data/Cloudera/gitRepos/cloudera-labs/ansible_dev/collections:/usr/share/ansible/collections/"
+    # image: ghcr.io/cloudera-labs/cldr-runner:pvc-tmp-devel-collections
+    image: ghcr.io/cloudera-labs/cldr-runner:aws-latest
     pull:
       arguments:
         - "--tls-verify=false"
     volume-mounts:
+      - src: "${ANSIBLE_COLLECTIONS_PATH}"
+        dest: "${ANSIBLE_COLLECTIONS_PATH}"
+        options: "Z"    
       - src: "${SSH_PUBLIC_KEY_FILE}"
         dest: "${SSH_PUBLIC_KEY_FILE}"
       - src: "${CDP_LICENSE_FILE}"

diff --git a/private-cloud/pvc-base-on-aws/cluster.yml b/private-cloud/pvc-base-on-aws/cluster.yml
@@ -96,6 +96,11 @@ clusters:
         STORAGE_CONTAINER_MANAGER:
           scm_max_heap_size: 2048      
         OZONE_DATANODE:
+          ozone-conf/ozone-site.xml_role_safety_valve: |
+             <property>
+               <name>hdds.datanode.client.port</name>
+               <value>9874</value>
+             </property>
           ozone_datanode_heap_size: 2048
         OZONE_PROMETHEUS:
           ozone.prometheus.http-port: 19090

diff --git a/private-cloud/pvc-base-on-aws/definition.yml b/private-cloud/pvc-base-on-aws/definition.yml
@@ -6,9 +6,9 @@ encryption_activated: True
 # autotls: False
 
 repositories:
-  # 7.1.8 CHF4
-  - https://archive.cloudera.com/p/cdh7/7.1.8.15/parcels/
-cloudera_manager_version: 7.10.1
+  # Offical CDH 7.1.9.0
+  - https://archive.cloudera.com/p/cdh7/7.1.9.0/parcels/
+cloudera_manager_version: 7.11.3
 
 jdk_version: 11
 
@@ -33,6 +33,8 @@ dns_provider: "freeipa"
 
 # Connect FreeIPA to Knox and Ranger
 freeipa_activated: yes
+# FreeIPA client install on cluster nodes is done during pre-setup
+freeipa_enroll: no 
 
 # FreeIPA realm settings
 freeipa:
@@ -54,15 +56,9 @@ skip_user_group_init: no
 # Cloudera Manager details
 
 cloudera_manager_options:
-  CUSTOM_BANNER_HTML: "1.5.1 - PvC Base Lab ({{ name_prefix }})"
+  CUSTOM_BANNER_HTML: "PvC Base Lab ({{ name_prefix }})"
   SESSION_TIMEOUT: 43200
   PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 194560
-  # NOTE: Disabled in ChuckL's latest definition
-  # PHONE_HOME: false
-  # PROXYUSER_KNOX_GROUPS: "*"
-  # PROXYUSER_KNOX_HOSTS: "*"
-  # PROXYUSER_KNOX_PRINCIPAL: "knox"
-  # PROXYUSER_KNOX_USERS: "*"
   KRB_AUTH_ENABLE: "true"
 
 # License options (this is due to a hardcoded tmp directory on the target/manager node)
@@ -74,8 +70,9 @@ license_local_tmp_path: /tmp/cloudera_license.txt
 database_type: postgresql
 database_version: 12
 database_tls: true
+database_default_password: "{{ common_password }}"
 
-#########++++
+###############
 
 ## Red Hat FreeIPA
 krb5_kdc_type: Red Hat IPA

diff --git a/private-cloud/pvc-base-on-aws/execution-environment.yml b/private-cloud/pvc-base-on-aws/execution-environment.yml
@@ -4,15 +4,21 @@ version: 3
 
 images:
   base_image:
-    name: ghcr.io/wmudge/cldr-runner:aws-tmp-devel-collections
+    name: ghcr.io/cloudera-labs/cldr-runner:aws-latest
 
 dependencies:
   galaxy:
     collections:
-      - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git
+      - name: https://github.com/jimright/cloudera.exe.git
         type: git
-        version: devel
-      - name: freeipa.ansible_freeipa
-        version: 1.11.0
-  python:
-    - dnspython
+        version: feature/tmp_pvc
+      # - name: https://github.infra.cloudera.com/GOES/cloudera.infra.git
+      #   type: git
+      #   version: devel
+      # - name: https://github.com/anisf/cloudera.cluster.git
+      #   type: git
+      #   version: "fix/#126"
+      # - name: freeipa.ansible_freeipa
+      #   version: 1.11.0
+  # python:
+  #   - dnspython
diff --git a/private-cloud/pvc-base-on-aws/pre_setup.yml b/private-cloud/pvc-base-on-aws/pre_setup.yml
@@ -21,7 +21,7 @@
     - name: Provision the infrastructure resources in the cloud provider
       when: infra_type != 'static'
       ansible.builtin.import_role:
-        name: cloudera.infra.provision
+        name: cloudera.exe.provision
       vars:
         provision_provider: "{{ infra_type }}"
         provision_inventory_file: "inventory_static_{{ name_prefix }}_{{ infra_type }}.ini"
@@ -66,7 +66,7 @@
     - name: Prepare storage volumes
       when: infra_type != "static" and storage_volumes | length > 0
       ansible.builtin.import_role:
-        name: cloudera.infra.mount
+        name: cloudera.exe.mount
       vars:
         mount_volumes: "{{ storage_volumes }}"
         mount_provider: "{{ infra_type }}"
@@ -90,8 +90,7 @@
   tasks:
     - name: Set up the FreeIPA server
       ansible.builtin.import_role:
-        # name: cloudera.exe.freeipa_server
-        name: cloudera.infra.freeipa_server
+        name: cloudera.exe.freeipa_server
       vars:
         ipaserver_hostname: "{{ inventory_hostname }}"
         ipaserver_realm: "{{ freeipa.realm }}"
@@ -214,8 +213,7 @@
     - name: Register host with FreeIPA services
       when: krb5_kdc_type == "Red Hat IPA"
       ansible.builtin.import_role:
-        # name: cloudera.exe.freeipa_client
-        name: cloudera.infra.freeipa_client
+        name: cloudera.exe.freeipa_client
       vars:
         ipaserver_domain: "{{ [name_prefix, domain] | join('.') }}"
         ipaserver_realm: "{{ freeipa.realm }}"

diff --git a/private-cloud/pvc-base-on-aws/pre_setup_resources.yml b/private-cloud/pvc-base-on-aws/pre_setup_resources.yml
@@ -13,4 +13,11 @@
           - tree 
           - jq 
           - zip
+          - python38
+        state: present
+
+    - name: Install required packages
+      ansible.builtin.pip:
+        name: psycopg2-binary==2.9.5
+        executable: pip3.8
         state: present
diff --git a/private-cloud/pvc-base-on-aws/pre_teardown.yml b/private-cloud/pvc-base-on-aws/pre_teardown.yml
@@ -11,7 +11,7 @@
     - name: Deprovision the infrastructure resources in the cloud provider
       when: groups['all'] | length > 0
       ansible.builtin.import_role:
-        name: cloudera.infra.provision
+        name: cloudera.exe.provision
       vars:
         provision_state: absent
         provision_provider: "{{ infra_type }}"