Fixed handling of custom roleConfig Groups (cloudera-labs#46)

* Fixed handling of custom roleConfigGroups. When a roleConfigGroup was referenced in multiple templates it would be duplicated in the final cluster template. This fix removes the duplication. Signed-off-by: Andre Araujo <araujo@cloudera.com> * Removed deprecated SMM parameters from template. Signed-off-by: Andre Araujo <araujo@cloudera.com> * Made error message more informative. Signed-off-by: Andre Araujo <araujo@cloudera.com> Signed-off-by: William Dyson <wdyson@cloudera.com>
WillDyson · Jul 15, 2022 · 44a96fb · 44a96fb
1 parent e160d8f
commit 44a96fb
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 10 deletions.
diff --git a/plugins/filter/filters.py b/plugins/filter/filters.py
@@ -211,17 +211,17 @@ def extract_role_and_group(self, role_spec):
         return (role, template_group)
 
     def extract_custom_roles(self, host_templates, service):
-        custom_roles = []
+        custom_roles = set([])
         for role_mapping in host_templates.values():
             if service in role_mapping:
                 for custom_role in filter(lambda x: '/' in x, role_mapping[service]):
-                    custom_roles.append(custom_role)
-        return custom_roles
+                    custom_roles.add(custom_role)
+        return list(custom_roles)
 
     def extract_custom_role_groups(self, host_templates):
-        custom_role_groups = []
+        custom_role_groups = set([])
         for role_mapping in host_templates.values():
             for (service, roles) in role_mapping.items():
                 for custom_role in filter(lambda x: '/' in x, roles):
-                    custom_role_groups.append("-".join([service.lower()] + custom_role.split("/")))
-        return custom_role_groups
+                    custom_role_groups.add("-".join([service.lower()] + custom_role.split("/")))
+        return list(custom_role_groups)
diff --git a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2
@@ -30,6 +30,3 @@ SPARK3_ON_YARN:
   GATEWAY:
     spark_io_encryption_enabled: true
     spark_network_encryption_enabled: true
-STREAMS_MESSAGING_MANAGER:
-  STREAMS_MESSAGING_MANAGER_SERVER:
-    streams.messaging.manager.kafka.client.security.protocol: {{ cluster.security.tls | default(false) | ternary('SASL_SSL', 'SASL_PLAINTEXT') }}
diff --git a/roles/config/cluster/base/templates/configs/tls-cm-7.j2 b/roles/config/cluster/base/templates/configs/tls-cm-7.j2
@@ -201,7 +201,6 @@ STREAMS_MESSAGING_MANAGER:
     ssl_server_keystore_keypassword: {{ tls_keystore_password }}
     ssl_server_keystore_location: {{ tls_keystore_path_generic }}
     ssl_server_keystore_password: {{ tls_keystore_password }}
-    streams.messaging.manager.kafka.client.security.protocol: {{ cluster.security.kerberos | default(false) | ternary('SASL_SSL', 'SSL') }}
   STREAMS_MESSAGING_MANAGER_UI:
     ssl_client_truststore_location: {{ tls_chain_path }}
     ssl_enabled: true

diff --git a/roles/verify/parcels_and_roles/tasks/check_template_roles.yml b/roles/verify/parcels_and_roles/tasks/check_template_roles.yml
@@ -32,6 +32,7 @@
         fail_msg: >-
           Unknown role(s) {{ invalid_roles }} for service '{{ template.service }}'
           defined in host template '{{ host_template.name }}'.
+          Valid roles are: {{ role_mappings[template.service] | list }}.
 
     - name: Ensure the Tez gateway has been deployed
       assert: