Adding support for SQL Stream Builder deployment (#48)

* New role to deploy the PostgreSQL connector

Signed-off-by: Andre Araujo <araujo@cloudera.com>

* Added support for SQL Stream Builder

Signed-off-by: Andre Araujo <araujo@cloudera.com>

roles/config/cluster/base/templates/configs/databases-7.1.0.j2

@@ -31,6 +31,18 @@ SCHEMAREGISTRY:
     database_name: {{ databases.SCHEMAREGISTRY.name }}
     database_user: {{ databases.SCHEMAREGISTRY.user }}
     database_password: {{ databases.SCHEMAREGISTRY.password }}
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    database_host: {{ databases.SQL_STREAM_BUILDER.host }}
+    database_port: {{ databases.SQL_STREAM_BUILDER.port }}
+    database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }}
+    database_schema: {{ databases.SQL_STREAM_BUILDER.name }}
+    database_user: {{ databases.SQL_STREAM_BUILDER.user }}
+    database_password: {{ databases.SQL_STREAM_BUILDER.password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }}
+    ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }}
+    ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }}
 STREAMS_MESSAGING_MANAGER:
   SERVICEWIDE:
     smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }}

roles/config/cluster/base/templates/configs/inter-service-dependencies.j2

@@ -71,9 +71,11 @@ HUE:
 FLINK:
   SERVICEWIDE:
     hdfs_service: hdfs
-    hive_service: hive
     yarn_service: yarn
-    zookeeper_service: zoookeeper
+    zookeeper_service: zookeeper
+{% if 'HIVE' in cluster.services %}
+    hive_service: hive
+{% endif %}
 {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %}
     atlas_service: atlas
 {% endif %}
@@ -174,6 +176,17 @@ SPARK3_ON_YARN:
     hive_service: hive
 {% endif %}
 
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    flink_service: flink
+    kafka_service: kafka
+{% if 'HIVE' in cluster.services %}
+    hive_service: hive
+{% endif %}
+{% if 'KNOX' in cluster.services %}
+    knox_service: knox
+{% endif %}
+
 TEZ:
   SERVICEWIDE:
     yarn_service: yarn

roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2

@@ -2,6 +2,9 @@
 CORE_SETTINGS:
   SERVICEWIDE:
     hadoop_secure_web_ui: true
+FLINK:
+  SERVICEWIDE:
+    kerberos.auth.enabled: true
 HBASE:
   SERVICEWIDE:
     hbase_restserver_security_authentication: kerberos

roles/config/cluster/base/templates/configs/tls-7.3.1.j2

@@ -1,4 +1,26 @@
 ---
 OZONE:
   OZONE_PROMETHEUS:
-    ozone.prometheus.ca.file: {{ tls_chain_path }}
+    ozone.prometheus.ca.file: {{ tls_chain_path }}
+SQL_STREAM_BUILDER:
+  STREAMING_SQL_ENGINE:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
+  STREAMING_SQL_CONSOLE:
+    ssl_client_truststore_location: {{ tls_chain_path }}
+    ssl_enabled: true
+    ssl_server_ca_certificate_location: {{ tls_chain_path }}
+    ssl_server_certificate_location: {{ tls_cert_path_generic }}
+    ssl_server_privatekey_location: {{ tls_key_path_generic }}
+    ssl_server_privatekey_password: {{ tls_key_password }}

roles/config/cluster/common/defaults/main.yml

@@ -81,6 +81,7 @@ cluster_services_ordered:
   - CRUISE_CONTROL
   - DAS
   - FLINK
+  - SQL_STREAM_BUILDER
   - SPARK
   - SPARK2_ON_YARN
   - SPARK3_ON_YARN

roles/deployment/databases/tasks/mariadb.yml

@@ -20,7 +20,7 @@
     encoding: "{{ service | cloudera.cluster.get_database_encoding_mysql }}"
     collation: "{{ service | cloudera.cluster.get_database_collation_mysql }}"
   become: yes
-  loop: "{{ databases | intersect(services) }}"
+  loop: "{{ databases }}"
   loop_control:
     loop_var: service
   delegate_to: "{{ databases[service].host }}"
@@ -36,7 +36,7 @@
     priv: "{{ databases[service].name }}.*:ALL"
   no_log: yes
   become: yes
-  loop: "{{ databases | intersect(services) }}"
+  loop: "{{ databases }}"
   loop_control:
     loop_var: service
   delegate_to: "{{ databases[service].host }}"

roles/deployment/databases/tasks/postgresql.yml

@@ -20,7 +20,7 @@
     password: "{{ databases[item].password }}"
   become: yes
   become_user: postgres
-  with_items: "{{ databases | intersect(services) }}"
+  with_items: "{{ databases }}"
   delegate_to: "{{ databases[item].host }}"
   connection: ssh
   when: databases[item].host in groups.db_server
@@ -32,7 +32,7 @@
     encoding: UTF-8
   become: yes
   become_user: postgres
-  with_items: "{{ databases | intersect(services) }}"
+  with_items: "{{ databases }}"
   delegate_to: "{{ databases[item].host }}"
   connection: ssh
   when: databases[item].host in groups.db_server

roles/deployment/definition/defaults/main.yml

@@ -179,6 +179,20 @@ database_defaults:
     name: schemaregistry
     user: schemaregistry
     password: "{{ database_default_password }}"
+  SQL_STREAM_BUILDER:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: "{{ database_type }}"
+    name: ssb_admin
+    user: ssb_admin
+    password: "{{ database_default_password }}"
+  SQL_STREAM_BUILDER_MVE:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: "{{ database_type }}"
+    name: ssb_mve
+    user: ssb_mve
+    password: "{{ database_default_password }}"
   STREAMS_MESSAGING_MANAGER:
     host: "{{ database_host }}"
     port: "{{ database_type | cloudera.cluster.default_database_port }}"

roles/deployment/repometa/templates/all_services.j2

@@ -1,57 +1,58 @@
-- KEYTRUSTEE_SERVER
-- ZOOKEEPER
-- INFRA_SOLR
-- RANGER
-- RANGER_KMS
-- RANGER_KMS_KTS
-- RANGER_RAZ
-- KMS
-- HDFS
-- SENTRY
-- LIVY
-- HBASE
-- SOLR
-- SQOOP
-- SQOOP_CLIENT
 - ACCUMULO16
 - ACCUMULO_C6
-- HIVE
-- TEZ
-- HIVE_ON_TEZ
-- KUDU
-- IMPALA
-- YARN
 - ADLS_CONNECTOR
+- ATLAS
 - AWS_S3
 - CDSW
 - CORE_SETTINGS
 - CRUISE_CONTROL
 - DAS
 - FLINK
 - FLUME
-- SPARK
-- SPARK2_ON_YARN
-- SPARK3_ON_YARN
-- SPARK_ON_YARN
+- HBASE
+- HDFS
+- HIVE
+- HIVE_ON_TEZ
+- HUE
+- IMPALA
+- INFRA_SOLR
 - ISILON
 - KAFKA
 - KEYTRUSTEE
+- KEYTRUSTEE_SERVER
+- KMS
 - KNOX
 - KS_INDEXER
+- KUDU
+- LIVY
 - LUNA_KMS
 - MAPREDUCE
 - NIFI
 - NIFIREGISTRY
 - NIFITOOLKITCA
+- OOZIE
 - OZONE
 - PHOENIX
 - QUEUEMANAGER
+- RANGER
+- RANGER_KMS
+- RANGER_KMS_KTS
+- RANGER_RAZ
 - SCHEMAREGISTRY
+- SENTRY
+- SOLR
+- SPARK
+- SPARK2_ON_YARN
+- SPARK3_ON_YARN
+- SPARK_ON_YARN
+- SQL_STREAM_BUILDER
+- SQOOP
+- SQOOP_CLIENT
 - STREAMS_MESSAGING_MANAGER
 - STREAMS_REPLICATION_MANAGER
+- TEZ
 - THALES_KMS
 - WXM
+- YARN
 - ZEPPELIN
-- ATLAS
-- OOZIE
-- HUE
+- ZOOKEEPER

roles/deployment/repometa/templates/role_mappings/flink.j2

@@ -1,3 +1,7 @@
 FLINK:
   - FLINK_HISTORY_SERVER
   - GATEWAY
+SQL_STREAM_BUILDER:
+  - STREAMING_SQL_ENGINE
+  - MATERIALIZED_VIEW_ENGINE
+  - STREAMING_SQL_CONSOLE

roles/prereqs/local_accounts_common/defaults/main.yml

@@ -48,6 +48,13 @@ local_accounts:
     comment: Flink
     keystore_acl: True
 
+  - user: ssb
+    home: /var/lib/ssb
+    comment: SQL Stream Builder
+    keystore_acl: True
+    key_acl: True
+    key_password_acl: True
+
   - user: flume
     home: /var/lib/flume-ng
     comment: Flume

roles/prereqs/postgresql_connector/defaults/main.yml

@@ -0,0 +1,18 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+postgresql_connector_url: https://jdbc.postgresql.org/download/postgresql-42.3.3.jar
+postgresql_connector_checksum: md5:bef0b2e1c6edcd8647c24bed31e1a4ac

roles/prereqs/postgresql_connector/meta/main.yml

@@ -0,0 +1,17 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+dependencies:
+  - role: cloudera.cluster.deployment.definition

roles/prereqs/postgresql_connector/tasks/main.yml

@@ -0,0 +1,37 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Download PostgreSQL Connector
+  get_url:
+    url: "{{ postgresql_connector_url }}"
+    dest: "{{ local_temp_dir }}/postgresql-connector-java.jar"
+    checksum: "{{ postgresql_connector_checksum }}"
+    mode: 0644
+  become: no
+  run_once: true
+  delegate_to: localhost
+
+- name: Create /usr/share/java directory
+  file:
+    path: /usr/share/java
+    state: directory
+    mode: 0755
+
+- name: Copy PostgreSQL Connector jar file to correct location
+  copy:
+    src: "{{ local_temp_dir }}/postgresql-connector-java.jar"
+    dest: /usr/share/java/postgresql-connector-java.jar
+    mode: 0644