Added support for SQL Stream Builder

Signed-off-by: Andre Araujo <araujo@cloudera.com>
cloudera-labs · Mar 2, 2022 · 47d736a · 47d736a
1 parent 84d735d
commit 47d736a
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 31 deletions.
diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2
@@ -31,6 +31,18 @@ SCHEMAREGISTRY:
     database_name: {{ databases.SCHEMAREGISTRY.name }}
     database_user: {{ databases.SCHEMAREGISTRY.user }}
     database_password: {{ databases.SCHEMAREGISTRY.password }}
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    database_host: {{ databases.SQL_STREAM_BUILDER.host }}
+    database_port: {{ databases.SQL_STREAM_BUILDER.port }}
+    database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }}
+    database_schema: {{ databases.SQL_STREAM_BUILDER.name }}
+    database_user: {{ databases.SQL_STREAM_BUILDER.user }}
+    database_password: {{ databases.SQL_STREAM_BUILDER.password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }}
+    ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }}
+    ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }}
 STREAMS_MESSAGING_MANAGER:
   SERVICEWIDE:
     smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }}

diff --git a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2
@@ -71,9 +71,11 @@ HUE:
 FLINK:
   SERVICEWIDE:
     hdfs_service: hdfs
-    hive_service: hive
     yarn_service: yarn
-    zookeeper_service: zoookeeper
+    zookeeper_service: zookeeper
+{% if 'HIVE' in cluster.services %}
+    hive_service: hive
+{% endif %}
 {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %}
     atlas_service: atlas
 {% endif %}
@@ -174,6 +176,17 @@ SPARK3_ON_YARN:
     hive_service: hive
 {% endif %}
 
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    flink_service: flink
+    kafka_service: kafka
+{% if 'HIVE' in cluster.services %}
+    hive_service: hive
+{% endif %}
+{% if 'KNOX' in cluster.services %}
+    knox_service: knox
+{% endif %}
+
 TEZ:
   SERVICEWIDE:
     yarn_service: yarn

diff --git a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2
@@ -2,6 +2,9 @@
 CORE_SETTINGS:
   SERVICEWIDE:
     hadoop_secure_web_ui: true
+FLINK:
+  SERVICEWIDE:
+    kerberos.auth.enabled: true
 HBASE:
   SERVICEWIDE:
     hbase_restserver_security_authentication: kerberos

diff --git a/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2
@@ -1,4 +1,26 @@
 ---
 OZONE:
   OZONE_PROMETHEUS:
-    ozone.prometheus.ca.file: {{ tls_chain_path }}
+    ozone.prometheus.ca.file: {{ tls_chain_path }}
+SQL_STREAM_BUILDER:
+  STREAMING_SQL_ENGINE:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
+  STREAMING_SQL_CONSOLE:
+    ssl_client_truststore_location: {{ tls_chain_path }}
+    ssl_enabled: true
+    ssl_server_ca_certificate_location: {{ tls_chain_path }}
+    ssl_server_certificate_location: {{ tls_cert_path_generic }}
+    ssl_server_privatekey_location: {{ tls_key_path_generic }}
+    ssl_server_privatekey_password: {{ tls_key_password }}
diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml
@@ -75,6 +75,7 @@ cluster_services_ordered:
   - CRUISE_CONTROL
   - DAS
   - FLINK
+  - SQL_STREAM_BUILDER
   - SPARK
   - SPARK2_ON_YARN
   - SPARK3_ON_YARN

diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml
@@ -99,6 +99,20 @@ database_defaults:
     name: schemaregistry
     user: schemaregistry
     password: "{{ database_default_password }}"
+  SQL_STREAM_BUILDER:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: "{{ database_type }}"
+    name: ssb_admin
+    user: ssb_admin
+    password: "{{ database_default_password }}"
+  SQL_STREAM_BUILDER_MVE:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: "{{ database_type }}"
+    name: ssb_mve
+    user: ssb_mve
+    password: "{{ database_default_password }}"
   STREAMS_MESSAGING_MANAGER:
     host: "{{ database_host }}"
     port: "{{ database_type | cloudera.cluster.default_database_port }}"

diff --git a/roles/deployment/repometa/templates/all_services.j2 b/roles/deployment/repometa/templates/all_services.j2
@@ -1,57 +1,58 @@
-- KEYTRUSTEE_SERVER
-- ZOOKEEPER
-- INFRA_SOLR
-- RANGER
-- RANGER_KMS
-- RANGER_KMS_KTS
-- RANGER_RAZ
-- KMS
-- HDFS
-- SENTRY
-- LIVY
-- HBASE
-- SOLR
-- SQOOP
-- SQOOP_CLIENT
 - ACCUMULO16
 - ACCUMULO_C6
-- HIVE
-- TEZ
-- HIVE_ON_TEZ
-- KUDU
-- IMPALA
-- YARN
 - ADLS_CONNECTOR
+- ATLAS
 - AWS_S3
 - CDSW
 - CORE_SETTINGS
 - CRUISE_CONTROL
 - DAS
 - FLINK
 - FLUME
-- SPARK
-- SPARK2_ON_YARN
-- SPARK3_ON_YARN
-- SPARK_ON_YARN
+- HBASE
+- HDFS
+- HIVE
+- HIVE_ON_TEZ
+- HUE
+- IMPALA
+- INFRA_SOLR
 - ISILON
 - KAFKA
 - KEYTRUSTEE
+- KEYTRUSTEE_SERVER
+- KMS
 - KNOX
 - KS_INDEXER
+- KUDU
+- LIVY
 - LUNA_KMS
 - MAPREDUCE
 - NIFI
 - NIFIREGISTRY
 - NIFITOOLKITCA
+- OOZIE
 - OZONE
 - PHOENIX
 - QUEUEMANAGER
+- RANGER
+- RANGER_KMS
+- RANGER_KMS_KTS
+- RANGER_RAZ
 - SCHEMAREGISTRY
+- SENTRY
+- SOLR
+- SPARK
+- SPARK2_ON_YARN
+- SPARK3_ON_YARN
+- SPARK_ON_YARN
+- SQL_STREAM_BUILDER
+- SQOOP
+- SQOOP_CLIENT
 - STREAMS_MESSAGING_MANAGER
 - STREAMS_REPLICATION_MANAGER
+- TEZ
 - THALES_KMS
 - WXM
+- YARN
 - ZEPPELIN
-- ATLAS
-- OOZIE
-- HUE
+- ZOOKEEPER
diff --git a/roles/deployment/repometa/templates/role_mappings/cdh7.j2 b/roles/deployment/repometa/templates/role_mappings/cdh7.j2
@@ -11,6 +11,8 @@ CRUISE_CONTROL:
 DAS:
   - DAS_EVENT_PROCESSOR
   - DAS_WEBAPP
+FLINK:
+  - FLINK_HISTORY_SERVER
 HBASE:
   - GATEWAY
   - HBASERESTSERVER
@@ -103,6 +105,10 @@ SOLR:
 SPARK_ON_YARN:
   - GATEWAY
   - SPARK_YARN_HISTORY_SERVER
+SQL_STREAM_BUILDER:
+  - STREAMING_SQL_ENGINE
+  - MATERIALIZED_VIEW_ENGINE
+  - STREAMING_SQL_CONSOLE
 SQOOP_CLIENT:
   - GATEWAY
 STREAMS_MESSAGING_MANAGER:

diff --git a/roles/prereqs/local_accounts_common/defaults/main.yml b/roles/prereqs/local_accounts_common/defaults/main.yml
@@ -48,6 +48,13 @@ local_accounts:
     comment: Flink
     keystore_acl: True
 
+  - user: ssb
+    home: /var/lib/ssb
+    comment: SQL Stream Builder
+    keystore_acl: True
+    key_acl: True
+    key_password_acl: True
+
   - user: flume
     home: /var/lib/flume-ng
     comment: Flume