From 47d736a593dde888b871d42ae6aef651bfff52cf Mon Sep 17 00:00:00 2001 From: Andre Araujo Date: Thu, 3 Mar 2022 09:16:11 +1100 Subject: [PATCH] Added support for SQL Stream Builder Signed-off-by: Andre Araujo --- .../base/templates/configs/databases-7.1.0.j2 | 12 ++++ .../configs/inter-service-dependencies.j2 | 17 +++++- .../templates/configs/kerberos-6.x-7.x.j2 | 3 + .../base/templates/configs/tls-7.3.1.j2 | 24 +++++++- roles/config/cluster/common/defaults/main.yml | 1 + roles/deployment/definition/defaults/main.yml | 14 +++++ .../repometa/templates/all_services.j2 | 57 ++++++++++--------- .../repometa/templates/role_mappings/cdh7.j2 | 6 ++ .../local_accounts_common/defaults/main.yml | 7 +++ 9 files changed, 110 insertions(+), 31 deletions(-) diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 index 81668d8e..7a88a016 100644 --- a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 @@ -31,6 +31,18 @@ SCHEMAREGISTRY: database_name: {{ databases.SCHEMAREGISTRY.name }} database_user: {{ databases.SCHEMAREGISTRY.user }} database_password: {{ databases.SCHEMAREGISTRY.password }} +SQL_STREAM_BUILDER: + SERVICEWIDE: + database_host: {{ databases.SQL_STREAM_BUILDER.host }} + database_port: {{ databases.SQL_STREAM_BUILDER.port }} + database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }} + database_schema: {{ databases.SQL_STREAM_BUILDER.name }} + database_user: {{ databases.SQL_STREAM_BUILDER.user }} + database_password: {{ databases.SQL_STREAM_BUILDER.password }} + MATERIALIZED_VIEW_ENGINE: + ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }} + ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }} + ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }} STREAMS_MESSAGING_MANAGER: SERVICEWIDE: smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }} diff --git a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 index f3eca99c..095d9830 100644 --- a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 +++ b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 @@ -71,9 +71,11 @@ HUE: FLINK: SERVICEWIDE: hdfs_service: hdfs - hive_service: hive yarn_service: yarn - zookeeper_service: zoookeeper + zookeeper_service: zookeeper +{% if 'HIVE' in cluster.services %} + hive_service: hive +{% endif %} {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %} atlas_service: atlas {% endif %} @@ -174,6 +176,17 @@ SPARK3_ON_YARN: hive_service: hive {% endif %} +SQL_STREAM_BUILDER: + SERVICEWIDE: + flink_service: flink + kafka_service: kafka +{% if 'HIVE' in cluster.services %} + hive_service: hive +{% endif %} +{% if 'KNOX' in cluster.services %} + knox_service: knox +{% endif %} + TEZ: SERVICEWIDE: yarn_service: yarn diff --git a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 index 961f616a..59bf0bfd 100644 --- a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 +++ b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 @@ -2,6 +2,9 @@ CORE_SETTINGS: SERVICEWIDE: hadoop_secure_web_ui: true +FLINK: + SERVICEWIDE: + kerberos.auth.enabled: true HBASE: SERVICEWIDE: hbase_restserver_security_authentication: kerberos diff --git a/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 index bf0545d5..3b2473a2 100644 --- a/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 +++ b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 @@ -1,4 +1,26 @@ --- OZONE: OZONE_PROMETHEUS: - ozone.prometheus.ca.file: {{ tls_chain_path }} \ No newline at end of file + ozone.prometheus.ca.file: {{ tls_chain_path }} +SQL_STREAM_BUILDER: + STREAMING_SQL_ENGINE: + ssl_client_truststore_location: {{ tls_truststore_path }} + ssl_client_truststore_password: {{ tls_truststore_password }} + ssl_enabled: true + ssl_server_keystore_keypassword: {{ tls_keystore_password }} + ssl_server_keystore_location: {{ tls_keystore_path_generic }} + ssl_server_keystore_password: {{ tls_keystore_password }} + MATERIALIZED_VIEW_ENGINE: + ssl_client_truststore_location: {{ tls_truststore_path }} + ssl_client_truststore_password: {{ tls_truststore_password }} + ssl_enabled: true + ssl_server_keystore_keypassword: {{ tls_keystore_password }} + ssl_server_keystore_location: {{ tls_keystore_path_generic }} + ssl_server_keystore_password: {{ tls_keystore_password }} + STREAMING_SQL_CONSOLE: + ssl_client_truststore_location: {{ tls_chain_path }} + ssl_enabled: true + ssl_server_ca_certificate_location: {{ tls_chain_path }} + ssl_server_certificate_location: {{ tls_cert_path_generic }} + ssl_server_privatekey_location: {{ tls_key_path_generic }} + ssl_server_privatekey_password: {{ tls_key_password }} diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml index 867a382e..3cd52b8e 100644 --- a/roles/config/cluster/common/defaults/main.yml +++ b/roles/config/cluster/common/defaults/main.yml @@ -75,6 +75,7 @@ cluster_services_ordered: - CRUISE_CONTROL - DAS - FLINK + - SQL_STREAM_BUILDER - SPARK - SPARK2_ON_YARN - SPARK3_ON_YARN diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml index b2d6ffff..996232a7 100644 --- a/roles/deployment/definition/defaults/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -99,6 +99,20 @@ database_defaults: name: schemaregistry user: schemaregistry password: "{{ database_default_password }}" + SQL_STREAM_BUILDER: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: ssb_admin + user: ssb_admin + password: "{{ database_default_password }}" + SQL_STREAM_BUILDER_MVE: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: ssb_mve + user: ssb_mve + password: "{{ database_default_password }}" STREAMS_MESSAGING_MANAGER: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" diff --git a/roles/deployment/repometa/templates/all_services.j2 b/roles/deployment/repometa/templates/all_services.j2 index 8c495bbd..4f2f412d 100644 --- a/roles/deployment/repometa/templates/all_services.j2 +++ b/roles/deployment/repometa/templates/all_services.j2 @@ -1,27 +1,7 @@ -- KEYTRUSTEE_SERVER -- ZOOKEEPER -- INFRA_SOLR -- RANGER -- RANGER_KMS -- RANGER_KMS_KTS -- RANGER_RAZ -- KMS -- HDFS -- SENTRY -- LIVY -- HBASE -- SOLR -- SQOOP -- SQOOP_CLIENT - ACCUMULO16 - ACCUMULO_C6 -- HIVE -- TEZ -- HIVE_ON_TEZ -- KUDU -- IMPALA -- YARN - ADLS_CONNECTOR +- ATLAS - AWS_S3 - CDSW - CORE_SETTINGS @@ -29,29 +9,50 @@ - DAS - FLINK - FLUME -- SPARK -- SPARK2_ON_YARN -- SPARK3_ON_YARN -- SPARK_ON_YARN +- HBASE +- HDFS +- HIVE +- HIVE_ON_TEZ +- HUE +- IMPALA +- INFRA_SOLR - ISILON - KAFKA - KEYTRUSTEE +- KEYTRUSTEE_SERVER +- KMS - KNOX - KS_INDEXER +- KUDU +- LIVY - LUNA_KMS - MAPREDUCE - NIFI - NIFIREGISTRY - NIFITOOLKITCA +- OOZIE - OZONE - PHOENIX - QUEUEMANAGER +- RANGER +- RANGER_KMS +- RANGER_KMS_KTS +- RANGER_RAZ - SCHEMAREGISTRY +- SENTRY +- SOLR +- SPARK +- SPARK2_ON_YARN +- SPARK3_ON_YARN +- SPARK_ON_YARN +- SQL_STREAM_BUILDER +- SQOOP +- SQOOP_CLIENT - STREAMS_MESSAGING_MANAGER - STREAMS_REPLICATION_MANAGER +- TEZ - THALES_KMS - WXM +- YARN - ZEPPELIN -- ATLAS -- OOZIE -- HUE +- ZOOKEEPER diff --git a/roles/deployment/repometa/templates/role_mappings/cdh7.j2 b/roles/deployment/repometa/templates/role_mappings/cdh7.j2 index d5dc2a63..4124719a 100644 --- a/roles/deployment/repometa/templates/role_mappings/cdh7.j2 +++ b/roles/deployment/repometa/templates/role_mappings/cdh7.j2 @@ -11,6 +11,8 @@ CRUISE_CONTROL: DAS: - DAS_EVENT_PROCESSOR - DAS_WEBAPP +FLINK: + - FLINK_HISTORY_SERVER HBASE: - GATEWAY - HBASERESTSERVER @@ -103,6 +105,10 @@ SOLR: SPARK_ON_YARN: - GATEWAY - SPARK_YARN_HISTORY_SERVER +SQL_STREAM_BUILDER: + - STREAMING_SQL_ENGINE + - MATERIALIZED_VIEW_ENGINE + - STREAMING_SQL_CONSOLE SQOOP_CLIENT: - GATEWAY STREAMS_MESSAGING_MANAGER: diff --git a/roles/prereqs/local_accounts_common/defaults/main.yml b/roles/prereqs/local_accounts_common/defaults/main.yml index d288e442..eb8c90c3 100644 --- a/roles/prereqs/local_accounts_common/defaults/main.yml +++ b/roles/prereqs/local_accounts_common/defaults/main.yml @@ -48,6 +48,13 @@ local_accounts: comment: Flink keystore_acl: True + - user: ssb + home: /var/lib/ssb + comment: SQL Stream Builder + keystore_acl: True + key_acl: True + key_password_acl: True + - user: flume home: /var/lib/flume-ng comment: Flume