From e87bd1ea2fcfecd9ef200085cf06122872a4aa99 Mon Sep 17 00:00:00 2001 From: Andre Araujo Date: Sat, 30 Apr 2022 00:33:10 +0800 Subject: [PATCH] Adding support for SQL Stream Builder deployment (#48) * New role to deploy the PostgreSQL connector Signed-off-by: Andre Araujo * Added support for SQL Stream Builder Signed-off-by: Andre Araujo --- .../base/templates/configs/databases-7.1.0.j2 | 12 ++++ .../configs/inter-service-dependencies.j2 | 17 +++++- .../templates/configs/kerberos-6.x-7.x.j2 | 3 + .../base/templates/configs/tls-7.3.1.j2 | 24 +++++++- roles/config/cluster/common/defaults/main.yml | 1 + roles/deployment/databases/tasks/mariadb.yml | 4 +- .../deployment/databases/tasks/postgresql.yml | 4 +- roles/deployment/definition/defaults/main.yml | 14 +++++ .../repometa/templates/all_services.j2 | 57 ++++++++++--------- .../repometa/templates/role_mappings/flink.j2 | 4 ++ .../local_accounts_common/defaults/main.yml | 7 +++ .../postgresql_connector/defaults/main.yml | 18 ++++++ .../postgresql_connector/meta/main.yml | 17 ++++++ .../postgresql_connector/tasks/main.yml | 37 ++++++++++++ 14 files changed, 184 insertions(+), 35 deletions(-) create mode 100644 roles/prereqs/postgresql_connector/defaults/main.yml create mode 100644 roles/prereqs/postgresql_connector/meta/main.yml create mode 100644 roles/prereqs/postgresql_connector/tasks/main.yml diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 index 81668d8e..7a88a016 100644 --- a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 @@ -31,6 +31,18 @@ SCHEMAREGISTRY: database_name: {{ databases.SCHEMAREGISTRY.name }} database_user: {{ databases.SCHEMAREGISTRY.user }} database_password: {{ databases.SCHEMAREGISTRY.password }} +SQL_STREAM_BUILDER: + SERVICEWIDE: + database_host: {{ databases.SQL_STREAM_BUILDER.host }} + database_port: {{ databases.SQL_STREAM_BUILDER.port }} + database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }} + database_schema: {{ databases.SQL_STREAM_BUILDER.name }} + database_user: {{ databases.SQL_STREAM_BUILDER.user }} + database_password: {{ databases.SQL_STREAM_BUILDER.password }} + MATERIALIZED_VIEW_ENGINE: + ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }} + ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }} + ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }} STREAMS_MESSAGING_MANAGER: SERVICEWIDE: smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }} diff --git a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 index f3eca99c..095d9830 100644 --- a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 +++ b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 @@ -71,9 +71,11 @@ HUE: FLINK: SERVICEWIDE: hdfs_service: hdfs - hive_service: hive yarn_service: yarn - zookeeper_service: zoookeeper + zookeeper_service: zookeeper +{% if 'HIVE' in cluster.services %} + hive_service: hive +{% endif %} {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %} atlas_service: atlas {% endif %} @@ -174,6 +176,17 @@ SPARK3_ON_YARN: hive_service: hive {% endif %} +SQL_STREAM_BUILDER: + SERVICEWIDE: + flink_service: flink + kafka_service: kafka +{% if 'HIVE' in cluster.services %} + hive_service: hive +{% endif %} +{% if 'KNOX' in cluster.services %} + knox_service: knox +{% endif %} + TEZ: SERVICEWIDE: yarn_service: yarn diff --git a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 index b7dd4b11..b7b9569a 100644 --- a/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 +++ b/roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2 @@ -2,6 +2,9 @@ CORE_SETTINGS: SERVICEWIDE: hadoop_secure_web_ui: true +FLINK: + SERVICEWIDE: + kerberos.auth.enabled: true HBASE: SERVICEWIDE: hbase_restserver_security_authentication: kerberos diff --git a/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 index bf0545d5..3b2473a2 100644 --- a/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 +++ b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 @@ -1,4 +1,26 @@ --- OZONE: OZONE_PROMETHEUS: - ozone.prometheus.ca.file: {{ tls_chain_path }} \ No newline at end of file + ozone.prometheus.ca.file: {{ tls_chain_path }} +SQL_STREAM_BUILDER: + STREAMING_SQL_ENGINE: + ssl_client_truststore_location: {{ tls_truststore_path }} + ssl_client_truststore_password: {{ tls_truststore_password }} + ssl_enabled: true + ssl_server_keystore_keypassword: {{ tls_keystore_password }} + ssl_server_keystore_location: {{ tls_keystore_path_generic }} + ssl_server_keystore_password: {{ tls_keystore_password }} + MATERIALIZED_VIEW_ENGINE: + ssl_client_truststore_location: {{ tls_truststore_path }} + ssl_client_truststore_password: {{ tls_truststore_password }} + ssl_enabled: true + ssl_server_keystore_keypassword: {{ tls_keystore_password }} + ssl_server_keystore_location: {{ tls_keystore_path_generic }} + ssl_server_keystore_password: {{ tls_keystore_password }} + STREAMING_SQL_CONSOLE: + ssl_client_truststore_location: {{ tls_chain_path }} + ssl_enabled: true + ssl_server_ca_certificate_location: {{ tls_chain_path }} + ssl_server_certificate_location: {{ tls_cert_path_generic }} + ssl_server_privatekey_location: {{ tls_key_path_generic }} + ssl_server_privatekey_password: {{ tls_key_password }} diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml index b31436c5..eba37e42 100644 --- a/roles/config/cluster/common/defaults/main.yml +++ b/roles/config/cluster/common/defaults/main.yml @@ -81,6 +81,7 @@ cluster_services_ordered: - CRUISE_CONTROL - DAS - FLINK + - SQL_STREAM_BUILDER - SPARK - SPARK2_ON_YARN - SPARK3_ON_YARN diff --git a/roles/deployment/databases/tasks/mariadb.yml b/roles/deployment/databases/tasks/mariadb.yml index 3067f963..5ababe0d 100644 --- a/roles/deployment/databases/tasks/mariadb.yml +++ b/roles/deployment/databases/tasks/mariadb.yml @@ -20,7 +20,7 @@ encoding: "{{ service | cloudera.cluster.get_database_encoding_mysql }}" collation: "{{ service | cloudera.cluster.get_database_collation_mysql }}" become: yes - loop: "{{ databases | intersect(services) }}" + loop: "{{ databases }}" loop_control: loop_var: service delegate_to: "{{ databases[service].host }}" @@ -36,7 +36,7 @@ priv: "{{ databases[service].name }}.*:ALL" no_log: yes become: yes - loop: "{{ databases | intersect(services) }}" + loop: "{{ databases }}" loop_control: loop_var: service delegate_to: "{{ databases[service].host }}" diff --git a/roles/deployment/databases/tasks/postgresql.yml b/roles/deployment/databases/tasks/postgresql.yml index 8bb07d56..eb9793a7 100644 --- a/roles/deployment/databases/tasks/postgresql.yml +++ b/roles/deployment/databases/tasks/postgresql.yml @@ -20,7 +20,7 @@ password: "{{ databases[item].password }}" become: yes become_user: postgres - with_items: "{{ databases | intersect(services) }}" + with_items: "{{ databases }}" delegate_to: "{{ databases[item].host }}" connection: ssh when: databases[item].host in groups.db_server @@ -32,7 +32,7 @@ encoding: UTF-8 become: yes become_user: postgres - with_items: "{{ databases | intersect(services) }}" + with_items: "{{ databases }}" delegate_to: "{{ databases[item].host }}" connection: ssh when: databases[item].host in groups.db_server diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml index bcae31e6..64b072d2 100644 --- a/roles/deployment/definition/defaults/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -179,6 +179,20 @@ database_defaults: name: schemaregistry user: schemaregistry password: "{{ database_default_password }}" + SQL_STREAM_BUILDER: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: ssb_admin + user: ssb_admin + password: "{{ database_default_password }}" + SQL_STREAM_BUILDER_MVE: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: "{{ database_type }}" + name: ssb_mve + user: ssb_mve + password: "{{ database_default_password }}" STREAMS_MESSAGING_MANAGER: host: "{{ database_host }}" port: "{{ database_type | cloudera.cluster.default_database_port }}" diff --git a/roles/deployment/repometa/templates/all_services.j2 b/roles/deployment/repometa/templates/all_services.j2 index 8c495bbd..4f2f412d 100644 --- a/roles/deployment/repometa/templates/all_services.j2 +++ b/roles/deployment/repometa/templates/all_services.j2 @@ -1,27 +1,7 @@ -- KEYTRUSTEE_SERVER -- ZOOKEEPER -- INFRA_SOLR -- RANGER -- RANGER_KMS -- RANGER_KMS_KTS -- RANGER_RAZ -- KMS -- HDFS -- SENTRY -- LIVY -- HBASE -- SOLR -- SQOOP -- SQOOP_CLIENT - ACCUMULO16 - ACCUMULO_C6 -- HIVE -- TEZ -- HIVE_ON_TEZ -- KUDU -- IMPALA -- YARN - ADLS_CONNECTOR +- ATLAS - AWS_S3 - CDSW - CORE_SETTINGS @@ -29,29 +9,50 @@ - DAS - FLINK - FLUME -- SPARK -- SPARK2_ON_YARN -- SPARK3_ON_YARN -- SPARK_ON_YARN +- HBASE +- HDFS +- HIVE +- HIVE_ON_TEZ +- HUE +- IMPALA +- INFRA_SOLR - ISILON - KAFKA - KEYTRUSTEE +- KEYTRUSTEE_SERVER +- KMS - KNOX - KS_INDEXER +- KUDU +- LIVY - LUNA_KMS - MAPREDUCE - NIFI - NIFIREGISTRY - NIFITOOLKITCA +- OOZIE - OZONE - PHOENIX - QUEUEMANAGER +- RANGER +- RANGER_KMS +- RANGER_KMS_KTS +- RANGER_RAZ - SCHEMAREGISTRY +- SENTRY +- SOLR +- SPARK +- SPARK2_ON_YARN +- SPARK3_ON_YARN +- SPARK_ON_YARN +- SQL_STREAM_BUILDER +- SQOOP +- SQOOP_CLIENT - STREAMS_MESSAGING_MANAGER - STREAMS_REPLICATION_MANAGER +- TEZ - THALES_KMS - WXM +- YARN - ZEPPELIN -- ATLAS -- OOZIE -- HUE +- ZOOKEEPER diff --git a/roles/deployment/repometa/templates/role_mappings/flink.j2 b/roles/deployment/repometa/templates/role_mappings/flink.j2 index 9a063735..03e757f0 100644 --- a/roles/deployment/repometa/templates/role_mappings/flink.j2 +++ b/roles/deployment/repometa/templates/role_mappings/flink.j2 @@ -1,3 +1,7 @@ FLINK: - FLINK_HISTORY_SERVER - GATEWAY +SQL_STREAM_BUILDER: + - STREAMING_SQL_ENGINE + - MATERIALIZED_VIEW_ENGINE + - STREAMING_SQL_CONSOLE diff --git a/roles/prereqs/local_accounts_common/defaults/main.yml b/roles/prereqs/local_accounts_common/defaults/main.yml index d288e442..eb8c90c3 100644 --- a/roles/prereqs/local_accounts_common/defaults/main.yml +++ b/roles/prereqs/local_accounts_common/defaults/main.yml @@ -48,6 +48,13 @@ local_accounts: comment: Flink keystore_acl: True + - user: ssb + home: /var/lib/ssb + comment: SQL Stream Builder + keystore_acl: True + key_acl: True + key_password_acl: True + - user: flume home: /var/lib/flume-ng comment: Flume diff --git a/roles/prereqs/postgresql_connector/defaults/main.yml b/roles/prereqs/postgresql_connector/defaults/main.yml new file mode 100644 index 00000000..f6baf898 --- /dev/null +++ b/roles/prereqs/postgresql_connector/defaults/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +postgresql_connector_url: https://jdbc.postgresql.org/download/postgresql-42.3.3.jar +postgresql_connector_checksum: md5:bef0b2e1c6edcd8647c24bed31e1a4ac diff --git a/roles/prereqs/postgresql_connector/meta/main.yml b/roles/prereqs/postgresql_connector/meta/main.yml new file mode 100644 index 00000000..f63fb6e8 --- /dev/null +++ b/roles/prereqs/postgresql_connector/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.deployment.definition diff --git a/roles/prereqs/postgresql_connector/tasks/main.yml b/roles/prereqs/postgresql_connector/tasks/main.yml new file mode 100644 index 00000000..3ec10f16 --- /dev/null +++ b/roles/prereqs/postgresql_connector/tasks/main.yml @@ -0,0 +1,37 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Download PostgreSQL Connector + get_url: + url: "{{ postgresql_connector_url }}" + dest: "{{ local_temp_dir }}/postgresql-connector-java.jar" + checksum: "{{ postgresql_connector_checksum }}" + mode: 0644 + become: no + run_once: true + delegate_to: localhost + +- name: Create /usr/share/java directory + file: + path: /usr/share/java + state: directory + mode: 0755 + +- name: Copy PostgreSQL Connector jar file to correct location + copy: + src: "{{ local_temp_dir }}/postgresql-connector-java.jar" + dest: /usr/share/java/postgresql-connector-java.jar + mode: 0644