diff --git a/README.md b/README.md index 509021e9..07e69d32 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Ansible Playbooks for Cloudera Data Platform +# Ansible Collection for Cloudera Private Cloud ## Requirements @@ -8,7 +8,10 @@ **Do not use Ansible 2.9.0**. This version has an [issue with templating](https://github.com/ansible/ansible/issues/64745) which causes the playbook execution to fail. Instead, use any 2.8.x version or a later 2.9.x version as these are not affected. -## Supported Platforms +## Usage +This Ansible Collection is designed to work hand-in-hand with [Cloudera Deploy](https://github.com/cloudera-labs/cloudera-deploy), which contains reference Playbooks and Example Definitions. + +## Tested Platforms ### Cloudera Distributions @@ -24,22 +27,7 @@ Active development is focused on **CDP Private Cloud Base** (formerly CDP-DC) deployments and their respective platform compatibility matrices. -> While these playbooks can be used to deploy CDH 5.x and CDH 6.x environments, it is only possible to install a subset of their supported platform components (i.e JDK and database versions) using this tool. - -## Getting Started - -For help setting up the playbook, creating configs and deploying clusters, see the [Getting Started](docs/getting-started.md) guide. - -## How-to Guides - -For more detailed information, check the following guides: - -* Deploying [secure clusters](docs/security.md) -* Deploying [data contexts (SDX) and virtual private clusters](docs/how-to/virtual-private-clusters.md) - -## Help! - -Common issues and their solutions are documented on the [Troubleshooting](docs/troubleshooting.md) page. Check here first. +> While these roles etc. can be used to deploy CDH 5.x and CDH 6.x environments, it is only possible to install a subset of their supported platform components (i.e JDK and database versions) using this tooling. ## How do I contribute code? You need to first sign and return an diff --git a/create_freeipa.yml b/create_freeipa.yml deleted file mode 100644 index 4d473424..00000000 --- a/create_freeipa.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install FreeIPA server - hosts: krb5_server - become: yes - roles: - - role: freeipa.ansible_freeipa.ipaserver - state: present - ipaserver_realm: "{{ krb5_realm }}" - ipaserver_domain: "{{ krb5_domain | default(krb5_realm | lower) }}" - ipaserver_setup_firewalld: "no" - when: "krb5_kdc_type == 'Red Hat IPA'" - tags: - - security - - kerberos - - tls - -- name: Setup FreeIPA clients - hosts: cloudera_manager:cluster - become: yes - roles: - - role: freeipa.ansible_freeipa.ipaclient - state: present - ipaserver_realm: "{{ krb5_realm }}" - ipaserver_domain: "{{ krb5_domain | default(krb5_realm | lower) }}" - ipaclient_servers: "{{ groups['krb5_server'] }}" - when: "krb5_kdc_type == 'Red Hat IPA' and 'krb5_server' in groups" - tasks: - - name: Set sssd to enumerate users and groups - lineinfile: - path: /etc/sssd/sssd.conf - insertafter: "^\\[domain/.+\\]" - regexp: "^enumerate" - line: "enumerate = True" - when: "krb5_kdc_type == 'Red Hat IPA' and 'krb5_server' in groups" - notify: - - restart sssd - handlers: - - name: restart sssd - service: - name: sssd - state: restarted - tags: - - security - - kerberos - - tls - diff --git a/create_infrastructure.yml b/create_infrastructure.yml deleted file mode 100644 index 5885d0d1..00000000 --- a/create_infrastructure.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install custom parcel repository - hosts: custom_repo - become: yes - roles: - - infrastructure/custom_repo - tags: - - custom_repo - -- name: Install RDBMS - hosts: db_server - become: yes - roles: - - infrastructure/rdbms - tags: - - database - -- name: Install MIT KDC - hosts: krb5_server - become: yes - roles: - - role: infrastructure/krb5_server - when: "krb5_kdc_type == 'MIT KDC'" - tags: - - security - - kerberos - -- name: Install CA server - hosts: ca_server - become: yes - roles: - - infrastructure/ca_server - vars_files: - - vars/ca_server.yml - tags: - - security - - tls - -- name: Install HAProxy - hosts: haproxy - become: yes - roles: - - infrastructure/haproxy - tags: - - ha diff --git a/docs/definitions.md b/docs/definitions.md deleted file mode 100644 index 0a3b3c0a..00000000 --- a/docs/definitions.md +++ /dev/null @@ -1,42 +0,0 @@ -# Example Cluster Definitions - -## CDP Private Cloud Base 7.1.x - -### Basic - -TODO - -### "Express Wizard" - -These example templates are designed to replicate the options of Cloudera Manager's cluster installation wizard. - -- **Data Engineering** (`7.1.1/data-engineering`) - - This template includes a wide range of services. - -- **Data Mart** (`7.1.1/data-mart`) - - Includes a more limited range of services, specialising on interactive SQL analysis. - -- **Operational Database** (`7.1.1/operational-db`) - - Includes an even more limited range of services, specialising around HBase. - -### CDSW - -TODO - -### Secure - -TODO - -## CDH 6.3.x - -TODO - -## CDH 5.16.x - -TODO - - - diff --git a/docs/freeipa.md b/docs/freeipa.md deleted file mode 100644 index 6cd71c91..00000000 --- a/docs/freeipa.md +++ /dev/null @@ -1,141 +0,0 @@ -# FreeIPA - -FreeIPA is a product that provides (not exhaustive): - -- CA -- KDC -- LDAP -- Host management (including SSSD) - -The playbook is able to provision a FreeIPA server, or use an existing FreeIPA server, and automatically setup the cluster to make use of the CA, KDC and LDAP. - -Before continuing, a few questions you may want to ask yourself are: - -1. Are you using an existing FreeIPA server(s) or do you want the playbook to provision one? -2. Do you want to use the FreeIPA CA to sign the certificates or do you want to sign them externally? -3. Are you using AutoTLS or do you want the playbook to configure the host keys and certificates? - -## Common steps - -Regardless of how you choose to use FreeIPA in your deployment, you'll have to set the following variables via extra vars: - -- `krb5_realm` (e.g. `CLOUDERA.LOCAL`) -- `krb5_kdc_admin_user` (e.g. the FreeIPA default `admin@{{ krb5_realm }}`) -- `krb5_kdc_admin_password` (e.g. `{{ ipaadmin_password }}`) -- `krb5_enc_types` (e.g. `aes256-cts aes128-cts`) - -You must also set `krb5_kdc_type: "Red Hat IPA"`. - -## Existing FreeIPA or playbook-provisioned? - -### Existing FreeIPA - -This case is simple: - -Please set `krb5_kdc_host` to you FreeIPA server hostname. - -### Playbook-provisioned - -Here, you'll need to add a host to `krb5_server` and set the following variables: - -- `ipaadmin_password` -- `ipadm_password` - -The playbook will recognize that `krb5_kdc_type` is set to `Red Hat IPA` and bring up a FreeIPA server instead of MIT KDC. - -The playbook will not provision a firewall around the FreeIPA server. - -## FreeIPA CA signed certificates or externally signed certificates? - -In both cases, you'll want to refer to each CA certificate used (particularly important if you are using a different CA) by adding entries to `tls_ca_certs` e.g. (IPA CA) - -``` -tls_ca_certs: - - path: /etc/ipa/ca.crt - alias: ipaca -``` - -### FreeIPA CA signed certificates - -Here, nothing has to be done. - -Provided each host is enrolled as a FreeIPA client then the playbook will automatically sign (~~and enable the renewal of~~ ZOOKEEPER-3832) the host certificates using the hosts principal. - -### Externally signed certificates - -In this case, please set `skip_ipa_signing` to `true`. - -This will cause the playbook to stop after generating CSRs – identical to the non-FreeIPA case. - -## AutoTLS or playbook configured? - -### AutoTLS - -#### CM provisioned CA - -Remove any mention of TLS from the cluster definition and enable AutoTLS using the API or wizard. - -You may need to add the FreeIPA to the CA certs (via the API or wizard). - -#### FreeIPA or externally provisioned certificates - -Here, you'll want to unset any TLS configurations in the `cluster.yml` file. This is because AutoTLS takes on the role of configuring the cluster here. - -You'll then need to enable AutoTLS using the certificates provisioned (by default) under `/opt/cloudera/security/pki` using the API https://blog.cloudera.com/auto-tls-in-cloudera-data-platform-data-center/ immediately after installing the Cloudera Manager. - -You can then continue with the playbook installation. - -## LDAP configs - -To setup LDAP in CM and services automatically, we'll need to first define an auth provider. - -For FreeIPA, it might look something like: - -``` -base_dn: "dc={{ (krb5_realm | lower).split('.') | join(',dc=') }}" -user_dn: "cn=users,cn=accounts,{{ base_dn }}" -group_dn: "cn=groups,cn=accounts,{{ base_dn }}" - -auth_providers: - freeipa: - ldap_bind_user_dn: "uid=admin,{{ user_dn }}" - ldap_bind_password: "{{ ipaadmin_password }}" - ldap_search_base: - user: "{{ user_dn }}" - group: "{{ group_dn }}" - ldap_object_class: - user: "person" - group: "groupofnames" - ldap_attribute: - user: "uid" - group: "cn" - member: "member" - type: LDAP - ldap_url: "ldaps://{{ groups.krb5_server | first }}" -``` - -Once the auth provider is defined, we need to configure the playbook to use it to: - -Configure CM: - -``` -cloudera_manager_external_auth: - provider: freeipa - external_first: yes - external_only: no - role_mappings: - - group: CMTestGroup1 - roles: [ROLE_ADMIN] -``` - -Configure the services: - -``` -service_auth_provider: freeipa -``` - -## YCloud - -YCloud will work, but you will need to change the group running `systemd` (not pretty): - -`yum install -y gdb && yes | gdb -p 1 --ex 'call setgid(0)' --ex quit` diff --git a/docs/getting-started.md b/docs/getting-started.md deleted file mode 100644 index 3aad3266..00000000 --- a/docs/getting-started.md +++ /dev/null @@ -1,180 +0,0 @@ -# Getting Started - -Before you begin, ensure that Ansible and its dependencies are all properly configured as per the [Requirements](/README.md#requirements). - -``` -[root@localhost cloudera-playbook]# ansible --version -ansible 2.9.9 - config file = /root/.ansible.cfg - configured module search path = ['/root/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules'] - ansible python module location = /usr/local/lib/python3.6/site-packages/ansible - executable location = /usr/local/bin/ansible - python version = 3.6.8 (default, Aug 7 2019, 17:28:10) [GCC 4.8.5 20150623 (Red Hat 4.8.5-39)] -``` - -All the following steps should be done wherever Ansible is installed: - -## 1. Clone the repository - -If you are working from a fork, replace the repository URL below with your own. - -``` -$ git clone https://github.com/cloudera-labs/cloudera.cluster.git -$ cd cloudera.cluster -``` - -## 2. Install dependencies - -``` -$ ansible-galaxy role install -r requirements.yml -$ ansible-galaxy collection install -r requirements.yml -``` - -**Ansible 2.9.x or below** - -This step is required only when planning to provision database servers. We use roles by [Jeff Geerling](https://github.com/geerlingguy) from [Ansible Galaxy](https://galaxy.ansible.com/) for installing MySQL / MariaDB or PostgreSQL. - -**Ansible 2.10.x or above** - -Installing dependencies is **always** necessary for 2.10.x because from this version forward, much of Ansible's functionality has been pulled out of core and into [collections](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html). - -The additional Ansible collections required by these playbooks are: - -- [`ansible.posix`](https://galaxy.ansible.com/ansible/posix) -- [`community.crypto`](https://galaxy.ansible.com/community/crypto) -- [`community.general`](https://galaxy.ansible.com/community/general) -- [`freeipa.ansible_freeipa`](https://galaxy.ansible.com/freeipa/ansible_freeipa) - -## 3. Create a secrets file (optional, but recommended) - -Passwords are required at various points during the playbook run. It is recommended to use [Ansible Vault](https://docs.ansible.com/ansible/latest/user_guide/vault.html) and create an encrypted file in which to store these passwords and other sensitive variables securely. - -To create an encrypted secrets file, use the `ansible-vault` command: - -```bash -$ ansible-vault create secrets.yml -``` - -After choosing a password of your choice an editor opens, add at least the following two variables for Cloudera paywall credentials: - -```yaml -vault__cloudera_manager_repo_username: yourusername -vault__cloudera_manager_repo_password: yourpassword -``` - -> It is recommended to use the prefix `vault__` for your encrypted variable names in order to easily tell them apart from standard, non encrypted variables defined elsewhere. - -You could also include a variable containing database passwords, e.g: - -```yaml -vault__cloudera_database_passwords: - scm: password1 - hive: password2 - hue: password3 - oozie: password4 -... etc ... -``` - -The [docs](how-to/database-configuration.md) go through how these database password variables would be used in practice. - -If you want to include other passwords/secrets here for use in your config files, then add these to your vault file too. - -Closing the editor saves and encrypts the file. We can now store this file a git repository without any risk of exposing the credentials inside. - -## 4. Create an environment "definition" - -An environment definition is a directory with two files: - -- `extra_vars.yml` -- `cluster.yml` - -The file `extra_vars.yml` contains general variables to override during playbook execution. Documentation for how these variables affect individual roles is contained in the roles' own documentation page (inside `docs` folder). - -The file `cluster.yml` contains special variables specifically for cluster deployment: services, roles and configs. - -An example definition is specified in the [samples](../examples/sample) folder. You can use these as a base and customise as required for your own cluster design or requirements. - -## 5. Create an inventory file - -An inventory file is a grouped list which informs what the playbook will actually do, and to which servers. Some inventory groups are required, others are optional. - -Full details can be found in the [inventories](inventories.md) documentation page, and examples are provided inside the `examples` subfolder. - -> General guidance about inventories can also be found in [Ansible documentation](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html). - -## 6. Verify connectivity to your servers - -The Ansible node will need to connect to all the servers in your inventory via SSH. By default, Ansible assumes you are using SSH keys to connect to remote machines. This is the recommended method. - -If you cannot set up passwordless SSH using keys, and instead you need to enter a password to authenticate, then you must include the `-k/--ask-pass` option to your `ansible-playbook` command and type in the password before the playbook run can start. - -## 7. Run core playbooks for deployment - -```bash -$ ansible-playbook -i /path/to/your-inventory-file \ - --extra-vars @/path/to/definitions/sample/extra_vars.yml \ - --extra-vars @secrets.yml \ - --ask-vault-pass \ - site.yml -``` - -> To avoid having to enter a vault password every time, set up a vault password file as explained in [Ansible Vault docs](https://docs.ansible.com/ansible/latest/user_guide/vault.html#providing-vault-passwords). - -The `site.yml` playbook imports a number of other playbooks to run a full deployment end-to-end: - -### Core playbooks - -- [`verify_everything.yml`](playbooks/verify_everything.md) - - Runs a series of pre-deployment checks against the inventory and cluster definition files to surface problems early and prevent failures much later in the process. - -- [`create_infrastructure.yml`](playbooks/create_infrastructure.md) - - Creates platform level infrastructure components, if required. Examples: MIT KDC, RDBMS (MariaDB or PostgreSQL), TLS CA server, or HAProxy. - -- [`prepare_nodes.yml`](playbooks/prepare_nodes.md) - - Install required software packages and apply pre-requisite operating system configurations. - -- [`install_cloudera_manager.yml`](playbooks/install_cloudera_manager.md) - - Install Cloudera Manager Server and agents, configure its database and apply license if available. - -- [`prepare_security.yml`](playbooks/prepare_security.md) - - Install required software packages and apply pre-requisite configurations for Kerberos authentication. - -- [`install_cluster.yml`](playbooks/install_cluster.md) - - Deploy Cloudera Management Service and a CDH or CDP Private Cloud Base cluster. - -- [`setup_hdfs_encryption.yml`](playbooks/setup_hdfs_encryption.md) - - Additional steps to configure HDFS encryption when relevant configs and inventory groups for KMS and Key Trustee servers has been provided. - -You are free to run any of these playbooks independently. For example, to only install Cloudera Manager you could run just this playbook in place of `site.yml`: - -``` -$ cd /path/to/cloudera.cluster -$ ansible-playbook -i /path/to/your-inventory-file \ - --extra-vars @/path/to/definitions/7.1.x/basic/extra_vars.yml \ - --extra-vars @secrets.yml \ - --ask-vault-pass \ - install_cloudera_manager.yml -``` - -## 8. Run extra playbooks for post-install configuration (optional) - -Some playbooks are provided but are not included in `site.yml`. - -These are for optional functionality that can be layered onto an existing deployment, such as security or high availability configuration. - -### Extra playbooks available - -- [`teardown.yml`](playbooks/teardown.md) - - This playbook attempts to clean-up the hosts, that already have clusters installed (or partially installed), to allow to user to install a new environment. - -- `enable_autotls.yml` - - Enables Auto-TLS on an existing Cloudera Manager and CDP Private Cloud Base 7.1 environment. diff --git a/docs/how-to/clear-ca-server-state.md b/docs/how-to/clear-ca-server-state.md deleted file mode 100644 index f9d78218..00000000 --- a/docs/how-to/clear-ca-server-state.md +++ /dev/null @@ -1,59 +0,0 @@ -# How to: Clear CA Server state - -The CA Server (using `openssl ca`) can be used to sign the host certificates used to deploy the cluster. - -There may be cases where you wish to replace the host keys and certificates. - -# Replace the CA - -On each host in the cluster, move or delete the following directory: - -``` -/opt/cloudera/security/pki -``` - -On the `ca_server` host, move or delete the following directory: - -``` -/ca -``` - -Re-run the playbook. - -At a minimum, re-run the following in the order they appear: - -- `create_infrastructure.yml` -- `prepare_tls.yml` - -# Keep the CA but replace host certificates - -On each host in the cluster where you wish to replace the host keys and certificates, move or delete the following directory: - -``` -/opt/cloudera/security/pki -``` - -On the `ca_server` host, execute the following for each host you wish to replace: - -``` -openssl ca -config /ca/intermediate/openssl.cnf -revoke /ca/intermediate/cert/.pem -passin pass: -``` - -__Note:__ The default CA password is `password` and can be configured by setting: - -- `ca_server_root_key_password` -- `ca_server_intermediate_key_password` - -Now move or delete the corresponding host certificates from: - -``` -/ca/intermediate/cert -``` - -Re-run the playbook. - -At a minimum, re-run `prepare_tls.yml`. - -__Note:__ You can skip the revoke step by setting `unique_subject` in `/ca/intermediate/index.txt.attr` to `no`. - -__Note:__ The CA Server is meant for testing only. Please use an enterprise CA Server in production. diff --git a/docs/how-to/database-configuration.md b/docs/how-to/database-configuration.md deleted file mode 100644 index de12a62e..00000000 --- a/docs/how-to/database-configuration.md +++ /dev/null @@ -1,80 +0,0 @@ -# How to: Database Configuration - -This guide will run through the specifics of explicitly configuring databases for Cloudera Manager and the cluster services. - -It assumes you are already familiar with deploying standard clusters using these playbooks. - -### Create a cluster definition - -Create the cluster definition you wish to deploy. - -### Configure the Cloudera Manager database - -Add the following variables to `extra_vars.yml`: - -- `cloudera_manager_database_host` -- `cloudera_manager_database_name` -- `cloudera_manager_database_user` -- `cloudera_manager_database_password` - -You can also set the following variable if the Cloudera Manager database type is different from the global `database_type` setting: - -- `cloudera_manager_database_type` - -### Configure the cluster service databases - -**Note:** The type of every service database is expected match `database_type` (default postgres). - -For each cluster in the cluster list, add a dictionary entry `databases:`, level with `configs:`. - -Add an entry to the `databases:` dictionary for each service that requires a database. The service label should match the label in `services:` and `configs:`. - -For each service, set the following variables: - -- `name` (name of the database) -- `host` -- `user` -- `password` - -Following these changes, your clusters should look like: - -``` -clusters: - - name: Test Cluster - services: [HDFS, ...] - databases: - HIVE: - name: "metastore" - host: "db-1.example.com" - user: "hive" - password: "{{ hive_password }}" - ... - ... - ... -``` - -### Configure the mgmt databases - -**Note:** The type of every mgmt role database is expected match `database_type` (default postgres). - -To config mgmt databases, following the method explained above for clusters, placing `databases:` in `mgmt:`. - -Following these changes, mgmt should look like: - -``` -mgmt: - name: Cloudera Management Service - services: [REPORTSMANAGER, ...] - databases: - REPORTSMANAGER: - name: "rman" - host: "db-2.example.com" - user: "rman" - password: "{{ rman_password }}" - ... - ... -``` - -### **Deploy!** - -**Note:** If the configured database host is provisioned by the playbook (part of the `db_servers` host group) the database will be created automatically. diff --git a/docs/how-to/externally-signed.md b/docs/how-to/externally-signed.md deleted file mode 100644 index 904cb50d..00000000 --- a/docs/how-to/externally-signed.md +++ /dev/null @@ -1,100 +0,0 @@ -# How to: Setup externally signed TLS certificates - -The playbook is able to provision an internal CA and automatically sign the host certificates. - -However, it is more likely that an external CA will be used in production environments. - -This guide will run through specifics for deploying clusters using an external CA to sign the host certificates. - -At a high-level, the playbook generates and distributes the TLS keys and certificates in three steps: - -- Phase 1 (generation) - * Create the host keys - * Generate the keystores - * Generate the CSRs - * Copy the CSRs to the Ansible controller -- Phase 2 (signing) - * Sign the certificates -- Phase 3 (installation) - * Copy the signed certificates to the hosts - * Copy the CA certificates to the hosts - -When `ca_server` is used, this all occurs automatically. - -However, when an external CA is used, a few manual steps are required to complete the deployment: - -Starting with the definition, assuming nothing has been deployed: - -## Ensure that `ca_server` is **not** present in the inventory - -The `ca_server` is the playbook's internal CA. - -As we wish to use an external CA, it should be omitted from the inventory file. - -## Set `tls_ca_certs` to point to the external CA certificates - -Here, we need to set `tls_ca_certs` in `extra_vars.yml` to point to the external CA certificates (on the Ansible controller): - -``` -tls_ca_certs: - - alias: ipaca - path: /root/ca.crt -``` - -In this example, our CA certificate exists on the Ansible controller at `/root/ca.crt`. - -**Note:** If you have an intermediate CA and a root CA, please include both certificates here. - -## Run the playbook deployment as usual - -If everything is configured correctly, it should run as usual until we reach `prepare_tls.yml`. - -Here, in `prepare_tls.yml`, it will fail with a message: - -> Signed cert for could not be found. If manual signing is required, do this now and re-run the playbook with 'tls_signed_certs_dir' variable set. - -E.g. - -``` -TASK [security/tls_install_certs : fail] ************************************************************************************************************** -fatal: [host-1.example.com] FAILED! => {"changed": false, "msg": "\"Signed cert for host-1.example.com could not be found. If manual signing is required, do this now and re-run the playbook with 'tls_signed_certs_dir' variable set.\n"} -fatal: [host-2.example.com] FAILED! => {"changed": false, "msg": "\"Signed cert for host-1.example.com could not be found. If manual signing is required, do this now and re-run the playbook with 'tls_signed_certs_dir' variable set.\n"} -fatal: [host-3.example.com] FAILED! => {"changed": false, "msg": "\"Signed cert for host-1.example.com could not be found. If manual signing is required, do this now and re-run the playbook with 'tls_signed_certs_dir' variable set.\n"} -fatal: [host-4.example.com] FAILED! => {"changed": false, "msg": "\"Signed cert for host-1.example.com could not be found. If manual signing is required, do this now and re-run the playbook with 'tls_signed_certs_dir' variable set.\n"} -``` - -This is expected. - -At this point, the playbook will have generated the TLS keys and CSRs and copied the CSRs to the Ansible controller (stage 1). - -It is up to us now to sign the CSRs and copy the signed certificates back to the Ansible controller: - -## Sign the CSRs generated by the playbook - -You will find copies of the TLS CSRs in `{{ local_temp_dir }}/csrs` on the Ansible controller (default `/tmp/csrs`). - -``` -# ls /tmp/csrs -host-1.example.com.csr host-3.example.com.csr -host-2.example.com.csr host-4.example.com.csr -``` - -Sign the CSRs and copy the signed certificates to the Ansible controller using the same filename, replacing `.csr` with `.pem`. - -Place these signed certificates in `{{ local_temp_dir }}/certs` on the Ansible controller (default `/tmp/certs`). - -``` -# ls /tmp/certs -host-1.example.com.pem host-3.example.com.pem -host-2.example.com.pem host-4.example.com.pem -``` - -## Rerun the playbook - -The playbook can be restarted now that the signed certificates exist on the Ansible controller (stage 2). - -**Note:** To save time, you can start the playbook from `prepare_tls.yml` – skipping completed steps. - -The playbook will distribute the new signed certificates and continue as usual. - -No other steps are required. diff --git a/docs/how-to/oracle.md b/docs/how-to/oracle.md deleted file mode 100644 index e78abd18..00000000 --- a/docs/how-to/oracle.md +++ /dev/null @@ -1,169 +0,0 @@ -# How to: Oracle - -The guide will run through specifics for deploying clusters using Oracle. - -It assumes you are already familiar with deploying standard clusters using these playbooks. - -1. Create the databases and users - -As the Oracle database is external to the playbook, this step is not completed automatically. - -Here, it is important you ensure that each service has the required permissions and that each node can access the database. - -2. Ensure the JDBC driver is available - -The playbook will install the JDBC driver via maven. - -By default, the playbook will search for the public maven repository, however, this can be overridden by setting `maven_repo` in `extra_vars.yml` to your maven repository. - -3. Ensure the instant client is available - -The playbook will install the instant client using zip files local to the Ansible controller. - -Please upload the instant client basic and sdk zip to the Ansible controller and set `oracle_instantclient_basic_zip` and `oracle_instantclient_sdk_zip` to these locations. - -4. Configure `extra_vars.yml` with the connection information - -Here, you'll want to set: - -- `database_type` to `oracle` -- `database_host` to the Oracle host -- `database_name` to the Oracle database name (optional but helps below) -- `cloudera_manager_database_name` to the appropriate database -- `cloudera_manager_database_user` to the appropriate user -- `cloudera_manager_database_password` to the appropriate password - -5. Configure `cluster.yml` with the service database information - -This step is a bit more involved and is mostly identical to the steps required for configuring any other external database. - -Here, you'll want to add a `databases` dictionary to each cluster and the mgmt cluster (at the same level as `name`). - -This dictionary contains an entry for each service with a database in the cluster: - -e.g. For Hive: - -``` -HIVE: - host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" - type: "{{ database_type }}" - name: "{{ database_name }}" - user: hive - password: "{{ vault__database_hive }}" -``` - -E.g. Complete example: - -``` ---- -clusters: - - name: Cluster - services: [HDFS, HIVE, HIVE_ON_TEZ, OOZIE, TEZ, QUEUEMANAGER, YARN, ZOOKEEPER] - repositories: - - https://archive.cloudera.com/p/cdh7/7.1.4.2/parcels/ - databases: - HIVE: - host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" - type: "{{ database_type }}" - name: "{{ database_name }}" - user: hive - password: "{{ database_default_password }}" - OOZIE: - host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" - type: "{{ database_type }}" - name: "{{ database_name }}" - user: oozie - password: "{{ database_default_password }}" - configs: - ZOOKEEPER: - SERVICEWIDE: - zookeeper_datadir_autocreate: true - host_templates: - Master1: - HDFS: [NAMENODE, SECONDARYNAMENODE, HTTPFS] - HIVE: [HIVEMETASTORE, GATEWAY] - HIVE_ON_TEZ: [HIVESERVER2] - OOZIE: [OOZIE_SERVER] - QUEUEMANAGER: [QUEUEMANAGER_STORE, QUEUEMANAGER_WEBAPP] - TEZ: [GATEWAY] - YARN: [RESOURCEMANAGER, JOBHISTORY] - ZOOKEEPER: [SERVER] - Workers: - HDFS: [DATANODE] - HIVE: [GATEWAY] - HIVE_ON_TEZ: [GATEWAY] - TEZ: [GATEWAY] - YARN: [NODEMANAGER] - -mgmt: - name: Cloudera Management Service - services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - databases: - REPORTSMANAGER: - host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" - type: "{{ database_type }}" - name: "{{ database_name }}" - user: rman - password: "{{ database_default_password }}" - -hosts: - configs: - host_default_proc_memswap_thresholds: - warning: never - critical: never - host_memswap_thresholds: - warning: never - critical: never - host_config_suppression_agent_system_user_group_validator: true -``` - -Examples of cluster services requiring a database include: - -- DAS -- HIVE -- HUE -- OOZIE -- RANGER -- SCHEMAREGISTRY -- STREAMS_MESSAGING_MANAGER -- SENTRY - -Examples of CMS services requiring a database include: - -- ACTIVITYMONITOR -- NAVIGATOR -- NAVIGATORMETASERVER -- REPORTSMANAGER - -6. Deploy as usual - -Provided the steps above have been completed successfully, everything else should follow normally. - -7. Configure an oracle client for tablespace teardown (optional) - -Optionally, the playbook can be configured to delete each Oracle user's tablespace on teardown. - -Please first ensure you have access (via SSH) to a host that is able to access the Oracle database using the `sqlplus` client. - -Then, in `extra_vars.yml`, configure the following: - -- `teardown_oracle_preamb` (shell commands required to setup environment) -- `teardown_oracle_client_host` (hostname of database client) -- `teardown_oracle_user` (local user to become) - -E.g. - -``` -teardown_oracle_preamb: | - export ORACLE_HOME=/opt/oracle/product/19c/dbhome_1 - export PATH=${PATH}:${ORACLE_HOME}/bin - export ORACLE_SID=ORCLPDB2 -teardown_oracle_client_host: db-client-1.example.com -teardown_oracle_user: oracle -``` - -With these configs in place, teardown will clean up the tablespaces. diff --git a/docs/how-to/virtual-private-clusters.md b/docs/how-to/virtual-private-clusters.md deleted file mode 100644 index 8a08a0a2..00000000 --- a/docs/how-to/virtual-private-clusters.md +++ /dev/null @@ -1,95 +0,0 @@ -# How to: Deploy Virtual Private Clusters - -This guide will run through specifics for deploying Virtual Private Clusters. - -It assumes you are already familiar with deploying standard clusters using these playbooks. - -For a good introduction to Virtual Private Clusters, read the Cloudera blog post [Improving Multi-tenancy with Virtual Private Clusters](https://blog.cloudera.com/improving-multi-tenancy-with-virtual-private-clusters/) by Tom Deane. Also, before you start, review the [Cloudera documentation for VPCs](https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_sdx_vpc.html) to learn about [compatibility considerations](https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_sdx_ki.html) and [performance trade-offs](https://docs.cloudera.com/documentation/enterprise/latest/topics/cm_sdx_vpc.html#concept_ys1_4gz_1hb). - - -1. **Create a definition containing your base cluster** - -Create a cluster definition with a base cluster containing, at least, all of the services you wish to include in the shared data context. - -In order for a cluster to serve as the base for VPCs, it must have a **data context**. - -Define a `data_contexts` block inside your base cluster definition and provide a list of services which will be shared: - -``` -clusters: - - name: Example Base Cluster - ... - data_contexts: - - name: SDX - services: [HDFS, HIVE, ATLAS, RANGER] -``` - -> **Note:** A data context called `SDX`, including all sharable services, will be automatically created if no data context is specified. - -The name of the data context can be any string, but take note because it is required in the next step. - -The services list is also optional. If this is not included, the full set of possible shareable services will be added for you, i.e Hive, HDFS, Sentry (CDH) or Atlas and Ranger (CDP Private Cloud Base). - -> **Important**: Your base cluster **must** be configured for HDFS high availability. - -2. Add a compute cluster to your definition - -A cluster block for a compute cluster (VPC) must have `type: compute`. You must also include a `base_cluster` block, which refers to your base cluster's name and the name of the data context defined there: - -``` -- name: Example Compute Cluster - type: compute -... - base_cluster: - name: Example Base Cluster - data_context: SDX -``` - -Apart from these specific requirements, defining services, repositories, configs and role layouts is done exactly the same way as it would be for base clusters. - -> **Important**: If your base cluster is configured for Kerberos, your compute cluster **must** also have Kerberos enabled. - -3. **Define your inventory** - -For a simple cluster we define inventory groups for master nodes, worker nodes and so on. When working with virtual private clusters we create extra, separate groups for the compute clusters too. - -> Important: The `[cluster:children]` group must contain the groups for **all** clusters. This will ensure all servers are initialised correctly, have Cloudera Manager agent installed, etc. - -**Example:** -``` - -[base_master_nodes] -node1.example.com host_template=Base_Master1 -node2.example.com host_template=Base_Master2 - -[base_worker_nodes] -node3.example.com -node4.example.com -node5.example.com - -[base_worker_nodes:vars] -host_template=Base_Worker - -[vpc_master_nodes] -node6.example.com host_template=VPC_Master1 -node7.example.com host_template=VPC_Master2 - -[vpc_worker_nodes] -node8.example.com -node9.example.com -node10.example.com - -[vpc_worker_nodes:vars] -host_template=VPC_Worker - -[cluster:children] -base_master_nodes -base_worker_nodes -vpc_master_nodes -vpc_worker_nodes - -``` - -For further information about the `host_template` variables above, see [How to: Connect Inventories to Cluster Definitions](TODO). - -4. **Deploy!** diff --git a/docs/inventories.md b/docs/inventories.md deleted file mode 100644 index 2245f26f..00000000 --- a/docs/inventories.md +++ /dev/null @@ -1,205 +0,0 @@ -# Inventory files - -## Required Groups - -### Cloudera Manager - -The `cloudera_manager` group must contain a single server which will become the Cloudera Manager server. (Cloudera Manager HA is not supported at this time.) - -```ini -[cloudera_manager] -host-1.example.com -``` - -### Cluster Nodes - -A group named `cluster` is **required**. This is the set of nodes which will have a Cloudera Manager agent and receive pre-requisite configurations like OS tuning, database client and JDK installation and Kerberos configs. - -Usually, `cluster` will be composed of child groups like this: - -```ini -[cluster:children] -cluster_master_nodes -cluster_worker_nodes -cluster_edge_nodes -``` - -... with the actual servers being listed like this: - -```ini -[cluster_master_nodes] -host-1.example.com host_template=Master1 - -[cluster_worker_nodes] -host-2.example.com -host-3.example.com -host-4.example.com - -[cluster_edge_nodes] -host-5.example.com -host-6.example.com -``` - -The names of the cluster sub-groups are arbitrary. These are only for convenience when assigning host template names and to make the inventory easier to read and understand. - -#### Assigning Host Templates - -The variable `host_template` must be defined against each cluster server. The host template label used must then also be defined in `cluster.yml`, which allows the defined services and roles to be mapped to the individual servers. - -```ini -[cluster_worker_nodes] -host-2.example.com host_template=Workers -host-3.example.com host_template=Workers -host-4.example.com host_template=Workers -``` - -For convenience, it is possible to assign the `host_template` variable to all members of a group using the `group_name:vars` syntax: - -```ini -[cluster_worker_nodes] -host-2.example.com -host-3.example.com -host-4.example.com - -[cluster_worker_nodes:vars] -host_template=Workers -``` - -**Important:** it is only possible to assign one host template to a host. If you set the `host_template` variable to a host in one group, then set it again on the same host but in a different inventory group, the first value will be overwritten. For example, when co-locating master and worker roles, you may try the following: - -``` -[cluster_master_nodes] -host-1.example.com host_template=Master - -[cluster_worker_nodes] -host-1.example.com host_template=Workers -``` - -This does **not** work. Only the `Workers` group will be set in this scenario. - -Instead, you must create a merged template in `cluster.yml` containing all the master and worker roles you want to assign in one go, then set this in the inventory like so: - -``` -[cluster_worker_nodes] -host-1.example.com host_template=MasterWorker -``` - -## Optional Groups - -### CDSW - -Servers which will form a Cloudera Data Science Workbench (CDSW) cluster need to be added in the following way. The aggregate group `cdsw` is **important**. This ensures that the special pre-requisite tasks for CDSW are correctly applied. - -```ini -[cdsw_master_nodes] -cdsw-1.example.com host_template=CDSW-Master - -[cdsw_worker_nodes] -cdsw-2.example.com -cdsw-3.example.com -cdsw-4.example.com - -[cdsw_worker_nodes:vars] -host_template=CDSW-Workers - -[cdsw:children] -cdsw_master_nodes -cdsw_worker_nodes - -[cluster:children] -cluster_master_nodes -cluster_worker_nodes -cdsw -``` - -### HDFS Encryption (KMS / Key Trustee Server) - -Configuring HDFS encryption requires two extra groups `kts_active` and `kms_servers` with a third, optional (but **recommended**) group `kts_passive` to enable Key Trustee Server high availability. - -The `kts_active` and `kts_passive` groups must contain a single node each. The KMS group `kms_servers` must have at least one host but can have as many as desired. - -```ini -[kms_servers] -kms-1.example.com -kms-2.example.com -kms-3.example.com - -[kts_active] -kts-1.example.com - -[kts_passive] -kts-2.example.com - -[cluster:children] -cluster_master_nodes -cluster_worker_nodes -kts_active -kts_passive -kms_servers -``` - -### Infrastructure - -The following host groups are **optional**. - -If these are omitted or left empty, the steps to provision these items will be skipped. This is for cases where such infrastructure is already provided and we do not want to create it from scratch. - -#### Database server - -```ini -[db_server] -host-10.example.com -``` - -#### Kerberos KDC - -```ini -[krb5_server] -host-10.example.com -``` - -#### Load balancer (HAProxy) - -```ini -[haproxy] -host-10.example.com -``` - -#### TLS CA Server - -```ini -[ca_server] -host-10.example.com -``` - -#### Custom parcel repository - -```ini -[custom_repo] -host-10.example.com -``` - -## Multiple Clusters - -It is possible to define multiple clusters in the inventory. The key point is that the `cluster` group must contain **all** servers which will be under Cloudera Manager's control, regardless of which cluster they belong to. - - -The inventory group names `cluster1` and `cluster2` are arbitrary. This is just a convenience to make the inventory easier to understand. - -```ini -[cluster1] -host-2.example.com host_template=Cluster1-Master -host-3.example.com host_template=Cluster1-Worker -host-4.example.com host_template=Cluster1-Worker -host-5.example.com host_template=Cluster1-Worker - -[cluster2] -host-6.example.com host_template=Cluster2-Master -host-7.example.com host_template=Cluster2-Worker -host-8.example.com host_template=Cluster2-Worker -host-9.example.com host_template=Cluster2-Worker - -[cluster:children] -cluster1 -cluster2 -``` \ No newline at end of file diff --git a/docs/playbooks/create_infrastructure.md b/docs/playbooks/create_infrastructure.md deleted file mode 100644 index 39ce8c07..00000000 --- a/docs/playbooks/create_infrastructure.md +++ /dev/null @@ -1,36 +0,0 @@ -# Playbook: Create Infrastructure - -Creates platform level infrastructure components, if required. - -## Required inventory groups - -- None - -## Optional inventory groups -- `ca_server` -- `db_server` -- `custom_repo` -- `haproxy` -- `krb5_server` - -## Plays - -### Install custom parcel repository - -Applies role [`infrastructure/custom_repo`](/docs/roles/infrastructure/custom_repo.md) to hosts in group `custom_repo` - -### Install RDBMS - -Applies role [`infrastructure/rdbms`](/docs/roles/infrastructure/rdbms.md) to hosts in group `db_server` - -### Install MIT KDC - -Applies role [`infrastructure/krb5_server`](/docs/roles/infrastructure/krb5_server.md) to hosts in group `krb5_server` - -### Install CA server - -Applies role [`infrastructure/ca_server`](/docs/roles/infrastructure/ca_server.md) to hosts in group `ca_server` - -### Install HAProxy - -Applies role [`infrastructure/haproxy`](/docs/roles/infrastructure/haproxy.md) to hosts in group `haproxy` diff --git a/docs/playbooks/install_cloudera_manager.md b/docs/playbooks/install_cloudera_manager.md deleted file mode 100644 index 18920645..00000000 --- a/docs/playbooks/install_cloudera_manager.md +++ /dev/null @@ -1,25 +0,0 @@ -# Playbook: Install Cloudera Manager - -Install Cloudera Manager Server and agents, configure its database and apply license if available. - -## Required inventory groups -- `cluster` -- `cloudera_manager` - -## Plays - -### Install Cloudera Manager server - -Applies role [`cloudera_manager/server`](/docs/roles/cloudera_manager/server.md) to hosts in group `cloudera_manager` - -### Install Cloudera Manager agents - -Applies role [`cloudera_manager/agent`](/docs/roles/cloudera_manager/agent.md) to hosts in group `cloudera_manager`, `cluster` - -### Apply Cloudera Manager license - -Runs role [`cloudera_manager/license`](/docs/roles/cloudera_manager/license.md) locally - -### Configure Cloudera Manager server - -Runs role [`cloudera_manager/config`](/docs/roles/cloudera_manager/config.md) locally diff --git a/docs/playbooks/install_cluster.md b/docs/playbooks/install_cluster.md deleted file mode 100644 index 54aaae2c..00000000 --- a/docs/playbooks/install_cluster.md +++ /dev/null @@ -1,22 +0,0 @@ -# Playbook: TODO - -TODO - Description - -## Required inventory groups - -None - -## Plays - -### Deploy Cloudera Management Service - -Runs role [`deployment/mgmt_service`](/docs/roles/deployment/mgmt_service.md) locally. - -Creates databases and users for Cloudera Management Services if required. -Generates a service template for Cloudera Management Services and posts to Cloudera Manager API. - -### Deploy clusters - -Runs role [`deployment/cluster`](/docs/roles/deployment/cluster.md) locally. - -Generates one or more cluster template and posts to Cloudera Manager API. \ No newline at end of file diff --git a/docs/playbooks/prepare_nodes.md b/docs/playbooks/prepare_nodes.md deleted file mode 100644 index 33c60e14..00000000 --- a/docs/playbooks/prepare_nodes.md +++ /dev/null @@ -1,26 +0,0 @@ -# Playbook: Prepare nodes - -Install required software packages and apply pre-requisite operating system configurations. - -## Required inventory groups -- `cloudera_manager` -- `cluster` - -## Plays - -### Apply OS pre-requisite configurations - -Applies role [`prereqs/os`](/docs/roles/prereqs/os.md) to hosts in group `cloudera_manager` and `cluster` - -### Install JDK - -Applies role [`prereqs/jdk`](/docs/roles/prereqs/jdk.md) to hosts in group `cloudera_manager` and `cluster` - -### Download MySQL Connector - -Runs role [`prereqs/mysql_connector/download`](/docs/roles/prereqs/mysql_connector/download.md) locally, only when the following condition is met: `database_type == 'mysql' or database_type == 'mariadb'` - -### Install MySQL Connector - -Applies role [`prereqs/mysql_connector/install`](/docs/roles/prereqs/mysql_connector/install.md) to hosts in group `cloudera_manager` and `cluster`, only when the following condition is met: `database_type == 'mysql' or database_type == 'mariadb'` - diff --git a/docs/playbooks/prepare_security.md b/docs/playbooks/prepare_security.md deleted file mode 100644 index 7eda1339..00000000 --- a/docs/playbooks/prepare_security.md +++ /dev/null @@ -1,27 +0,0 @@ -# Playbook: Prepare for security - -Install required software packages and apply pre-requisite configurations for Kerberos authentication. - -## Required inventory groups -- `cloudera_manager` -- `cluster` - -## Plays - -### Install pre-requisite packages for Kerberos - -Applies role [`prereqs/kerberos`](/docs/roles/prereqs/kerberos.md) to hosts in group `cloudera_manager` and `cluster` - -### Configure Cloudera Manager server for Kerberos - -Runs role [`cloudera_manager/config`](/docs/roles/cloudera_manager/config.md) locally to send the following configuration to Cloudera Manager API: -``` -KDC_HOST: "{{ krb5_kdc_host }}" -KDC_TYPE: "{{ krb5_kdc_type }}" -KRB_ENC_TYPES: "{{ krb5_enc_types }}" -SECURITY_REALM: "{{ krb5_realm }}" -``` - -### Import KDC admin creds to Cloudera Manager - -Runs role [`security/import_kdc_admin_creds`](/docs/roles/security/import_kdc_admin_creds.md) locally, only when variable `krb5_kdc_admin_user` is defined. diff --git a/docs/playbooks/teardown.md b/docs/playbooks/teardown.md deleted file mode 100644 index 059e12f4..00000000 --- a/docs/playbooks/teardown.md +++ /dev/null @@ -1,98 +0,0 @@ -# Playbook: Teardown - -This playbook tears down hosts that already have clusters installed (or partially installed), to allow to user to install a new environment. - -## Configuration options - -The teardown playbook supports the following options: - -- `teardown_everything` -- `teardown_cluster` -- `teardown_cms` -- `teardown_skip_cluster_deletion` -- `skip_cdsw_teardown` -- `teardown_preserve_parcels` -- `teardown_skip_daemon_package_deletion` - -The value you should set each of these configs to depends on what you are hoping to achieve with the teardown. - -__Note:__ `teardown_skip_cluster_deletion` is discussed at the bottom of this page. - -__Note:__ `teardown_preserve_parcels` stops the Cloudera Manager agents deleting the parcel content on cluster deletion. This should only be used in testing but can greatly reduce the time taken during the re-installation of a cluster. - -__Note:__ `teardown_skip_daemon_package_deletion` stops the teardown playbook from removing the daemon package. It can be used generally, provided the Cloudera Manager versions match. This flag reduces the time taken during the re-installation of a cluster (in the case where the Cloudera Manager server and agents are removed). - -### Full teardown - -If you want to teardown each cluster, the CMS service, agents and the Cloudera Manager server, please set: -``` -teardown_everything=true -``` - -Examples of when this might be useful include: -- Installing a completely different DEV environment on existing hosts. -- Re-installing a similar environment, but with the TLS or Kerberos configuration modified. - -### Cluster teardown - -If you want to teardown a single cluster, leaving all other clusters, the CMS service, agents and the Cloudera Manager server intact, please set: -``` -teardown_cluster='' -``` -where `` is replaced with the name of the cluster you wish to teardown. - -If you want to teardown every single cluster, leaving the CMS service, agents and the Cloudera Manager server intact, please set: -``` -teardown_cluster=all -``` -instead. - -Examples of when this might be useful include: -- Re-installing a cluster following a failure during it's first run. -- Re-installing a cluster following changes to service configurations. - -### CMS teardown - -If you want to teardown the CMS service in CM, leaving all clusters, agents and the server intact, please set: - -``` -teardown_cms=true -``` - -## Running the playbook tasks - -Once you've decided how you are going to configure the teardown, you'll want to prepare a copy of the definition for the existing environment, already installed on the hosts (you can use the definition of the new environment if this is unavailable). - -Finally, start the teardown process by running: -```shell -ansible-playbook -i path/to/hosts \ - --extra-vars=@path/to/definition/extra_vars.yml \ - --extra-vars="" \ - /path/to/playbook/teardown.yml -``` -where `` is replaced by the teardown configs you are using (space separated). - -_For example_, if you wanted to teardown a cluster named `Cluster1`, you may run something like: -```shell -ansible-playbook -i hosts \ - --extra-vars=@example_definitions/simple/extra_vars.yml \ - --extra-vars="teardown_cluster='Cluster1'" \ - cloudara-playbook-v2/teardown.yml -``` - -Once running, the playbook will: -- Attempt to stop and delete each cluster (that is included in the teardown) from the Cloudera Manager server. -- Scan the environment's definition for databases that are included in the teardown and either: - * Remove the database if the database host is controlled by the playbook - * (__important__) Warn the user that the database must be deleted manually. -- Scan the environment's definition for key directories that are included in the teardown and are known to cause problems during re-installation and remove them from the hosts. -- Stop the agent and Cloudera Manager server services and remove the packages from the hosts, in the case where `teardown_everything=true`. - -After the playbook is finished, you should be able to continue with your planned re-installation. - -## Important notes and limitations - -- The teardown playbook is (understandably) quite __destructive__. Please carefully consider the impact before executing the playbook. -- The teardown playbook does not clean hosts. It will remove databases and directories that would otherwise cause the next installation to fail but there will still be a number of files left from the original environment. If this is required, then you should clean the disks and re-install the OS. -- In the case that the Cloudera Manager server and/or agents are not healthy, the playbook may be unable to stop and delete the clusters. Even with `teardown_everything=true`, this error is fatal as stopping the cluster helps clean-up the running processes. If you still wish to continue with the teardown, you can set `teardown_skip_cluster_deletion=true` but please reboot the hosts after the teardown to clean-up the running processes. -- If you are tearing down an environment in order to change the major version of the Cloudera Manager server, please reboot the hosts before containing with the re-installation. diff --git a/docs/playbooks/verify_everything.md b/docs/playbooks/verify_everything.md deleted file mode 100644 index d8693989..00000000 --- a/docs/playbooks/verify_everything.md +++ /dev/null @@ -1,21 +0,0 @@ -# Playbook Verify Definition - -Checks a number of assertion against the inventory, cluster definition and the parcels used. This playbook can be run in complete isolation of the hosts, provided the parcels can still be accessed. - -The goal of this playbook is to reduce the number of runtime failures. - -## Plays - -Before running any of the verify roles, `verify_everything.yml` groups the hosts by their host template using the `group_by` module. This step does not required connectivity to the hosts themselves. - -### Checks Inventory - -Runs role [`verify/inventory`](/docs/roles/verify/inventory.md) locally - -### Checks Cluster Definition - -Runs role [`verify/definition`](/docs/roles/verify/definition.md) locally - -### Checks Services, Roles and Parcels - -Runs role [`verify/parcels_and_roles`](/docs/roles/verify/parcels_and_roles.md) locally diff --git a/docs/roles/cloudera_manager/repo.md b/docs/roles/cloudera_manager/repo.md deleted file mode 100644 index 5f7393fb..00000000 --- a/docs/roles/cloudera_manager/repo.md +++ /dev/null @@ -1,62 +0,0 @@ - -# Cloudera Manager Repository - -## Install the default Cloudera Manager version - -``` -(no extra vars) -``` - -With no variables overridden, the default behaviour is to install the latest or recommended version of Cloudera Manager (currently `7.0.3`). The version number is specified as `cloudera_manager_version` in the role default variables (`roles/cloudera_manager/repo/defaults/main.yml`). - -## Install a custom Cloudera Manager version (public archive, not paywalled) - -``` -cloudera_manager_version: 6.3.1 -``` - -The version specified in `cloudera_manager_version` is used to construct repository URLs from the public `archive.cloudera.com` repository. - -## Install a custom Cloudera Manager version (public archive, paywalled) - -``` -cloudera_manager_version: 6.3.3 -cloudera_manager_repo_username: myusername -cloudera_manager_repo_password: mypassword -``` - -The version specified in `cloudera_manager_version` is used to construct repository URLs from the public `archive.cloudera.com` repository. Because `cloudera_manager_repo_username` is defined, the role knows to add credentials to the repository details when installing them. - -To avoid entering plaintext credentials, use Ansible vault encrypted values, e.g. - -``` -cloudera_manager_version: 6.3.3 -cloudera_manager_repo_username: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 6534653166 ... snip ... -cloudera_manager_repo_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 3338663739 ... snip ... -``` - -## Install a custom Cloudera Manager version (custom archive) - -### Method 1: - -If the custom archive follows the same directory structure as Cloudera's public archive, it is possible to set a base URL -and a version. This method is OS agnostic. For example, if deploying on RHEL or CentOS 7, the playbook will automatically append -the `redhat7/yum` portion of the final repo location. - -``` -cloudera_archive_base_url: http://cloudera-build-us-west-1.vpc.cloudera.com/s3/build/2535749 -cloudera_manager_version: 7.1.1 -``` - -### Method 2: - -If the custom archive does not follow the same directory structure as Cloudera's public archive, we can totally override the -repository location. Note: `cloudera_manager_repo_url` takes precedence over `cloudera_manager_version`. Any specified version number is ignored when a custom repository URL is set in this manner. - -``` -cloudera_manager_repo_url: http://cloudera-build-us-west-1.vpc.cloudera.com/s3/build/2400091/cm7/7.1.1/redhat7/yum/ -``` diff --git a/docs/roles/verify/definition.md b/docs/roles/verify/definition.md deleted file mode 100644 index 2df71bf8..00000000 --- a/docs/roles/verify/definition.md +++ /dev/null @@ -1,13 +0,0 @@ -# Definition Verification - -This role asserts expectations on the cluster definition. - -Here we focus on the clusters in aggregation. - -Examples include: -- Ensure that TLS is configured in the inventory when specified in a cluster. -- Each host template in the definition is matched to hosts in the inventory and vice versa. -- All KTS/KMS configurations are set as expected. -- Kerberos is enabled when Ranger or Sentry is present in the cluster. - -This will catch high-level errors when creating cluster definitions. diff --git a/docs/roles/verify/inventory.md b/docs/roles/verify/inventory.md deleted file mode 100644 index 470ca299..00000000 --- a/docs/roles/verify/inventory.md +++ /dev/null @@ -1,8 +0,0 @@ -# Inventory Verification - -This role asserts basic expectations on the format of the inventory in isolation of the cluster definitions. - -Examples include: -- Ensure that each group has at least 1 host - -This will allow us to make basic assumptions on the inventory's format. diff --git a/docs/roles/verify/parcels_and_roles.md b/docs/roles/verify/parcels_and_roles.md deleted file mode 100644 index c6b26aef..00000000 --- a/docs/roles/verify/parcels_and_roles.md +++ /dev/null @@ -1,14 +0,0 @@ -# Parcels and Roles Verification - -The role ensures the service and roles configured in each cluster pass a number of basic assertions. - -For each cluster, this role downloads the manifest of each repository and, combining this with a service-role mapping, verifies that the services and roles configured in each cluster matches the parcels included. - -Here we focus on individual clusters. - -Examples include: -- Ensure that all services configured match the parcels services. -- Ensure that all roles configured have the correct parent service -- Ensure that all roles in `configs` are included in the templates. - -This will catch many systematic errors when creating cluster definitions. diff --git a/docs/security.md b/docs/security.md deleted file mode 100644 index ea066c39..00000000 --- a/docs/security.md +++ /dev/null @@ -1,106 +0,0 @@ -# Creating Secure Clusters - -## Enabling Kerberos - -To enable Kerberos for a cluster, set `kerberos: true` inside the `security` block of its definition. -Kerberos **must** be enabled if you want to include certain services in the cluster, e.g. Ranger or Sentry. - -```yaml -clusters: - - name: CDP PvC Base Cluster - services: [HDFS, RANGER, SOLR, ZOOKEEPER] - security: - kerberos: true -``` - -If a KDC will be provisioned by the playbook and you are happy with default values for realm names, encryption types etc, no other configuration is required. - -## Enabling TLS - -To enable TLS for a cluster, the following configurations are required: - -1) Set `tls: true` inside the `security` block of its definition. - -```yaml -clusters: - - name: CDP PvC Base Cluster - services: [HDFS, RANGER, SOLR, ZOOKEEPER] - security: - tls: true -``` - -2) Set `tls: True` on all inventory host groups for which TLS keystores and truststores need to be generated. - -```ini -[cluster:vars] -tls=True -``` - -3a) If you wish to create a standalone CA server and sign certificates against this for a fully automated (but less secure) cluster, add a `[ca_server]` block to the inventory. - -```ini -[ca_server] -ca-server-1.example.com -``` - -3b) If you wish to manually sign certificates against an -external CA, like Active Directory, add the path where signed certificates will be stored and root CA certificate details in `extra_vars.yml` - -```yaml -tls_ca_certs: - - alias: ScaleAD - path: /path/to/root/certs/scale-ad.pem - -tls_signed_certs_dir: /path/to/signed/certs -``` - -The directory `tls_signed_certs_dir` should contain PEM format certificates, one per server, named with the FQDN of the host, e.g `host-1.fully.qualified.example.com.pem` - -## Enabling HDFS Encryption - -To enable HDFS Encryption for a cluster, the following steps must be followed: - -1. Set `hdfs_encryption: true` inside the `security` block of the main cluster definition. -2. Add a second cluster definition for Key Trustee Server, marked `type: kts` - -Example: - -```yaml -clusters: - - - name: CDP PvC Base Cluster - services: [HDFS, RANGER, SOLR, ZOOKEEPER] - repositories: - - 'http://archive/c7/parcel/repo' - security: - hdfs_encryption: true - - - name: KTS Cluster - type: kts - repositories: - - 'http://archive/keytrustee-server/parcel/repo' - - 'http://archive/c7/parcel/repo' -``` - -**It is not necessary to include KMS services in the cluster service list or host templates.** These will be added for you automatically. - -Note: at the moment it is necessary to add a CDH/CDP Private Cloud Base parcel repository to the Key Trustee Server cluster as well as the base cluster, even though no CDH or CDP Private Cloud Base services will run there. - -Add the following new blocks to your **inventory** to specify which hosts should be used for Key Trustee and KMS servers. - -```ini -[kts_active] -kts-1.example.com - -[kts_passive] -kts-2.example.com - -[kms_servers] -kms-1.example.com -kms-2.example.com -kms-3.example.com -``` - -If you do not want Key Trustee Server to be highly available, you can omit the `kts_passive` group (this is **not** recommended). - -The KMS group `kms_servers` must have at least one host but can have as many as desired. Unlike KTS, KMS nodes are all active and load balanced. \ No newline at end of file diff --git a/docs/self-contained-playbooks.md b/docs/self-contained-playbooks.md deleted file mode 100644 index deefd270..00000000 --- a/docs/self-contained-playbooks.md +++ /dev/null @@ -1,131 +0,0 @@ -# Self-Contained Playbooks - -This doc provides brief set of instructions on how to create and utilise a self-contained "playbook" distro. The main idea is to create a virtual environment with cloudera-playbook itself, Ansible and its dependencies that can be packaged and used later for an on-site installation. - -## Why we might need a self-contained playbook: - -- networking restrictions (e.g. VPN + jump host that has to be used for the installation) that make it impossible to run playbook from our laptop or other controlled environment -- customer requirement to work from their laptop/VDI/workstation -- customer requirement to have all bits and pieces used for the install -- sticking to specific versions of ansible, python, etc. for reproducible installs - -## What is being packaged: - -- the playbook itself -- ansible -- ansible dependencies -- ansible galaxy roles playbook depends on -- python (might not be strictly required) - -It's questionable whether python has to be part of the distribution since all operating systems supported by Cloudera platform have python available. - -## How to prepare a package (with Python): - -Prepare the package on the OS version that's going to be used during the playbook run. E.g. if you are going to install the cluster from a RHEL7 jump host, spinup a RHEL7 VM and create the package on that VM. - -``` -# Create a working directory (you'll have to use the same absolute path when using the package) -mkdir -p /opt/cloudera/playbook-env - -# install python from source -yum install yum-utils make wget -yum-builddep python -wget https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tgz -tar -xvf Python-3.8.2.tgz -cd Python-3.8.2 -./configure --prefix=/opt/cloudera/playbook-env/python && make && make install - -# install virtualenv -cd /opt/cloudera/playbook-env -python/bin/pip3 install virtualenv - -# create virtual environment for ansible -python/bin/virtualenv ansible-venv - -# install ansible -cd ansible-venv/ -source bin/activate -pip install ansible==2.9.7 -pip install jmespath - -# clone playbook -git clone https://github.com/cloudera-labs/cloudera.cluster.git - -# install roles from ansible galaxy required by the playbook to ansible-venv directory -ansible-galaxy install --roles-path ./galaxy_roles -r cloudera.cluster/requirements.yml - -# test everything works as expected -# cd cloudera.cluster/ -# ANSIBLE_ROLES_PATH=../galaxy_roles ansible-playbook -i examples/sample/hosts --extra-vars @examples/sample/extra_vars.yml create_infrastructure.yml - -# clean things up -cd /opt/cloudera/ -find ./playbook-env/ -name "*.pyc" -delete -rm -rf playbook-env/ansible-venv/cloudera.cluster/.git - -# create an archive -tar -zcvf /tmp/playbook-env-with-python.tbz /opt/cloudera/playbook-env -``` - -## How to prepare a package (without Python): - -``` -# Create a working directory (you'll have to use the same absolute path when using the package) -mkdir -p /opt/cloudera/playbook-env - -# install virtualenv -cd /opt/cloudera/playbook-env -python/bin/pip3 install virtualenv - -# create virtual environment for ansible -python/bin/virtualenv ansible-venv - -# install ansible -cd ansible-venv/ -source bin/activate -pip install ansible==2.9.7 -pip install jmespath - -# clone playbook -git clone https://github.com/cloudera-labs/cloudera.cluster.git - -# install roles from ansible galaxy required by the playbook to ansible-venv directory -ansible-galaxy install --roles-path ./galaxy_roles -r cloudera.cluster/requirements.yml - -# test everything works as expected -# cd cloudera.cluster/ -# ANSIBLE_ROLES_PATH=../galaxy_roles ansible-playbook -i examples/sample/hosts --extra-vars @examples/sample/extra_vars.yml create_infrastructure.yml - -# clean things up -cd /opt/cloudera/ -find ./playbook-env/ -name "*.pyc" -delete -rm -rf playbook-env/ansible-venv/cloudera.cluster/.git - -# create an archive -tar -zcvf /tmp/playbook-env-with-python.tbz /opt/cloudera/playbook-env -``` - -## How to deploy the package: - -``` -# Extract the archive tar -C/ -xvf playbook-env.tgz -cd /opt/cloudera/playbook-env/ansible-venv/ - -# Activate the virtual environment -source bin/activate - -# Edit extra_vars.yml, cluster.yml and hosts files if needed and run the playbook -ANSIBLE_ROLES_PATH=../galaxy_roles ansible-playbook -i examples/7.1.x/single-node/hosts --extra-vars @examples/7.1.x/single-node/extra_vars.yml site.yml - -``` - -## What's left out of scope: - -Given this setup is mostly needed in isolated environments where there's no internet access from cluster node, it's likely that the installation will be performed from a local repository. This is covered pretty well in Cloudera's documentation (e.g. [Configuring a Local Parcel Repository](https://docs.cloudera.com/documentation/enterprise/upgrade/topics/cm_ig_create_local_parcel_repo.html) for CDH5 and CDH6). - -There are several OS level dependencies when performing a cluster installation (e.g. Java, Kerberos libraries, rngd). In case of an isolated environment, the customer should have a local repo of some sort where those packages are available (list of requirements is covered in deployment and security prerequisites documents). - -In case MySQL database is used as backend, MySQL JDBC driver should be made available in a local repository (mysql_connector_url playbook variable should be modified to point to a local repo url with a zip archive with the driver). - - - diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index ba95289e..00000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1,94 +0,0 @@ -# Troubleshooting - -## Common issue #1 - -``` -fatal: [host.example.com]: UNREACHABLE! => {“changed”: false, “msg”: “Failed to connect to the host via ssh: Permission denied (publickey,gssapi-keyex,gssapi-with-mic,password).“, “unreachable”: true} -``` - -or - -``` -fatal: [host.example.com]: UNREACHABLE! => {"changed": false, "msg": "SSH password was requested, but none specified", "unreachable": true} -``` - -**Problem**: - - - Passwordless SSH connectivity is not correctly set up between your Ansible server and the remote servers in the inventory. - -**Solutions**: - - - _(recommended)_ Create an SSH key pair for the Ansible server and ensure the public key is placed in all the servers' `authorized_keys` file for the connecting user, e.g. `/root/.ssh/authorized_keys` or `/home//.ssh/authorized_keys` - - - _(alternative)_ Provide a password for SSH connectivity during the playbook run: - - ```bash - $ ansible-playbook --ask-pass ...other args - ``` - -## Common issue #2 - -``` -fatal: [host.example.com]: UNREACHABLE! => {"changed": false, "msg": "Host key checking is enabled, and SSH reported an unrecognized or mismatching host key.", "unreachable": true} -``` - -**Problem**: - - - SSH host key checking is enabled (Ansible default) but the servers you are trying to connect to are not present in `known_hosts` (or their key has changed) - -**Solutions**: - -- _(easiest, but least secure)_ Disable Ansible host key checking. In your Ansible configuration file, enter the following: - - ```ini - [defaults] - host_key_checking = False - ``` - -- _(annoying, but more secure)_ Connect to each of your servers manually from the Ansible host once, so that the `known_hosts` file becomes populated. - -## Common issue #3 - -``` -{"msg": "The task includes an option with an undefined variable. The error was: 'dict object' has no attribute u’some.hostname.com'\n\nThe error appears to be in '/home/ansible/cloudera-playbook/roles/deployment/services/mgmt/tasks/main.yml': line 28, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n- name: Define target host ID for Cloudera Management Service installation\n ^ here\n"} -``` - -**Problem**: - -The cause is likely to be one of the following: - -1) Inconsistent hostname resolution. The Cloudera Manager is reporting itself with a different hostname to that contained in your inventory file. - -2) Cloudera Manager agent(s) are not heartbeating correctly. - -**Solution**: - -By this stage of the playbook execution, Cloudera Manager server will be running. Log into Cloudera Manager and view the **Hosts** page: - -- If hosts appear, check the list to ensure that the hostnames shown match your inventory file. If they do not match, either update your inventory file with these hostnames or update the cluster DNS so that the same names can be resolved consistently everywhere. - -- If no hosts appear, log into the server indicated in the error message, and: - - - Check that the `cloudera-manager-agent` service is running. - - - Check the Cloudera Manager agent log file `/var/log/cloudera-scm-agent/cloudera-scm-agent.log`. Any error message there should give a clue as to why communication with the Cloudera Manager server is failing. - -## Common issue #4 - -``` -ERROR! couldn't resolve module/action 'cm_api'. This often indicates a misspelling, missing collection, or incorrect module path. -``` - -**Problem**: - -- The `cm_api` action is not available in a custom role. - -**Solution**: - -- Add the following into the role's `meta/main.yml` file. - - ```yaml - --- - dependencies: - - role: cloudera_manager/api_client - ``` \ No newline at end of file diff --git a/examples/basic-7.1.x/cluster.yml b/examples/basic-7.1.x/cluster.yml deleted file mode 100644 index daea99b7..00000000 --- a/examples/basic-7.1.x/cluster.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -clusters: - - name: Basic Cluster - services: [HDFS, YARN, ZOOKEEPER] - repositories: - - https://archive.cloudera.com/p/cdh7/7.1.6.0/parcels/ - configs: - HDFS: - DATANODE: - dfs_data_dir_list: /dfs/dn - dfs_datanode_failed_volumes_tolerated: 0 - NAMENODE: - dfs_name_dir_list: /dfs/nn - SECONDARYNAMENODE: - fs_checkpoint_dir_list: /dfs/snn - YARN: - RESOURCEMANAGER: - yarn_scheduler_maximum_allocation_mb: 4096 - yarn_scheduler_maximum_allocation_vcores: 4 - NODEMANAGER: - yarn_nodemanager_resource_memory_mb: 4096 - yarn_nodemanager_resource_cpu_vcores: 4 - yarn_nodemanager_local_dirs: /tmp/nm - yarn_nodemanager_log_dirs: /var/log/nm - GATEWAY: - mapred_submit_replication: 3 - mapred_reduce_tasks: 6 - ZOOKEEPER: - SERVICEWIDE: - zookeeper_datadir_autocreate: true - host_templates: - Master1: - HDFS: [NAMENODE, SECONDARYNAMENODE] - YARN: [RESOURCEMANAGER, JOBHISTORY] - ZOOKEEPER: [SERVER] - Workers: - HDFS: [DATANODE] - YARN: [NODEMANAGER] - -mgmt: - name: Cloudera Management Service - services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - -hosts: - configs: - host_default_proc_memswap_thresholds: - warning: never - critical: never - host_memswap_thresholds: - warning: never - critical: never - host_config_suppression_agent_system_user_group_validator: true diff --git a/examples/basic-7.1.x/extra_vars.yml b/examples/basic-7.1.x/extra_vars.yml deleted file mode 100644 index 5235e118..00000000 --- a/examples/basic-7.1.x/extra_vars.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -cluster_definition: examples/basic-7.1.x -cloudera_manager_version: 7.3.1 -cloudera_manager_repo_username: "{{ vault__cloudera_manager_repo_username }}" -cloudera_manager_repo_password: "{{ vault__cloudera_manager_repo_password }}" -cloudera_manager_license_type: enterprise -cloudera_manager_license_file: testing/license.txt -cloudera_manager_options: - CUSTOM_BANNER_HTML: "Cloudera Ansible Playbook v2 - {{ cluster_definition }}" diff --git a/examples/basic-7.1.x/hosts b/examples/basic-7.1.x/hosts deleted file mode 100644 index 7dc029cd..00000000 --- a/examples/basic-7.1.x/hosts +++ /dev/null @@ -1,20 +0,0 @@ -[cloudera_manager] -host-1.example.com - -[cluster_master_nodes] -host-1.example.com host_template=Master1 - -[cluster_worker_nodes] -host-2.example.com -host-3.example.com -host-4.example.com - -[cluster_worker_nodes:vars] -host_template=Workers - -[cluster:children] -cluster_master_nodes -cluster_worker_nodes - -[db_server] -host-1.example.com diff --git a/examples/sample/cluster.yml b/examples/sample/cluster.yml deleted file mode 100644 index d0da0997..00000000 --- a/examples/sample/cluster.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -clusters: - - name: Data Engineering Cluster - services: [ATLAS, DAS, HBASE, HDFS, HIVE, HIVE_ON_TEZ, HUE, IMPALA, INFRA_SOLR, KAFKA, OOZIE, RANGER, QUEUEMANAGER, SOLR, SPARK_ON_YARN, TEZ, YARN, ZOOKEEPER] - repositories: - - https://archive.cloudera.com/p/cdh7/7.1.6.0/parcels/ - security: - kerberos: true - configs: - ATLAS: - ATLAS_SERVER: - atlas_authentication_method_file: true - atlas_admin_password: password123 - atlas_admin_username: admin - HDFS: - DATANODE: - dfs_data_dir_list: /dfs/dn - dfs_datanode_failed_volumes_tolerated: 0 - NAMENODE: - dfs_name_dir_list: /dfs/nn - SECONDARYNAMENODE: - fs_checkpoint_dir_list: /dfs/snn - IMPALA: - IMPALAD: - enable_audit_event_log: true - scratch_dirs: /tmp/impala - YARN: - RESOURCEMANAGER: - yarn_scheduler_maximum_allocation_mb: 4096 - yarn_scheduler_maximum_allocation_vcores: 4 - NODEMANAGER: - yarn_nodemanager_resource_memory_mb: 4096 - yarn_nodemanager_resource_cpu_vcores: 4 - yarn_nodemanager_local_dirs: /tmp/nm - yarn_nodemanager_log_dirs: /var/log/nm - GATEWAY: - mapred_submit_replication: 3 - mapred_reduce_tasks: 6 - ZOOKEEPER: - SERVICEWIDE: - zookeeper_datadir_autocreate: true - host_templates: - Master1: - ATLAS: [ATLAS_SERVER] - DAS: [DAS_EVENT_PROCESSOR, DAS_WEBAPP] - HBASE: [MASTER, HBASERESTSERVER, HBASETHRIFTSERVER] - HDFS: [NAMENODE, SECONDARYNAMENODE, HTTPFS] - HIVE: [HIVEMETASTORE, GATEWAY] - HIVE_ON_TEZ: [HIVESERVER2] - HUE: [HUE_SERVER, HUE_LOAD_BALANCER] - IMPALA: [STATESTORE, CATALOGSERVER] - INFRA_SOLR: [SOLR_SERVER] - OOZIE: [OOZIE_SERVER] - QUEUEMANAGER: [QUEUEMANAGER_STORE, QUEUEMANAGER_WEBAPP] - RANGER: [RANGER_ADMIN, RANGER_TAGSYNC, RANGER_USERSYNC] - SPARK_ON_YARN: [SPARK_YARN_HISTORY_SERVER] - TEZ: [GATEWAY] - YARN: [RESOURCEMANAGER, JOBHISTORY] - ZOOKEEPER: [SERVER] - Workers: - HBASE: [REGIONSERVER] - HDFS: [DATANODE] - HIVE: [GATEWAY] - HIVE_ON_TEZ: [GATEWAY] - IMPALA: [IMPALAD] - KAFKA: [KAFKA_BROKER] - SOLR: [SOLR_SERVER] - SPARK_ON_YARN: [GATEWAY] - TEZ: [GATEWAY] - YARN: [NODEMANAGER] - -mgmt: - name: Cloudera Management Service - services: [ALERTPUBLISHER, EVENTSERVER, HOSTMONITOR, REPORTSMANAGER, SERVICEMONITOR] - -hosts: - configs: - host_default_proc_memswap_thresholds: - warning: never - critical: never - host_memswap_thresholds: - warning: never - critical: never - host_config_suppression_agent_system_user_group_validator: true diff --git a/examples/sample/extra_vars.yml b/examples/sample/extra_vars.yml deleted file mode 100644 index 76298808..00000000 --- a/examples/sample/extra_vars.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -cluster_definition: definitions/sample -cloudera_manager_version: 7.3.1 -cloudera_manager_repo_username: "{{ vault__cloudera_manager_repo_username }}" -cloudera_manager_repo_password: "{{ vault__cloudera_manager_repo_password }}" -cloudera_manager_license_type: enterprise -cloudera_manager_license_file: testing/license.txt -cloudera_manager_options: - CUSTOM_BANNER_HTML: "Cloudera Ansible Playbook v2 - {{ cluster_definition }}" diff --git a/examples/sample/hosts b/examples/sample/hosts deleted file mode 100644 index 911ca718..00000000 --- a/examples/sample/hosts +++ /dev/null @@ -1,23 +0,0 @@ -[cloudera_manager] -host-1.example.com - -[cluster_master_nodes] -host-2.example.com host_template=Master1 - -[cluster_worker_nodes] -host-3.example.com -host-4.example.com -host-5.example.com - -[cluster_worker_nodes:vars] -host_template=Workers - -[cluster:children] -cluster_master_nodes -cluster_worker_nodes - -[krb5_server] -host-6.example.com - -[db_server] -host-6.example.com \ No newline at end of file diff --git a/filter_plugins/filters.py b/filter_plugins/filters.py deleted file mode 100644 index 740d20c2..00000000 --- a/filter_plugins/filters.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class FilterModule(object): - - def filters(self): - return { - 'flatten_dict_list': self.flatten_dict_list - } - - def flatten_dict_list(self, item, level=2, sep='_', show_index=False): - ''' flatten a structure of dicts and lists into a flat array - - e.g. { "a": [1, 2, 3], "b": { "c": "d", "e": "f" } } - with level=2 - becomes ["a_1", "a_2", "a_3", "b_c", "b_d"] - ''' - - state = [] - - def _flatten_dict_list(i, l, parents): - if l > 0: - if isinstance(i, dict): - for key, value in i.items(): - _flatten_dict_list(value, l-1, parents + [str(key)]) - - elif isinstance(i, list): - for index, value in enumerate(i): - if show_index: - _flatten_dict_list(value, l, parents + [str(index)]) - else: - _flatten_dict_list(value, l, parents) - - else: - state.append(sep.join(parents + [str(i)])) - - if l == 0 and len(parents) > 0: - state.append(sep.join(parents)) - - _flatten_dict_list(item, level, []) - - return state diff --git a/galaxy.yml b/galaxy.yml new file mode 100644 index 00000000..fd1123de --- /dev/null +++ b/galaxy.yml @@ -0,0 +1,70 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### REQUIRED + +# The namespace of the collection. This can be a company/brand/organization or product namespace under which all +# content lives. May only contain alphanumeric characters and underscores. Additionally namespaces cannot start with +# underscores or numbers and cannot contain consecutive underscores +namespace: cloudera + +# The name of the collection. Has the same character restrictions as 'namespace' +name: cluster + +# The version of the collection. Must be compatible with semantic versioning +version: 3.0.3 + +# The path to the Markdown (.md) readme file. This path is relative to the root of the collection +readme: README.md + +# A list of the collection's content authors. Can be just the name or in the format 'Full Name (url) +# @nicks:irc/im.site#channel' +authors: [] + + +### OPTIONAL but strongly recommended + +# A short summary description of the collection +description: Cloudera assets for managing Cloudera Clusters + +# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only +# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file' +license: +- GPL-2.0-or-later + +# The path to the license file for the collection. This path is relative to the root of the collection. This key is +# mutually exclusive with 'license' +license_file: '' + +# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character +# requirements as 'namespace' and 'name' +tags: [] + +# Collections that this collection requires to be installed for it to be usable. The key of the dict is the +# collection label 'namespace.name'. The value is a version range +# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version +# range specifiers can be set and are separated by ',' +dependencies: {} + +# The URL of the originating SCM repository +repository: http://github.com/cloudera-labs + +# The URL to any online docs +documentation: http://docs.example.com + +# The URL to the homepage of the collection/project +homepage: http://example.com + +# The URL to the collection issue tracker +issues: http://example.com/issue/tracker diff --git a/group_vars/all b/group_vars/all deleted file mode 100644 index bddc247f..00000000 --- a/group_vars/all +++ /dev/null @@ -1,53 +0,0 @@ ---- -database_host: "{{ groups['db_server'][0] | default('localhost') }}" -database_type: postgresql -database_version: 10 - -# You can skip users and groups creation if you ensure they will exist -# This may be useful if the users and groups are managed externally -# The playbook will check the existing users and groups and fail if they do not exist -skip_user_group_init: false - -krb5_realm: CLOUDERA.LOCAL -krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" -krb5_kdc_admin_password: changeme -krb5_kdc_type: MIT KDC -krb5_enc_types: "aes256-cts aes128-cts" - -local_temp_dir: /tmp - -base_dir_security: /opt/cloudera/security -base_dir_security_pki: "{{ base_dir_security }}/pki" - -# You can disable the playbooks certificate functionality with this flag -# if you are handling that separately -# Setting this to true will also enable the TLS configs for CM -# Clusters TLS configs can be configured by the playbook by setting -# security.tls to true in the cluster definition -manual_tls_cert_distribution: false - -tls_cert_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.pem" -tls_cert_path_generic: "{{ base_dir_security_pki }}/host.pem" -tls_chain_path: "{{ base_dir_security_pki }}/chain.pem" -tls_csr_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.csr" -tls_csr_config_path: "{{ base_dir_security_pki }}/csr.cnf" -tls_key_password: changeme -tls_key_password_file: "{{ base_dir_security_pki }}/host.key.pw" -tls_key_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.key" -tls_key_path_generic: "{{ base_dir_security_pki }}/host.key" -tls_key_path_plaintext: "{{ tls_key_path }}.unenc" -tls_key_path_plaintext_generic: "{{ tls_key_path_generic }}.unenc" -tls_keystore_password: changeme -tls_keystore_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.jks" -tls_keystore_path_generic: "{{ base_dir_security_pki }}/host.jks" -tls_truststore_password: changeme -tls_truststore_path: "{{ base_dir_security_pki }}/truststore.jks" -tls_uber_truststore_path: "{{ base_dir_security_pki }}/full_truststore.jks" - -maven_repo: "https://repo1.maven.org/maven2" - -ca_server_attrs_general: - OU: PS - O: Cloudera, Inc. - ST: CA - C: US diff --git a/install_cloudera_manager.yml b/install_cloudera_manager.yml deleted file mode 100644 index ba532038..00000000 --- a/install_cloudera_manager.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Install Cloudera Manager daemons - hosts: cloudera_manager, cluster - become: yes - any_errors_fatal: true - roles: - - cloudera_manager/daemons - -- name: Install Cloudera Manager agents - hosts: cloudera_manager, cluster - become: yes - any_errors_fatal: true - roles: - - cloudera_manager/agent - -- name: Install Cloudera Manager server - hosts: cloudera_manager - become: yes - roles: - - cloudera_manager/server - -- name: Apply Cloudera Manager license - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera_manager/license - -- name: Configure Cloudera Manager server for TLS - hosts: cloudera_manager - become: yes - gather_facts: no - roles: - - role: cloudera_manager/server_tls - when: tls | default(False) or manual_tls_cert_distribution | default(False) - -- name: Configure Cloudera Manager agents - hosts: cloudera_manager, cluster - become: yes - any_errors_fatal: true - roles: - - cloudera_manager/agent_config - -- name: Configure Cloudera Manager server - hosts: localhost - connection: local - gather_facts: no - roles: - - cloudera_manager/config - vars: - api_config_keys_uppercase: True - api_configs: "{{ cloudera_manager_options | default({}) }}" - -- name: Configure Cloudera Manager external auth - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera_manager/external_auth - -- name: Configure Cloudera Manager external accounts - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera_manager/external_account - diff --git a/install_cluster.yml b/install_cluster.yml deleted file mode 100644 index 6102145b..00000000 --- a/install_cluster.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Ensure that the agents are heartbeating - hosts: cloudera_manager:cluster - gather_facts: yes - any_errors_fatal: yes - roles: - - { role: cloudera_manager/wait_for_heartbeat, - when: cloudera_manager_agent_wait_for_heartbeat | default(True) } - tags: - - heartbeat - -- name: Group hosts using host template information - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts.yml - tags: - - always - -- name: Deploy Cloudera Management Service - hosts: localhost - connection: local - gather_facts: no - roles: - - deployment/services/mgmt - tags: - - mgmt - -- name: Deploy clusters - hosts: localhost - connection: local - gather_facts: no - roles: - - deployment/cluster - tags: - - cluster diff --git a/nifi_workaround.yml b/nifi_workaround.yml deleted file mode 100644 index 812cc677..00000000 --- a/nifi_workaround.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Setup symlinks for NiFi TLS keystore and truststore - hosts: cluster - become: yes - gather_facts: no - tasks: - - name: Perform the NiFi workaround for explicit TLS - block: - - - name: Ensure the NiFi home directory exists - file: - path: /var/lib/nifi - owner: nifi - group: nifi - state: directory - - - name: Ensure the link for the NiFi keystore exists - file: - src: "{{ tls_keystore_path }}" - dest: /var/lib/nifi/cm-auto-host_keystore.jks - state: link - - - name: Ensure the link for the NiFi truststore exists - file: - src: "{{ tls_truststore_path }}" - dest: /var/lib/nifi/cm-auto-in_cluster_truststore.jks - state: link - - - name: Ensure the NiFi Registry home directory exists - file: - path: /var/lib/nifiregistry - owner: nifiregistry - group: nifiregistry - state: directory - - - name: Ensure the link for the NiFi Registry keystore exists - file: - src: "{{ tls_keystore_path }}" - dest: /var/lib/nifiregistry/cm-auto-host_keystore.jks - state: link - - - name: Ensure the link for the NiFi Registry truststore exists - file: - src: "{{ tls_truststore_path }}" - dest: /var/lib/nifiregistry/cm-auto-in_cluster_truststore.jks - state: link - - when: > - (tls | default(False) - or manual_tls_cert_distribution | default(False)) - and not (autotls | default(False)) diff --git a/plugins/action/cm_api.py b/plugins/action/cm_api.py new file mode 100644 index 00000000..f7f3ffc2 --- /dev/null +++ b/plugins/action/cm_api.py @@ -0,0 +1,117 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display + +import time + +display = Display() + +class ActionModule(ActionBase): + + def build_url(self, api_base, api_endpoint): + if not api_endpoint.startswith("/"): + api_endpoint = "/" + api_endpoint + return api_base + api_endpoint + + def build_args(self, task_vars, additional_args=dict()): + args = dict( + body_format = "json", + force_basic_auth=True, + url_username=task_vars['cloudera_manager_api_user'], + url_password=task_vars['cloudera_manager_api_password'], + return_content=True, + validate_certs=task_vars['cloudera_manager_tls_validate_certs'] + ) + args.update(additional_args) + return args + + def get_api_base_url(self, task_vars): + # If there's already a value in task vars, just use that + if 'cloudera_manager_api' in task_vars: + api_base = task_vars['cloudera_manager_api'] + result = None + else: + # Call /api/version endpoint to find the correct API version number. + url = self.build_url(task_vars['cloudera_manager_url'], '/api/version') + args = self.build_args(task_vars, dict(url=url)) + result = self._execute_module('uri', module_args=args, task_vars=task_vars, wrap_async=self._task.async_val) + # We use the URL returned in the result rather than the one we defined originally. + # This has the benefit of allowing to use TLS-enabled endpoints later if the call was redirected. + api_base = result["url"].replace("version", result['content']) if result['status'] == 200 else None + return (api_base, result) + + def poll_command_status(self, task_vars, api_base_url, command_id): + args = self.build_args(task_vars, additional_args=dict( + url = self.build_url(api_base_url, "/commands/" + str(command_id)) + )) + result = self._execute_module('uri', module_args=args, task_vars=task_vars, wrap_async=self._task.async_val) + return result + + def run(self, tmp=None, task_vars=None): + + result = super(ActionModule, self).run(tmp, task_vars) + + # Get Cloudera Manager API base url from task vars, or work it out ourselves + api_base_url, api_base_result = self.get_api_base_url(task_vars) + if not api_base_url: + result.update(api_base_result) + return result + + # Add endpoint and request method to base args containing creds etc + uri_module_args = self.build_args(task_vars, additional_args=dict( + url = self.build_url(api_base_url, self._task.args['endpoint']), + method = self._task.args.get('method') or 'GET', + status_code = self._task.args.get('status_code') or '200', + timeout = self._task.args.get('timeout') or '30' + )) + + poll_duration = int(self._task.args.get('poll_duration') or 10) + poll_max_failed_retries = int(self._task.args.get('poll_max_failed_retries') or 3) + + # Add request body if necessary + if 'body' in self._task.args: + uri_module_args.update(body = self._task.args['body']) + + # Send request to CM API + uri_result = self._execute_module('uri', module_args=uri_module_args, task_vars=task_vars, wrap_async=self._task.async_val) + result.update(uri_result) + + # If we get ApiCommand response, and it is active, wait for completion + failed_polls = 0 + if 'json' in uri_result: + response = uri_result['json'] + if 'id' in response and 'active' in response: + command_id = response['id'] + command_name = response['name'] + command_active = response['active'] + while command_active and failed_polls < poll_max_failed_retries: + time.sleep(poll_duration) + display.vv("Waiting for {} command ({}) to complete...".format(command_name, command_id)) + command_status = self.poll_command_status(task_vars, api_base_url, command_id) + if 'json' in command_status: + failed_polls = 0 + response = command_status['json'] + command_active = response['active'] + else: + failed_polls += 1 + response = {'success': False} + display.vv("Failed to poll command ({}) for status (attempt {} of {})...".format( + command_id, failed_polls, poll_max_failed_retries)) + result.update(command_status) + result['failed'] = not response['success'] + + return result diff --git a/plugins/filter/filters.py b/plugins/filter/filters.py new file mode 100644 index 00000000..356ccd5b --- /dev/null +++ b/plugins/filter/filters.py @@ -0,0 +1,189 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import (absolute_import, division, print_function) +import re + +__metaclass__ = type + + +class FilterModule(object): + + def filters(self): + return { + 'flatten_dict_list': self.flatten_dict_list, + 'extract_products_from_manifests': self.extract_products_from_manifests, + 'format_database_type': self.format_database_type, + 'get_product_version': self.get_product_version, + 'get_major_version': self.get_major_version, # Unused + 'append_database_port': self.append_database_port, + 'default_database_port': self.default_database_port, + 'get_database_encoding_mysql': self.get_database_encoding_mysql, + 'get_database_collation_mysql': self.get_database_collation_mysql, + 'filter_null_configs': self.filter_null_configs, + 'to_ldap_type_enum': self.to_ldap_type_enum, + 'extract_parcel_urls': self.extract_parcel_urls, + 'cluster_service_role_hosts': self.cluster_service_role_hosts, + 'find_clusters': self.find_clusters + } + + def flatten_dict_list(self, item, level=2, sep='_', show_index=False): + """ flatten a structure of dicts and lists into a flat array + + e.g. { "a": [1, 2, 3], "b": { "c": "d", "e": "f" } } + with level=2 + becomes ["a_1", "a_2", "a_3", "b_c", "b_d"] + """ + + state = [] + + def _flatten_dict_list(i, l, parents): + if l > 0: + if isinstance(i, dict): + for key, value in i.items(): + _flatten_dict_list(value, l - 1, parents + [str(key)]) + + elif isinstance(i, list): + for index, value in enumerate(i): + if show_index: + _flatten_dict_list(value, l, parents + [str(index)]) + else: + _flatten_dict_list(value, l, parents) + + else: + state.append(sep.join(parents + [str(i)])) + + if l == 0 and len(parents) > 0: + state.append(sep.join(parents)) + + _flatten_dict_list(item, level, []) + + return state + + def extract_products_from_manifests(self, manifests): + products = dict() + for manifest in manifests: + for parcel in manifest['parcels']: + # take first parcel, strip off OS name and file extension + parcel_name = re.sub(r"-[a-z0-9]+\.parcel$", "", str(parcel['parcelName'])) + # the product name is before the first dash + product = parcel_name[:parcel_name.index("-")] + if product not in products: + # the version string is everything after the first dash + version = parcel_name[parcel_name.index("-") + 1:] + products[product] = version + return products + + def extract_parcel_urls(self, manifest_results): + parcels = list() + for result in manifest_results: + manifest_url = result['invocation']['module_args']['url'] + base_url = '/'.join(manifest_url.rsplit('/')[:-1]) + parcel_names = [x['parcelName'] for x in result['json']['parcels']] + parcels += ['/'.join([str(base_url), str(y)]) for y in parcel_names] + return parcels + + def format_database_type(self, database_type): + if database_type == "mariadb": + return "mysql" + return database_type.lower() + + def get_product_version(self, products, product_name): + for product in products: + if product['product'] == product_name: + version = product['version'] + return version[:version.index('-')] if "-" in version else version + + def get_major_version(self, products, product_name): + version = self.get_product_version(products, product_name) + if version: + return version.split('.')[0] + + def append_database_port(self, database_host, database_port=None): + if ":" not in database_host and database_port: + return database_host + ":" + database_port + return database_host + + def default_database_port(self, database_type): + if database_type == "postgresql": + return 5432 + if database_type == "mysql" or database_type == "mariadb": + return 3306 + if database_type == "oracle": + return 1521 + return None + + def get_database_encoding_mysql(self, service_name): + # workaround for https://jira.cloudera.com/browse/CDPD-9290 + if service_name == "RANGER": + database_encoding = "latin1" + else: + database_encoding = "utf8" + return database_encoding + + def get_database_collation_mysql(self, service_name): + # workaround for https://jira.cloudera.com/browse/CDPD-9290 + if service_name == "RANGER": + database_collation = "latin1_swedish_ci" + else: + database_collation = "utf8_general_ci" + return database_collation + + def filter_null_configs(self, configs, existing_configs): + filtered_configs = dict(configs) + for item, value in configs.items(): + if item not in existing_configs and not value: + del filtered_configs[item] + return filtered_configs + + def to_ldap_type_enum(self, s): + if s == "AD": + return "ACTIVE_DIRECTORY" + return s.replace(" ", "_").upper() + + def cluster_service_role_hosts(self, cluster, hostvars, service, roles=None): + candidate_templates = [] + + if 'host_templates' in cluster: + templates = cluster['host_templates'] + + if roles: + for role in roles: + for t_name, t_services in templates.items(): + if service in t_services and role in t_services[service]: + if t_name not in candidate_templates: + candidate_templates.append(t_name) + + else: + for t_name, t_services in templates.items(): + if service in t_services: + candidate_templates.append(t_name) + + hosts = [] + for t_name in candidate_templates: + t_hosts = [ + host + for host, hostvar in hostvars.items() + if host not in hosts + if hostvar.get('host_template') == t_name] + + hosts = hosts + t_hosts + + return hosts + + def find_clusters(self, clusters, name): + return [ + cluster + for cluster in clusters + if cluster.get('name') == name] diff --git a/prepare_nodes.yml b/prepare_nodes.yml deleted file mode 100644 index dfab2323..00000000 --- a/prepare_nodes.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Group hosts by whether TLS is enabled - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts_by_tls.yml - -- name: Apply OS pre-requisite configurations - hosts: cloudera_manager, cluster - become: yes - roles: - - prereqs/os - tags: - - os - -- name: Create local user accounts - hosts: cloudera_manager, cluster, tls - become: yes - gather_facts: no - roles: - - prereqs/user_accounts - tags: - - users - -- name: Install JDK - hosts: cloudera_manager, cluster, tls - become: yes - roles: - - prereqs/jdk - tags: - - jdk - -- name: Install MySQL Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: prereqs/mysql_connector - when: database_type == 'mysql' or database_type == 'mariadb' - tags: - - mysql_connector - -- name: Install Oracle Connector - hosts: cloudera_manager, cluster - gather_facts: no - become: yes - roles: - - role: prereqs/oracle_connector - when: database_type == 'oracle' - tags: - - oracle_connector diff --git a/prepare_tls.yml b/prepare_tls.yml deleted file mode 100644 index 63f697da..00000000 --- a/prepare_tls.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Group hosts by whether TLS is enabled - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts_by_tls.yml - -- name: Create temp directories - hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - file: - path: "{{ local_temp_dir }}/{{ dir }}" - state: directory - loop: - - csrs - - certs - loop_control: - loop_var: dir - -- name: Fetch CA certificates - hosts: ca_server - become: yes - gather_facts: no - tasks: - - fetch: - src: "{{ cert.src }}" - dest: "{{ cert.dest }}" - flat: yes - loop: - - src: "{{ ca_server_root_cert_path }}" - dest: "{{ local_temp_dir }}/certs/cluster_rootca.pem" - - src: "{{ ca_server_intermediate_cert_path }}" - dest: "{{ local_temp_dir }}/certs/cluster_intca.pem" - loop_control: - loop_var: cert - vars_files: - vars/ca_server.yml - tags: - - fetch_ca - -- name: Build TLS keystores and truststores - hosts: tls - become: yes - roles: - - security/tls_generate_csr - - security/tls_signing - - security/tls_install_certs - vars: - local_certs_dir: "{{ local_temp_dir }}/certs" - local_csrs_dir: "{{ local_temp_dir }}/csrs" - -- name: Delete temp directories - hosts: localhost - connection: local - become: no - gather_facts: no - tasks: - - file: - path: "{{ local_temp_dir }}/{{ dir }}" - state: absent - loop: - - csrs - - certs - loop_control: - loop_var: dir diff --git a/prepare_ycloud.yml b/prepare_ycloud.yml deleted file mode 100644 index 065abb23..00000000 --- a/prepare_ycloud.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Prepare YCloud hosts for Cloudera playbook execution - hosts: all - become: yes - tasks: - - - name: Install required packages - package: - name: "{{ package_name }}" - loop: "{{ required_packages[ansible_os_family] | flatten(1) }}" - loop_control: - loop_var: package_name - - - name: Create wheel group - group: - name: wheel - state: present - - - name: Create non-root user with sudo rights for playbook execution - user: - name: playbook - group: wheel - state: present - - - name: Set authorized key - authorized_key: - user: playbook - state: present - key: https://github.infra.cloudera.com/raw/QE/deploy/master/cdep/data/id_rsa.pub - - # ycloud comes with this pre-installed, but we want to test creating it ourselves - - name: Delete hadoop group - group: - name: hadoop - state: absent - - # Workaround for - # https://jira.cloudera.com/browse/COMOPS-931 - # Please remove this command and the gdb package once resolved - - name: Change the group of systemd - shell: - cmd: "yes | gdb -p 1 --ex 'call setgid(0)' --ex quit" - - vars: - required_packages: - RedHat: - - bind-utils - - policycoreutils - - policycoreutils-python - - selinux-policy - - selinux-policy-targeted - - wget - - gdb - Debian: - - dnsutils - - gdb diff --git a/requirements.yml b/requirements.yml index 7ea8537e..eaa40325 100644 --- a/requirements.yml +++ b/requirements.yml @@ -15,7 +15,6 @@ --- roles: - - name: geerlingguy.postgresql version: 2.2.0 diff --git a/roles/cloudera_manager/admin_password/check/meta/main.yml b/roles/cloudera_manager/admin_password/check/meta/main.yml index d5e6594b..263e003f 100644 --- a/roles/cloudera_manager/admin_password/check/meta/main.yml +++ b/roles/cloudera_manager/admin_password/check/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/common + - role: cloudera.cluster.cloudera_manager.common diff --git a/roles/cloudera_manager/admin_password/check/tasks/main.yml b/roles/cloudera_manager/admin_password/check/tasks/main.yml index a9776608..850548aa 100644 --- a/roles/cloudera_manager/admin_password/check/tasks/main.yml +++ b/roles/cloudera_manager/admin_password/check/tasks/main.yml @@ -14,6 +14,15 @@ --- +- name: Wait for Cloudera Manager Port to be up + delegate_to: "{{ groups.cloudera_manager[0] if 'cloudera_manager' in groups else 'localhost' }}" + ansible.builtin.wait_for: + port: "{{ cloudera_manager_port }}" + host: localhost + connect_timeout: 180 + sleep: 5 + +# https://github.com/ansible/ansible/issues/34989 - name: Check the default Cloudera Manager admin password uri: url: "{{ cloudera_manager_protocol }}://{{ cloudera_manager_host }}:{{ cloudera_manager_port }}/api/v2/tools/echo" @@ -21,8 +30,12 @@ url_username: "admin" url_password: "admin" force_basic_auth: yes + status_code: [200, 401] register: default_admin_password_check - failed_when: default_admin_password_check.status not in [200, 401] + retries: 30 + until: + - default_admin_password_check.status is defined + - default_admin_password_check.status != -1 run_once: True when: cloudera_manager_admin_password is defined @@ -36,4 +49,4 @@ run_once: True when: - cloudera_manager_admin_password is defined - - default_admin_password_check.status == 401 + - default_admin_password_check.status == 401 \ No newline at end of file diff --git a/roles/cloudera_manager/admin_password/set/meta/main.yml b/roles/cloudera_manager/admin_password/set/meta/main.yml index 1aee5eba..3a5a0054 100644 --- a/roles/cloudera_manager/admin_password/set/meta/main.yml +++ b/roles/cloudera_manager/admin_password/set/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/admin_password/set/tasks/main.yml b/roles/cloudera_manager/admin_password/set/tasks/main.yml index 6d0e1efb..e9c5cd65 100644 --- a/roles/cloudera_manager/admin_password/set/tasks/main.yml +++ b/roles/cloudera_manager/admin_password/set/tasks/main.yml @@ -15,7 +15,7 @@ --- - name: Update the Cloudera Manager admin password - cm_api: + cloudera.cluster.cm_api: endpoint: /users/admin method: PUT body: diff --git a/roles/cloudera_manager/agent/tasks/main.yml b/roles/cloudera_manager/agent/tasks/main.yml index 890387ce..564e4509 100644 --- a/roles/cloudera_manager/agent/tasks/main.yml +++ b/roles/cloudera_manager/agent/tasks/main.yml @@ -15,7 +15,8 @@ --- - name: Install Cloudera Manager agent packages - package: + ansible.builtin.package: + lock_timeout: 60 name: cloudera-manager-agent update_cache: yes state: present diff --git a/roles/cloudera_manager/agent_config/meta/main.yml b/roles/cloudera_manager/agent_config/meta/main.yml index d5e6594b..721d1d65 100644 --- a/roles/cloudera_manager/agent_config/meta/main.yml +++ b/roles/cloudera_manager/agent_config/meta/main.yml @@ -13,5 +13,7 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/common + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/cloudera_manager/agent_config/tasks/main.yml b/roles/cloudera_manager/agent_config/tasks/main.yml index 2b566076..e1d349b7 100644 --- a/roles/cloudera_manager/agent_config/tasks/main.yml +++ b/roles/cloudera_manager/agent_config/tasks/main.yml @@ -18,7 +18,7 @@ lineinfile: dest: "{{ cloudera_manager_agent_config_file }}" regexp: "^server_host" - line: "server_host={{ cloudera_manager_host }}" + line: "server_host={{ cloudera_manager_host_local }}" notify: - restart cloudera-scm-agent diff --git a/roles/cloudera_manager/api_client/meta/main.yml b/roles/cloudera_manager/api_client/meta/main.yml index 0761a2c4..45aa8eef 100644 --- a/roles/cloudera_manager/api_client/meta/main.yml +++ b/roles/cloudera_manager/api_client/meta/main.yml @@ -14,5 +14,5 @@ --- dependencies: - - role: cloudera_manager/common - - role: cloudera_manager/admin_password/check + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.cloudera_manager.admin_password.check diff --git a/roles/cloudera_manager/api_hosts/tasks/main.yml b/roles/cloudera_manager/api_hosts/tasks/main.yml index a3c63181..28e80b8a 100644 --- a/roles/cloudera_manager/api_hosts/tasks/main.yml +++ b/roles/cloudera_manager/api_hosts/tasks/main.yml @@ -15,7 +15,7 @@ --- - name: Get the host identifiers and names from Cloudera Manager - cm_api: + cloudera.cluster.cm_api: endpoint: /hosts method: GET register: api_hosts_response diff --git a/roles/cloudera_manager/auth_shell/library/auth_shell.py b/roles/cloudera_manager/auth_shell/library/auth_shell.py deleted file mode 100644 index 3a9fe518..00000000 --- a/roles/cloudera_manager/auth_shell/library/auth_shell.py +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/python - -from __future__ import absolute_import, division, print_function -__metaclass__ = type - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.urls import fetch_file, fetch_url, basic_auth_header - -import json -import os -import inspect -import tempfile -from ssl import SSLError -import time - -try: - from urllib2 import HTTPRedirectHandler, HTTPError, build_opener, URLError -except ImportError: - from urllib.request import HTTPRedirectHandler, build_opener - from urllib.error import HTTPError, URLError - - -def get_cm_url(module, cm_host, cm_port): - cm_init_url = 'http://{}:{}'.format(cm_host, cm_port) - - redirect_urls = [] - - def no_redirect_opener(store): - class NoRedirect(HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, hdrs, newurl): - store.append(newurl) - - return NoRedirect() - - try: - build_opener(no_redirect_opener(redirect_urls)).open(cm_init_url) - - except HTTPError as e: - if e.getcode() == 302 and redirect_urls: - return redirect_urls[0].rstrip('/') - else: - module.fail_json(msg='failed to connect to cm', e=e) - - except URLError as e: - if isinstance(e.reason, SSLError): - if e.reason.reason == 'CERTIFICATE_VERIFY_FAILED': - return cm_init_url.rstrip('/') - - module.fail_json(msg='failed to connect to cm', e=e) - - return cm_init_url.rstrip('/') - - -def ensure_cm_version(module, cm_url, cm_user, cm_pass): - echo_url = cm_url + '/api/v41/tools/echo' - - headers = { 'Authorization': basic_auth_header(cm_user, cm_pass) } - - resp, info = fetch_url(module, echo_url, headers=headers, method='GET') - - if info['status'] == 404: - module.fail_json(msg='cm must be 7.1 or above') - - if info['status'] != 200: - module.fail_json(msg='failed to authenticte with cm', e=info) - - -def get_cm_kerbose_info(module, cm_url, cm_user, cm_pass): - kerberos_url = cm_url + '/api/v41/cm/kerberosInfo' - - headers = { 'Authorization': basic_auth_header(cm_user, cm_pass) } - - resp, info = fetch_url(module, kerberos_url, headers=headers, method='GET') - - if info['status'] != 200: - module.fail_json(msg='failed to retrieve kerberos info from cm', e=info) - - return json.load(resp) - - -def generate_prinicipal(module, cm_url, cm_user, cm_pass, principal): - gen_url = cm_url + '/api/v41/cm/commands/generateCredentialsAdhoc' - - headers = { - 'Authorization': basic_auth_header(cm_user, cm_pass), - 'Content-Type': 'application/json' - } - - data = json.dumps({ - 'items': [principal] - }) - - resp, info = fetch_url(module, gen_url, data=data, headers=headers, method='POST') - - if info['status'] != 200: - module.fail_json(msg='failed to generate principal', e=info) - - command_id = json.load(resp).get('id') - - if not command_id: - module.fail_json(msg='failed to generate principal – no command id', e=info) - - command_url = cm_url + '/api/v41/commands/' + str(command_id) - - generated = False - - while not generated: - time.sleep(1) - - resp, info = fetch_url(module, command_url, headers=headers, method='GET') - - if info['status'] != 200: - module.fail_json(msg='failed to generate principal – command ran', e=info) - - command_details = json.load(resp) - - if command_details.get('success'): - generated = True - - if not command_details.get('active') and not command_details.get('success'): - module.fail_json(msg='failed to generate principal – command did not finish', e=info) - - -def get_cm_keytab(module, cm_url, cm_user, cm_pass, prinicpal): - keytab_url = cm_url + '/api/v41/cm/retrieveKeytab' - - headers = { - 'Authorization': basic_auth_header(cm_user, cm_pass), - 'Content-Type': 'application/json' - } - - data = json.dumps({ - 'items': [prinicpal] - }) - - old_mask = os.umask(0o277) - - keytab_file = fetch_file(module, keytab_url, data=data, headers=headers, method='POST') - - os.umask(old_mask) - - return keytab_file - - -def run_module(): - module_args = dict( - cmd=dict(type='str', required=True), - cm_host=dict(type='str', required=True), - cm_port=dict(type='int', required=False, default=7180), - cm_user=dict(type='str', required=False, default='admin'), - cm_pass=dict(type='str', required=False, default='admin', no_log=True), - identity=dict(type='str', required=True), - host=dict(type='str', required=False), - fallback=dict(type='str', required=False), - realm=dict(type='str', required=False), - set_hadoop_env_on_failure=dict(type='bool', default=True), - validate_certs=dict(type='bool', default=True) - ) - - module = AnsibleModule( - argument_spec=module_args, - supports_check_mode=True - ) - - cmd = module.params['cmd'] - cm_host = module.params['cm_host'] - cm_port = module.params['cm_port'] - cm_user = module.params['cm_user'] - cm_pass = module.params['cm_pass'] - identity = module.params['identity'] - host = module.params['host'] - fallback = module.params['fallback'] - set_hadoop_env_on_failure = module.params['set_hadoop_env_on_failure'] - - cm_url = get_cm_url(module, cm_host, cm_port) - - ensure_cm_version(module, cm_url, cm_user, cm_pass) - - cm_kerb_info = get_cm_kerbose_info(module, cm_url, cm_user, cm_pass) - - prelude = '' - - if cm_kerb_info['kerberized']: - realm = module.params['realm'] or cm_kerb_info['kerberosRealm'] - - if host: - principal = '{}/{}@{}'.format(identity, host, realm) - else: - principal = '{}@{}'.format(identity, realm) - - generate_prinicipal(module, cm_url, cm_user, cm_pass, principal) - - cache_file = tempfile.NamedTemporaryFile(dir=module.tmpdir, delete=False) - module.add_cleanup_file(cache_file.name) - - keytab_locaction = get_cm_keytab(module, cm_url, cm_user, cm_pass, principal) - - prelude = inspect.cleandoc(''' - export KRB5CCNAME=FILE:{} - export KEYTAB_PRICIPAL={} - export KEYTAB_LOCATION={} - kinit -kt $KEYTAB_LOCATION $KEYTAB_PRICIPAL - trap kdestroy EXIT - '''.format(cache_file.name, principal, keytab_locaction)) - - if set_hadoop_env_on_failure and not cm_kerb_info['kerberized']: - prelude = 'export HADOOP_USER_NAME={}'.format(fallback or identity) - - if module.check_mode: - module.exit_json(msg='would execute if not in check mode', cmd=prelude+'\n'+cmd) - - final_cmd = prelude+'\n'+cmd - - rc, out, err = module.run_command( - args=final_cmd, - use_unsafe_shell=True - ) - - if rc != 0: - module.fail_json(msg='rc non-zero [{}]'.format(rc), cmd=final_cmd, rc=rc, stdout=out, stderr=err) - - result = { - 'cmd': final_cmd, - 'changed': True, - 'rc': rc, - 'stdout': out, - 'stderr': err - } - - module.exit_json(**result) - - -def main(): - run_module() - - -if __name__ == '__main__': - main() diff --git a/roles/cloudera_manager/autotls/defaults/main.yml b/roles/cloudera_manager/autotls/defaults/main.yml index 30ee7069..ef22c48f 100644 --- a/roles/cloudera_manager/autotls/defaults/main.yml +++ b/roles/cloudera_manager/autotls/defaults/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + host_ssh_username: root host_ssh_password: cloudera diff --git a/roles/cloudera_manager/autotls/meta/main.yml b/roles/cloudera_manager/autotls/meta/main.yml index 1aee5eba..3a5a0054 100644 --- a/roles/cloudera_manager/autotls/meta/main.yml +++ b/roles/cloudera_manager/autotls/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/autotls/tasks/main.yml b/roles/cloudera_manager/autotls/tasks/main.yml index 36a64737..dc3308f0 100644 --- a/roles/cloudera_manager/autotls/tasks/main.yml +++ b/roles/cloudera_manager/autotls/tasks/main.yml @@ -15,7 +15,7 @@ --- - name: Check Cloudera Manager version - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/version register: response @@ -24,7 +24,7 @@ when: response.json.version is version('7.1', '<') - name: Enable Auto-TLS - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/commands/generateCmca method: POST body: "{{ lookup('template', 'request.j2', convert_data=False) }}" @@ -40,6 +40,6 @@ meta: flush_handlers - name: Restart Cloudera Management Service - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service/commands/restart method: POST diff --git a/roles/cloudera_manager/common/defaults/main.yml b/roles/cloudera_manager/common/defaults/main.yml index fe98ef51..a8acb883 100644 --- a/roles/cloudera_manager/common/defaults/main.yml +++ b/roles/cloudera_manager/common/defaults/main.yml @@ -15,7 +15,11 @@ --- cloudera_manager_agent_config_file: /etc/cloudera-scm-agent/config.ini cloudera_manager_protocol: http -cloudera_manager_host: "{{ groups.cloudera_manager | first | default('localhost') }}" +# Allows the cloudera_manager_host to be set to the public FQDN for proxied connections like EC2 when ansible_host is set +cloudera_manager_host_local: "{{ groups.cloudera_manager | first | default('localhost') }}" +cloudera_manager_host_remote: "{{ hostvars[groups.cloudera_manager[0]].ansible_host | default(cloudera_manager_host_local) }}" +cloudera_manager_host: "{{ cloudera_manager_host_remote if 'localhost' in inventory_hostname else cloudera_manager_host_local }}" +# cloudera_manager_port: 7180 cloudera_manager_database_embedded: False cloudera_manager_database_host: "{{ database_host }}" diff --git a/verify_everything.yml b/roles/cloudera_manager/common/meta/main.yml similarity index 85% rename from verify_everything.yml rename to roles/cloudera_manager/common/meta/main.yml index 14db789f..f63fb6e8 100644 --- a/verify_everything.yml +++ b/roles/cloudera_manager/common/meta/main.yml @@ -13,5 +13,5 @@ # limitations under the License. --- -- import_playbook: verify_inventory_and_definition.yml -- import_playbook: verify_parcels_and_hosts.yml +dependencies: + - role: cloudera.cluster.deployment.definition diff --git a/roles/cloudera_manager/config/defaults/main.yml b/roles/cloudera_manager/config/defaults/main.yml index 7b3bfa67..00cab716 100644 --- a/roles/cloudera_manager/config/defaults/main.yml +++ b/roles/cloudera_manager/config/defaults/main.yml @@ -15,4 +15,6 @@ --- api_config_endpoint: cm/config api_config_keys_uppercase: True -api_configs: {} +cm_api_defaults: + PARCEL_DISTRIBUTE_RATE_LIMIT_KBS_PER_SECOND: 256000 +api_configs: "{{ cm_api_defaults | combine(cloudera_manager_options | default({}), recursive=True) }}" diff --git a/roles/cloudera_manager/config/tasks/main.yml b/roles/cloudera_manager/config/tasks/main.yml index d016d617..d306d718 100644 --- a/roles/cloudera_manager/config/tasks/main.yml +++ b/roles/cloudera_manager/config/tasks/main.yml @@ -15,14 +15,14 @@ --- - name: Get existing configs - cm_api: + cloudera.cluster.cm_api: endpoint: "{{ api_config_endpoint }}" register: response - name: Filter out null configs if necessary set_fact: filtered_configs: > - {{ api_configs | filter_null_configs(response.json['items'] | + {{ api_configs | cloudera.cluster.filter_null_configs(response.json['items'] | items2dict(key_name='name', value_name='value')) }} - name: Update configuration endpoint to contain change message @@ -31,7 +31,7 @@ when: message is defined and "message" not in api_config_endpoint - name: Update configuration (via Cloudera Manager API) - cm_api: + cloudera.cluster.cm_api: endpoint: "{{ api_config_endpoint }}" body: "{{ lookup('template', 'config.j2', convert_data=False) }}" method: PUT diff --git a/templates/cm_api.j2 b/roles/cloudera_manager/config/templates/cm_api.j2 similarity index 100% rename from templates/cm_api.j2 rename to roles/cloudera_manager/config/templates/cm_api.j2 diff --git a/roles/cloudera_manager/csds/meta/main.yml b/roles/cloudera_manager/csds/meta/main.yml index d5e6594b..cb61cbd0 100644 --- a/roles/cloudera_manager/csds/meta/main.yml +++ b/roles/cloudera_manager/csds/meta/main.yml @@ -14,4 +14,5 @@ --- dependencies: - - role: cloudera_manager/common + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.credential diff --git a/roles/cloudera_manager/daemons/meta/main.yml b/roles/cloudera_manager/daemons/meta/main.yml index 99e5dcd9..8323b84f 100644 --- a/roles/cloudera_manager/daemons/meta/main.yml +++ b/roles/cloudera_manager/daemons/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/repo + - role: cloudera.cluster.cloudera_manager.repo diff --git a/roles/cloudera_manager/daemons/tasks/main.yml b/roles/cloudera_manager/daemons/tasks/main.yml index 449c9c7c..4d4e6e99 100644 --- a/roles/cloudera_manager/daemons/tasks/main.yml +++ b/roles/cloudera_manager/daemons/tasks/main.yml @@ -15,7 +15,8 @@ --- - name: Install Cloudera Manager daemons package - package: + ansible.builtin.package: + lock_timeout: 180 name: cloudera-manager-daemons update_cache: yes state: present diff --git a/roles/cloudera_manager/database/tasks/embedded.yml b/roles/cloudera_manager/database/tasks/embedded.yml index 04ee79fd..7f8f7159 100644 --- a/roles/cloudera_manager/database/tasks/embedded.yml +++ b/roles/cloudera_manager/database/tasks/embedded.yml @@ -15,7 +15,8 @@ --- - name: Install Cloudera Manager embedded database - package: + ansible.builtin.package: + lock_timeout: 60 name: - cloudera-manager-server-db-2 state: present diff --git a/roles/cloudera_manager/external_account/meta/main.yml b/roles/cloudera_manager/external_account/meta/main.yml index 1aee5eba..524838cc 100644 --- a/roles/cloudera_manager/external_account/meta/main.yml +++ b/roles/cloudera_manager/external_account/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/external_account/tasks/main.yml b/roles/cloudera_manager/external_account/tasks/main.yml index b032aac8..de36fb8f 100644 --- a/roles/cloudera_manager/external_account/tasks/main.yml +++ b/roles/cloudera_manager/external_account/tasks/main.yml @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- + - name: Create Cloudera Manager external accounts - cm_api: + cloudera.cluster.cm_api: endpoint: /externalAccounts/create body: "{{ lookup('template', 'external_account_configs.j2', convert_data=False) }}" status_code: [200, 400] diff --git a/roles/cloudera_manager/external_auth/defaults/main.yml b/roles/cloudera_manager/external_auth/defaults/main.yml index 80f0a8dd..5df31ba9 100644 --- a/roles/cloudera_manager/external_auth/defaults/main.yml +++ b/roles/cloudera_manager/external_auth/defaults/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + cloudera_manager_external_auth: external_first: no external_only: no diff --git a/roles/cloudera_manager/external_auth/meta/main.yml b/roles/cloudera_manager/external_auth/meta/main.yml index 1aee5eba..3a5a0054 100644 --- a/roles/cloudera_manager/external_auth/meta/main.yml +++ b/roles/cloudera_manager/external_auth/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/cloudera_manager/external_auth/tasks/create_mapping.yml b/roles/cloudera_manager/external_auth/tasks/create_mapping.yml index 8613645e..79cb8657 100644 --- a/roles/cloudera_manager/external_auth/tasks/create_mapping.yml +++ b/roles/cloudera_manager/external_auth/tasks/create_mapping.yml @@ -13,7 +13,7 @@ # limitations under the License. - name: Create Cloudera Manager external auth user to role mapping - cm_api: + cloudera.cluster.cm_api: endpoint: /externalUserMappings body: "{{ lookup('template', 'external_auth_mapping.j2', convert_data=False) }}" status_code: [200, 400] diff --git a/roles/cloudera_manager/external_auth/tasks/main.yml b/roles/cloudera_manager/external_auth/tasks/main.yml index 60b64050..2a3ecf6d 100644 --- a/roles/cloudera_manager/external_auth/tasks/main.yml +++ b/roles/cloudera_manager/external_auth/tasks/main.yml @@ -30,7 +30,7 @@ - block: - name: Get auth roles from Cloudera Manager - cm_api: + cloudera.cluster.cm_api: endpoint: /authRoles register: response @@ -40,7 +40,7 @@ ## BUG: Can't set all mappings in one API call because of OPSAPS-56242 # - name: Set Cloudera Manager external auth user to role mappings - # cm_api: + # cloudera.cluster.cm_api: # endpoint: /externalUserMappings # body: "{{ lookup('template', 'external_auth_mappings.j2', convert_data=False) }}" # method: POST diff --git a/roles/cloudera_manager/external_auth/templates/cm_api.j2 b/roles/cloudera_manager/external_auth/templates/cm_api.j2 new file mode 100644 index 00000000..88ab1447 --- /dev/null +++ b/roles/cloudera_manager/external_auth/templates/cm_api.j2 @@ -0,0 +1,90 @@ +{% macro ApiConfig(k, v, force_uppercase_keys=False) -%} + {%- set config = {} -%} + {%- if force_uppercase_keys -%} + {{ config.update({"name": k | upper}) }} + {%- else -%} + {{ config.update({"name": k}) }} + {%- endif -%} + {# the extra to_json filter renders a dict into properly escaped JSON string for the CM API call #} + {%- if v is mapping -%} + {{ config.update({"value": v | to_json}) }} + {%- else -%} + {{ config.update({"value": v}) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiConfigList(configs, force_uppercase_keys=False) -%} + {%- set config_list = { "items": [] } -%} + {%- if configs is mapping -%} + {%- for (k, v) in configs.items() -%} + {%- set config = ApiConfig(k, v, force_uppercase_keys) -%} + {{ config_list["items"].append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfig(key, service_type, role_type, value, value_type="value") -%} + {%- set config = {} -%} + {{ config.update({"name": key}) }} + {%- if value_type == "ref" or value_type == "value" -%} + {{ config.update({ value_type: value }) }} + {%- else -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {{ config.update({ "variable": var_name | upper }) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfigList(service_merged_configs, service_type, role_type) -%} + {%- set config_list = [] -%} + {%- if service_type in service_merged_configs -%} + {%- if service_merged_configs[service_type][role_type] is mapping -%} + {%- for (k, v) in service_merged_configs[service_type][role_type].items() -%} + {%- if k.endswith("_service") and not "suppression" in k -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "ref") -%} + {%- else -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%} + {%- endif -%} + {{ config_list.append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiService(service) -%} +{} +{%- endmacro %} + +{% macro ApiServiceList(services) -%} + {%- set api_service_list = { "items": [] } -%} + {%- if services -%} + {%- for service in services -%} + {%- set api_service = ApiService(service) -%} + {{ api_service_list["items"].append(api_service | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ api_service_list | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMapping(auth_type, user_mapping) %} + {%- set external_user_mapping = {'name': user_mapping.group, 'type': auth_type, 'authRoles': []} -%} + {%- for role in user_mapping.roles -%} + {{ external_user_mapping['authRoles'].append(ApiAuthRoleRef(role) | from_json) }} + {%- endfor -%} + {{ external_user_mapping | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMappingList(auth_type, auth_user_mappings) %} + {%- set external_user_mapping_list = { "items": [] } -%} + {%- for user_mapping in auth_user_mappings -%} + {{ external_user_mapping_list["items"].append(ApiExternalUserMapping(auth_type, user_mapping) | from_json) }} + {%- endfor -%} + {{ external_user_mapping_list | to_json }} +{%- endmacro %} + +{% macro ApiAuthRoleRef(name) -%} + { "uuid": {{ auth_role_uuids[name] | to_json }} } +{%- endmacro %} diff --git a/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 b/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 index bb535b03..bd23c832 100644 --- a/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 +++ b/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 @@ -15,7 +15,7 @@ LDAP_BIND_PW: {{ auth_provider.ldap_bind_password | default(None) }} LDAP_DN_PATTERN: {{ auth_provider.ldap_dn_pattern | default(None) }} LDAP_GROUP_SEARCH_BASE: {{ auth_provider.ldap_search_base.group | default(None) }} LDAP_GROUP_SEARCH_FILTER: "({{ auth_provider.ldap_attribute.member | default('member') }}={0})" -LDAP_TYPE: {{ auth_provider.type | to_ldap_type_enum | default(None) }} +LDAP_TYPE: {{ auth_provider.type | cloudera.cluster.to_ldap_type_enum | default(None) }} LDAP_URL: {{ auth_provider.ldap_url | default(None) }} LDAP_USER_SEARCH_BASE: {{ auth_provider.ldap_search_base.user | default(None) }} LDAP_USER_SEARCH_FILTER: "({{ auth_provider.ldap_attribute.user | default('sAMAccountName') }}={0})" diff --git a/roles/cloudera_manager/external_auth/vars/main.yml b/roles/cloudera_manager/external_auth/vars/main.yml index 7497e889..0351c258 100644 --- a/roles/cloudera_manager/external_auth/vars/main.yml +++ b/roles/cloudera_manager/external_auth/vars/main.yml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- + auth_role_display_names: ROLE_ADMIN: Full Administrator ROLE_AUDITOR: Auditor diff --git a/roles/cloudera_manager/kerberos/meta/main.yml b/roles/cloudera_manager/kerberos/meta/main.yml index 1aee5eba..ca9fad9c 100644 --- a/roles/cloudera_manager/kerberos/meta/main.yml +++ b/roles/cloudera_manager/kerberos/meta/main.yml @@ -13,5 +13,7 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client + - role: cloudera.cluster.infrastructure.krb5_common diff --git a/roles/cloudera_manager/kerberos/tasks/main.yml b/roles/cloudera_manager/kerberos/tasks/main.yml index eb8b18b7..a15ff7b7 100644 --- a/roles/cloudera_manager/kerberos/tasks/main.yml +++ b/roles/cloudera_manager/kerberos/tasks/main.yml @@ -16,13 +16,13 @@ - name: Set Cloudera Manager Kerberos configs include_role: - name: cloudera_manager/config + name: cloudera.cluster.cloudera_manager.config vars: api_config_keys_uppercase: True api_configs: "{{ lookup('template', 'kerberos_configs.j2') | from_yaml }}" - name: Import KDC admin credentials - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/commands/importAdminCredentials?username={{ krb5_kdc_admin_user | urlencode }}&password={{ krb5_kdc_admin_password | urlencode }} method: POST register: result diff --git a/roles/cloudera_manager/license/meta/main.yml b/roles/cloudera_manager/license/meta/main.yml index 1aee5eba..09e142be 100644 --- a/roles/cloudera_manager/license/meta/main.yml +++ b/roles/cloudera_manager/license/meta/main.yml @@ -14,4 +14,5 @@ --- dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client + - role: cloudera.cluster.prereqs.license diff --git a/roles/cloudera_manager/license/tasks/enterprise.yml b/roles/cloudera_manager/license/tasks/enterprise.yml index 5cb00f01..440a9457 100644 --- a/roles/cloudera_manager/license/tasks/enterprise.yml +++ b/roles/cloudera_manager/license/tasks/enterprise.yml @@ -15,7 +15,7 @@ --- - name: Get current Cloudera license status - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/license method: GET status_code: 200,404 @@ -29,12 +29,21 @@ 'This installation currently has no license.' ]}} -- name: Upload license file to Cloudera Manager server - copy: - src: "{{ cloudera_manager_license_file }}" - dest: /tmp/license.txt - mode: 0600 - when: license_not_installed +- name: Check for Trial License + when: not license_not_installed + ansible.builtin.set_fact: + license_not_installed: "{{ 'Trial' in license_response.json.owner | default('') }}" + +- name: Determine if License File is present + ansible.builtin.stat: + path: "{{ license_local_tmp_path }}" + register: __cloudera_license_file + +- name: Check License File Stat + ansible.builtin.assert: + that: __cloudera_license_file.stat.exists + fail_msg: "Expected to find Cloudera License file at {{ license_local_tmp_path }}" + quiet: yes - name: Post license file to Cloudera Manager API shell: > @@ -42,7 +51,7 @@ --user {{ cloudera_manager_api_user }}:{{ cloudera_manager_api_password }} --request POST --header 'Content-Type:multipart/form-data' - --form license=@/tmp/license.txt + --form license=@{{ license_local_tmp_path }} {{ license_response.url }} args: warn: False diff --git a/roles/cloudera_manager/license/tasks/trial.yml b/roles/cloudera_manager/license/tasks/trial.yml index 89ced2e4..76d70466 100644 --- a/roles/cloudera_manager/license/tasks/trial.yml +++ b/roles/cloudera_manager/license/tasks/trial.yml @@ -15,7 +15,7 @@ --- - name: Begin Cloudera Manager trial license - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/trial/begin method: POST status_code: 200,204 diff --git a/roles/cloudera_manager/preload_parcels/defaults/main.yml b/roles/cloudera_manager/preload_parcels/defaults/main.yml new file mode 100644 index 00000000..df8417e1 --- /dev/null +++ b/roles/cloudera_manager/preload_parcels/defaults/main.yml @@ -0,0 +1,15 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +preload_parcels: "{{ definition.preload_parcels | default([]) }}" \ No newline at end of file diff --git a/roles/cloudera_manager/preload_parcels/tasks/main.yml b/roles/cloudera_manager/preload_parcels/tasks/main.yml new file mode 100644 index 00000000..33603fcd --- /dev/null +++ b/roles/cloudera_manager/preload_parcels/tasks/main.yml @@ -0,0 +1,47 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Ensure Parcel Repo directory is present + ansible.builtin.file: + path: /opt/cloudera/parcel-repo + state: directory + +# Cloudera Manager expects file hashes to be .sha extension +- name: Request Async Download of Parcels to Cloudera Manager Parcel Repo + when: preload_parcels + register: __infra_download_parcels_results + loop: "{{ preload_parcels | select('match', '.*parcel.*') | list }}" + loop_control: + loop_var: __parcel_download_item + async: 3600 + poll: 0 + ansible.builtin.get_url: + url: "{{ __parcel_download_item }}" + dest: "/opt/cloudera/parcel-repo/{{ __parcel_download_item | urlsplit('path') | basename | replace('.sha1', '.sha') }}" + +- name: Track async downloads to completion [ This may take a while if your files are very large or far away ] + when : preload_parcels + loop: "{{ __infra_download_parcels_results.results }}" + loop_control: + loop_var: __download_async_item + register: __async_download_results + until: __async_download_results.finished is defined and __async_download_results.finished + delay: 15 + retries: 300 + async_status: + jid: "{{ __download_async_item.ansible_job_id }}" + failed_when: + - __download_async_item.failed == True + - __download_async_item.finished != 1 diff --git a/roles/cloudera_manager/repo/defaults/main.yml b/roles/cloudera_manager/repo/defaults/main.yml index 4053da07..4215ee22 100644 --- a/roles/cloudera_manager/repo/defaults/main.yml +++ b/roles/cloudera_manager/repo/defaults/main.yml @@ -14,4 +14,6 @@ --- cloudera_archive_base_url: https://archive.cloudera.com -cloudera_manager_version: 7.1.1 +cloudera_manager_version: 7.1.4 + +install_repo_on_host: yes \ No newline at end of file diff --git a/roles/cloudera_manager/repo/meta/main.yml b/roles/cloudera_manager/repo/meta/main.yml index d5e6594b..cb61cbd0 100644 --- a/roles/cloudera_manager/repo/meta/main.yml +++ b/roles/cloudera_manager/repo/meta/main.yml @@ -14,4 +14,5 @@ --- dependencies: - - role: cloudera_manager/common + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.credential diff --git a/roles/cloudera_manager/repo/tasks/main.yml b/roles/cloudera_manager/repo/tasks/main.yml index e067ab07..c7b7f881 100644 --- a/roles/cloudera_manager/repo/tasks/main.yml +++ b/roles/cloudera_manager/repo/tasks/main.yml @@ -42,5 +42,6 @@ when: cloudera_manager_repo_key is not defined - name: Install Cloudera Manager repository + when: install_repo_on_host include_tasks: file: "main-{{ ansible_os_family }}.yml" diff --git a/roles/cloudera_manager/server/meta/main.yml b/roles/cloudera_manager/server/meta/main.yml index 99e5dcd9..c35c03c4 100644 --- a/roles/cloudera_manager/server/meta/main.yml +++ b/roles/cloudera_manager/server/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera_manager/repo + - role: cloudera.cluster.cloudera_manager.repo diff --git a/roles/cloudera_manager/server/tasks/main.yml b/roles/cloudera_manager/server/tasks/main.yml index 538f0ac9..553b2f8e 100644 --- a/roles/cloudera_manager/server/tasks/main.yml +++ b/roles/cloudera_manager/server/tasks/main.yml @@ -15,7 +15,8 @@ --- - name: Install Cloudera Manager Server - package: + ansible.builtin.package: + lock_timeout: 60 name: cloudera-manager-server state: present @@ -36,11 +37,11 @@ - name: Install database import_role: - name: cloudera_manager/database + name: cloudera.cluster.cloudera_manager.database - name: Install CSDs import_role: - name: cloudera_manager/csds + name: cloudera.cluster.cloudera_manager.csds - name: Start Cloudera Manager Server service: diff --git a/roles/cloudera_manager/server_tls/meta/main.yml b/roles/cloudera_manager/server_tls/meta/main.yml index 1aee5eba..55300fe4 100644 --- a/roles/cloudera_manager/server_tls/meta/main.yml +++ b/roles/cloudera_manager/server_tls/meta/main.yml @@ -13,5 +13,7 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/cloudera_manager/server_tls/tasks/main.yml b/roles/cloudera_manager/server_tls/tasks/main.yml index c534d742..ad900d36 100644 --- a/roles/cloudera_manager/server_tls/tasks/main.yml +++ b/roles/cloudera_manager/server_tls/tasks/main.yml @@ -16,7 +16,8 @@ - name: Set Cloudera Manager TLS configs include_role: - name: cloudera_manager/config + name: cloudera.cluster.cloudera_manager.config + when: tls | default(False) vars: api_config_keys_uppercase: True api_configs: "{{ lookup('template', 'tls_configs.j2') | from_yaml }}" @@ -38,6 +39,7 @@ - "'javax.net.ssl.trustStorePassword' not in opts" - name: Restart Cloudera Manager Server + when: tls | default(False) service: name: cloudera-scm-server state: restarted diff --git a/roles/config/cluster/base/meta/main.yml b/roles/config/cluster/base/meta/main.yml index 4e3a2f89..cf20f5e5 100644 --- a/roles/config/cluster/base/meta/main.yml +++ b/roles/config/cluster/base/meta/main.yml @@ -13,7 +13,9 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/common - - role: deployment/definition - - role: config/cluster/common + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.config.cluster.common + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/config/cluster/base/tasks/main.yml b/roles/config/cluster/base/tasks/main.yml index 876a899d..61854262 100644 --- a/roles/config/cluster/base/tasks/main.yml +++ b/roles/config/cluster/base/tasks/main.yml @@ -21,7 +21,7 @@ - name: Retrieve repository metadata include_role: - name: deployment/repometa + name: cloudera.cluster.deployment.repometa vars: repositories: "{{ cluster.repositories | default({}) }}" diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 index 5ed82fef..ce62c5f2 100644 --- a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 @@ -2,7 +2,7 @@ DAS: SERVICEWIDE: data_analytics_studio_database_host: {{ databases.DAS.host }} - data_analytics_studio_database_type: {{ databases.DAS.type | format_database_type }} + data_analytics_studio_database_type: {{ databases.DAS.type | cloudera.cluster.format_database_type }} data_analytics_studio_database_port: {{ databases.DAS.port }} data_analytics_studio_database_name: {{ databases.DAS.name }} data_analytics_studio_database_username: {{ databases.DAS.user }} @@ -11,7 +11,7 @@ RANGER: SERVICEWIDE: ranger_database_host: {{ databases.RANGER.host }} ranger_database_port: {{ databases.RANGER.port }} - ranger_database_type: {{ databases.RANGER.type | format_database_type }} + ranger_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }} ranger_database_name: {{ databases.RANGER.name }} ranger_database_user: {{ databases.RANGER.user }} ranger_database_password: {{ databases.RANGER.password }} @@ -19,7 +19,7 @@ SCHEMAREGISTRY: SERVICEWIDE: database_host: {{ databases.SCHEMAREGISTRY.host }} database_port: {{ databases.SCHEMAREGISTRY.port }} - database_type: {{ databases.SCHEMAREGISTRY.type | format_database_type }} + database_type: {{ databases.SCHEMAREGISTRY.type | cloudera.cluster.format_database_type }} database_name: {{ databases.SCHEMAREGISTRY.name }} database_user: {{ databases.SCHEMAREGISTRY.user }} database_password: {{ databases.SCHEMAREGISTRY.password }} @@ -27,7 +27,7 @@ STREAMS_MESSAGING_MANAGER: SERVICEWIDE: smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }} smm_database_port: {{ databases.STREAMS_MESSAGING_MANAGER.port }} - smm_database_type: {{ databases.STREAMS_MESSAGING_MANAGER.type | format_database_type }} + smm_database_type: {{ databases.STREAMS_MESSAGING_MANAGER.type | cloudera.cluster.format_database_type }} smm_database_name: {{ databases.STREAMS_MESSAGING_MANAGER.name }} smm_database_user: {{ databases.STREAMS_MESSAGING_MANAGER.user }} smm_database_password: {{ databases.STREAMS_MESSAGING_MANAGER.password }} diff --git a/roles/config/cluster/base/templates/configs/databases.j2 b/roles/config/cluster/base/templates/configs/databases.j2 index 9dd002cf..d4dd123d 100644 --- a/roles/config/cluster/base/templates/configs/databases.j2 +++ b/roles/config/cluster/base/templates/configs/databases.j2 @@ -3,7 +3,7 @@ HIVE: SERVICEWIDE: hive_metastore_database_host: {{ databases.HIVE.host }} hive_metastore_database_port: {{ databases.HIVE.port }} - hive_metastore_database_type: {{ databases.HIVE.type | format_database_type }} + hive_metastore_database_type: {{ databases.HIVE.type | cloudera.cluster.format_database_type }} hive_metastore_database_name: {{ databases.HIVE.name }} hive_metastore_database_user: {{ databases.HIVE.user }} hive_metastore_database_password: {{ databases.HIVE.password }} @@ -11,14 +11,14 @@ HUE: SERVICEWIDE: database_host: {{ databases.HUE.host }} database_port: {{ databases.HUE.port }} - database_type: {{ databases.HUE.type | format_database_type }} + database_type: {{ databases.HUE.type | cloudera.cluster.format_database_type }} database_name: {{ databases.HUE.name }} database_user: {{ databases.HUE.user }} database_password: {{ databases.HUE.password }} OOZIE: OOZIE_SERVER: - oozie_database_host: {{ databases.OOZIE.host | append_database_port(databases.OOZIE.port | string) }} - oozie_database_type: {{ databases.OOZIE.type | format_database_type }} + oozie_database_host: {{ databases.OOZIE.host | cloudera.cluster.append_database_port(databases.OOZIE.port | string) }} + oozie_database_type: {{ databases.OOZIE.type | cloudera.cluster.format_database_type }} oozie_database_name: {{ databases.OOZIE.name }} oozie_database_user: {{ databases.OOZIE.user }} oozie_database_password: {{ databases.OOZIE.password }} @@ -26,7 +26,7 @@ SENTRY: SERVICEWIDE: sentry_server_database_host: {{ databases.SENTRY.host }} sentry_server_database_port: {{ databases.SENTRY.port }} - sentry_server_database_type: {{ databases.SENTRY.type | format_database_type }} + sentry_server_database_type: {{ databases.SENTRY.type | cloudera.cluster.format_database_type }} sentry_server_database_name: {{ databases.SENTRY.name }} sentry_server_database_user: {{ databases.SENTRY.user }} sentry_server_database_password: {{ databases.SENTRY.password }} diff --git a/roles/config/cluster/base/templates/configs/schemaregistry.j2 b/roles/config/cluster/base/templates/configs/schemaregistry.j2 index c41d69ac..30cb52ea 100644 --- a/roles/config/cluster/base/templates/configs/schemaregistry.j2 +++ b/roles/config/cluster/base/templates/configs/schemaregistry.j2 @@ -1,6 +1,6 @@ --- SCHEMAREGISTRY: SCHEMA_REGISTRY_SERVER: - schema.registry.storage.connector.connectURI: "jdbc:{{ databases.SCHEMAREGISTRY.type | format_database_type }}://{{ databases.SCHEMAREGISTRY.host }}:{{ databases.SCHEMAREGISTRY.port }}/{{ databases.SCHEMAREGISTRY.name }}" + schema.registry.storage.connector.connectURI: "jdbc:{{ databases.SCHEMAREGISTRY.type | cloudera.cluster.format_database_type }}://{{ databases.SCHEMAREGISTRY.host }}:{{ databases.SCHEMAREGISTRY.port }}/{{ databases.SCHEMAREGISTRY.name }}" schema.registry.storage.connector.user: {{ databases.SCHEMAREGISTRY.user }} schema.registry.storage.connector.password: {{ databases.SCHEMAREGISTRY.password }} diff --git a/roles/config/cluster/base/templates/configs/smm.j2 b/roles/config/cluster/base/templates/configs/smm.j2 index 90d15a40..5042de33 100644 --- a/roles/config/cluster/base/templates/configs/smm.j2 +++ b/roles/config/cluster/base/templates/configs/smm.j2 @@ -8,10 +8,10 @@ SENTRY: sentry_service_allow_connect: hive,impala,hue,hdfs,solr,kafka,hbase,kudu,streamsmsgmgr STREAMS_MESSAGING_MANAGER: SERVICEWIDE: - cm.metrics.host: {{ cloudera_manager_host }} + cm.metrics.host: {{ cloudera_manager_host_local }} cm.metrics.password: {{ cloudera_manager_api_password }} cm.metrics.service.name: kafka STREAMS_MESSAGING_MANAGER_SERVER: - streams.messaging.manager.storage.connector.connectURI: "jdbc:{{ databases.STREAMS_MESSAGING_MANAGER.type | format_database_type }}://{{ databases.STREAMS_MESSAGING_MANAGER.host }}:{{ databases.STREAMS_MESSAGING_MANAGER.port }}/{{ databases.STREAMS_MESSAGING_MANAGER.name }}" + streams.messaging.manager.storage.connector.connectURI: "jdbc:{{ databases.STREAMS_MESSAGING_MANAGER.type | cloudera.cluster.format_database_type }}://{{ databases.STREAMS_MESSAGING_MANAGER.host }}:{{ databases.STREAMS_MESSAGING_MANAGER.port }}/{{ databases.STREAMS_MESSAGING_MANAGER.name }}" streams.messaging.manager.storage.connector.user: {{ databases.STREAMS_MESSAGING_MANAGER.user }} streams.messaging.manager.storage.connector.password: {{ databases.STREAMS_MESSAGING_MANAGER.password }} diff --git a/roles/config/cluster/base/vars/main.yml b/roles/config/cluster/base/vars/main.yml index a03138e4..37bda732 100644 --- a/roles/config/cluster/base/vars/main.yml +++ b/roles/config/cluster/base/vars/main.yml @@ -13,8 +13,9 @@ # limitations under the License. --- + custom_config_templates: -# Explicit defaults +# Explicit defaults - template: configs/defaults.j2 # Custom configurations for databases - template: configs/databases.j2 diff --git a/roles/config/cluster/common/defaults/main.yml b/roles/config/cluster/common/defaults/main.yml index 3a05dce2..facf8c87 100644 --- a/roles/config/cluster/common/defaults/main.yml +++ b/roles/config/cluster/common/defaults/main.yml @@ -13,10 +13,12 @@ # limitations under the License. --- -cluster_definition_file: cluster.yml + cluster_template_dry_run: False tls: False +default_cluster_type: base + kms_services: [KEYTRUSTEE, RANGER_KMS, RANGER_KMS_KTS] sdx_services: [ATLAS, HDFS, HIVE, RANGER, SENTRY] diff --git a/roles/config/cluster/kts/meta/main.yml b/roles/config/cluster/kts/meta/main.yml index 4e3a2f89..cf20f5e5 100644 --- a/roles/config/cluster/kts/meta/main.yml +++ b/roles/config/cluster/kts/meta/main.yml @@ -13,7 +13,9 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/common - - role: deployment/definition - - role: config/cluster/common + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.config.cluster.common + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/config/cluster/kts/vars/main.yml b/roles/config/cluster/kts/vars/main.yml index 34e5078e..044d9d3e 100644 --- a/roles/config/cluster/kts/vars/main.yml +++ b/roles/config/cluster/kts/vars/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + custom_config_templates: # Custom configurations for TLS - template: configs/tls.j2 diff --git a/roles/config/services/kms/meta/main.yml b/roles/config/services/kms/meta/main.yml index ba00be71..ddcd8f66 100644 --- a/roles/config/services/kms/meta/main.yml +++ b/roles/config/services/kms/meta/main.yml @@ -13,5 +13,7 @@ # limitations under the License. --- + dependencies: - - role: deployment/definition + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/config/services/kms/vars/main.yml b/roles/config/services/kms/vars/main.yml index 0f8ff614..b49c28f8 100644 --- a/roles/config/services/kms/vars/main.yml +++ b/roles/config/services/kms/vars/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + custom_config_templates: - template: configs/defaults.j2 - template: configs/tls.j2 diff --git a/roles/config/services/mgmt/meta/main.yml b/roles/config/services/mgmt/meta/main.yml index a3e864f6..867b8869 100644 --- a/roles/config/services/mgmt/meta/main.yml +++ b/roles/config/services/mgmt/meta/main.yml @@ -13,6 +13,8 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/common - - role: deployment/definition + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/config/services/mgmt/templates/configs/databases.j2 b/roles/config/services/mgmt/templates/configs/databases.j2 index b2114bb6..f5062010 100644 --- a/roles/config/services/mgmt/templates/configs/databases.j2 +++ b/roles/config/services/mgmt/templates/configs/databases.j2 @@ -1,25 +1,25 @@ --- ACTIVITYMONITOR: - firehose_database_host: {{ databases.ACTIVITYMONITOR.host | append_database_port(databases.ACTIVITYMONITOR.port | string) }} - firehose_database_type: {{ databases.ACTIVITYMONITOR.type | format_database_type }} + firehose_database_host: {{ databases.ACTIVITYMONITOR.host | cloudera.cluster.append_database_port(databases.ACTIVITYMONITOR.port | string) }} + firehose_database_type: {{ databases.ACTIVITYMONITOR.type | cloudera.cluster.format_database_type }} firehose_database_name: {{ databases.ACTIVITYMONITOR.name }} firehose_database_user: {{ databases.ACTIVITYMONITOR.user }} firehose_database_password: {{ databases.ACTIVITYMONITOR.password }} NAVIGATOR: - navigator_database_host: {{ databases.NAVIGATOR.host | append_database_port(databases.NAVIGATOR.port | string) }} - navigator_database_type: {{ databases.NAVIGATOR.type | format_database_type }} + navigator_database_host: {{ databases.NAVIGATOR.host | cloudera.cluster.append_database_port(databases.NAVIGATOR.port | string) }} + navigator_database_type: {{ databases.NAVIGATOR.type | cloudera.cluster.format_database_type }} navigator_database_name: {{ databases.NAVIGATOR.name }} navigator_database_user: {{ databases.NAVIGATOR.user }} navigator_database_password: {{ databases.NAVIGATOR.password }} NAVIGATORMETASERVER: - nav_metaserver_database_host: {{ databases.NAVIGATORMETASERVER.host | append_database_port(databases.NAVIGATOR.port | string) }} - nav_metaserver_database_type: {{ databases.NAVIGATORMETASERVER.type | format_database_type }} + nav_metaserver_database_host: {{ databases.NAVIGATORMETASERVER.host | cloudera.cluster.append_database_port(databases.NAVIGATOR.port | string) }} + nav_metaserver_database_type: {{ databases.NAVIGATORMETASERVER.type | cloudera.cluster.format_database_type }} nav_metaserver_database_name: {{ databases.NAVIGATORMETASERVER.name }} nav_metaserver_database_user: {{ databases.NAVIGATORMETASERVER.user }} nav_metaserver_database_password: {{ databases.NAVIGATORMETASERVER.password }} REPORTSMANAGER: - headlamp_database_host: {{ databases.REPORTSMANAGER.host | append_database_port(databases.REPORTSMANAGER.port | string) }} - headlamp_database_type: {{ databases.REPORTSMANAGER.type | format_database_type }} + headlamp_database_host: {{ databases.REPORTSMANAGER.host | cloudera.cluster.append_database_port(databases.REPORTSMANAGER.port | string) }} + headlamp_database_type: {{ databases.REPORTSMANAGER.type | cloudera.cluster.format_database_type }} headlamp_database_name: {{ databases.REPORTSMANAGER.name }} headlamp_database_user: {{ databases.REPORTSMANAGER.user }} headlamp_database_password: {{ databases.REPORTSMANAGER.password }} diff --git a/roles/deployment/cluster/meta/main.yml b/roles/deployment/cluster/meta/main.yml index 77c44f6b..1e6c9a0c 100644 --- a/roles/deployment/cluster/meta/main.yml +++ b/roles/deployment/cluster/meta/main.yml @@ -14,5 +14,5 @@ --- dependencies: - - role: cloudera_manager/api_hosts - - role: config/cluster/common + - role: cloudera.cluster.cloudera_manager.api_hosts + diff --git a/roles/deployment/cluster/tasks/create_base.yml b/roles/deployment/cluster/tasks/create_base.yml index fe7934cf..b476c026 100644 --- a/roles/deployment/cluster/tasks/create_base.yml +++ b/roles/deployment/cluster/tasks/create_base.yml @@ -32,7 +32,7 @@ #when: cluster_template_dry_run - name: Import cluster template - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/importClusterTemplate?addRepositories=true method: POST body: "{{ lookup('template', 'cluster_template/main.j2', convert_data=False) }}" diff --git a/roles/deployment/cluster/tasks/create_data_context.yml b/roles/deployment/cluster/tasks/create_data_context.yml index 3910c316..c3b08b5b 100644 --- a/roles/deployment/cluster/tasks/create_data_context.yml +++ b/roles/deployment/cluster/tasks/create_data_context.yml @@ -15,7 +15,7 @@ --- - name: Create data contexts - cm_api: + cloudera.cluster.cm_api: endpoint: /dataContexts method: POST body: "{{ lookup('template', 'sdx/data_context.j2', convert_data=False) }}" diff --git a/roles/deployment/cluster/tasks/create_kts.yml b/roles/deployment/cluster/tasks/create_kts.yml index d683acf6..dea39fc4 100644 --- a/roles/deployment/cluster/tasks/create_kts.yml +++ b/roles/deployment/cluster/tasks/create_kts.yml @@ -27,7 +27,7 @@ - block: - name: Import cluster template - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/importClusterTemplate?addRepositories=true method: POST body: "{{ lookup('template', 'cluster_template/main.j2', convert_data=False) }}" @@ -42,7 +42,7 @@ # If we have installed a cluster with Key Trustee Server HA, first run will have failed (but this is ok) # Stop the service now in preparation for remedial action - name: Stop Key Trustee Server service when first run failed - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/services/keytrustee_server/commands/stop method: POST when: "'kts_passive' in groups and first_run_failure" diff --git a/roles/deployment/cluster/tasks/main.yml b/roles/deployment/cluster/tasks/main.yml index a75f2412..87d4d41e 100644 --- a/roles/deployment/cluster/tasks/main.yml +++ b/roles/deployment/cluster/tasks/main.yml @@ -13,6 +13,10 @@ # limitations under the License. --- +- name: Include config cluster defaults for deployment + ansible.builtin.include_role: + name: cloudera.cluster.config.cluster.common + public: yes - name: Apply "all hosts" configs include_role: @@ -24,7 +28,7 @@ when: definition.hosts.configs is defined - name: Detect Cloudera Manager version - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/version register: cm_version_response @@ -32,13 +36,14 @@ cloudera_manager_version: "{{ cm_version_response.json.version }}" - name: Find existing clusters - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters?clusterType=any register: clusters_response - set_fact: existing_clusters: "{{ clusters_response.json | json_query('items[*].name') }}" +# If you get failures here check in CM to ensure you don't have empty clusters or other oddities - name: Create base clusters include_tasks: create_base.yml loop: "{{ definition.clusters }}" @@ -47,7 +52,7 @@ vars: cluster: "{{ default_cluster_base | combine(_cluster) }}" when: - - cluster.type | default('base') == 'base' + - cluster.type | default(default_cluster_type) == 'base' - cluster.name not in existing_clusters - name: Create base cluster data contexts (SDX) @@ -58,7 +63,7 @@ vars: cluster: "{{ default_cluster_base | combine(_cluster) }}" when: - - cluster.type | default('base') == 'base' + - cluster.type | default(default_cluster_type) == 'base' - cloudera_manager_version is version('6.2.0','>=') - name: Create Key Trustee server cluster @@ -69,7 +74,7 @@ vars: cluster: "{{ default_cluster_kts | combine(_cluster) }}" when: - - cluster.type | default('base') == 'kts' + - cluster.type | default(default_cluster_type) == 'kts' - cluster.name not in existing_clusters - '"kts_active" in groups' @@ -81,10 +86,10 @@ vars: cluster: "{{ default_cluster_compute | combine(_cluster) }}" when: - - cluster.type | default('base') == 'compute' + - cluster.type | default(default_cluster_type) == 'compute' - cluster.name not in existing_clusters - name: Restart Cloudera Management Service - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service/commands/restart method: POST diff --git a/roles/deployment/cluster/templates/cluster_template/cm_api.j2 b/roles/deployment/cluster/templates/cluster_template/cm_api.j2 new file mode 100644 index 00000000..88ab1447 --- /dev/null +++ b/roles/deployment/cluster/templates/cluster_template/cm_api.j2 @@ -0,0 +1,90 @@ +{% macro ApiConfig(k, v, force_uppercase_keys=False) -%} + {%- set config = {} -%} + {%- if force_uppercase_keys -%} + {{ config.update({"name": k | upper}) }} + {%- else -%} + {{ config.update({"name": k}) }} + {%- endif -%} + {# the extra to_json filter renders a dict into properly escaped JSON string for the CM API call #} + {%- if v is mapping -%} + {{ config.update({"value": v | to_json}) }} + {%- else -%} + {{ config.update({"value": v}) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiConfigList(configs, force_uppercase_keys=False) -%} + {%- set config_list = { "items": [] } -%} + {%- if configs is mapping -%} + {%- for (k, v) in configs.items() -%} + {%- set config = ApiConfig(k, v, force_uppercase_keys) -%} + {{ config_list["items"].append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfig(key, service_type, role_type, value, value_type="value") -%} + {%- set config = {} -%} + {{ config.update({"name": key}) }} + {%- if value_type == "ref" or value_type == "value" -%} + {{ config.update({ value_type: value }) }} + {%- else -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {{ config.update({ "variable": var_name | upper }) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfigList(service_merged_configs, service_type, role_type) -%} + {%- set config_list = [] -%} + {%- if service_type in service_merged_configs -%} + {%- if service_merged_configs[service_type][role_type] is mapping -%} + {%- for (k, v) in service_merged_configs[service_type][role_type].items() -%} + {%- if k.endswith("_service") and not "suppression" in k -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "ref") -%} + {%- else -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%} + {%- endif -%} + {{ config_list.append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiService(service) -%} +{} +{%- endmacro %} + +{% macro ApiServiceList(services) -%} + {%- set api_service_list = { "items": [] } -%} + {%- if services -%} + {%- for service in services -%} + {%- set api_service = ApiService(service) -%} + {{ api_service_list["items"].append(api_service | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ api_service_list | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMapping(auth_type, user_mapping) %} + {%- set external_user_mapping = {'name': user_mapping.group, 'type': auth_type, 'authRoles': []} -%} + {%- for role in user_mapping.roles -%} + {{ external_user_mapping['authRoles'].append(ApiAuthRoleRef(role) | from_json) }} + {%- endfor -%} + {{ external_user_mapping | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMappingList(auth_type, auth_user_mappings) %} + {%- set external_user_mapping_list = { "items": [] } -%} + {%- for user_mapping in auth_user_mappings -%} + {{ external_user_mapping_list["items"].append(ApiExternalUserMapping(auth_type, user_mapping) | from_json) }} + {%- endfor -%} + {{ external_user_mapping_list | to_json }} +{%- endmacro %} + +{% macro ApiAuthRoleRef(name) -%} + { "uuid": {{ auth_role_uuids[name] | to_json }} } +{%- endmacro %} diff --git a/roles/deployment/cluster/templates/cluster_template/main.j2 b/roles/deployment/cluster/templates/cluster_template/main.j2 index b05fc2ed..5a2f0e87 100644 --- a/roles/deployment/cluster/templates/cluster_template/main.j2 +++ b/roles/deployment/cluster/templates/cluster_template/main.j2 @@ -1,3 +1,4 @@ +{% import 'cm_api.j2' as cm_api with context %} { {% set cluster_type = cluster.type | default('base') %} {% set template_dir = (cluster_type == 'kts') | ternary('kts', 'base') %} diff --git a/roles/prereqs/kerberos/defaults/main.yml b/roles/deployment/credential/meta/main.yml similarity index 91% rename from roles/prereqs/kerberos/defaults/main.yml rename to roles/deployment/credential/meta/main.yml index f81e1b91..f46e2d5c 100644 --- a/roles/prereqs/kerberos/defaults/main.yml +++ b/roles/deployment/credential/meta/main.yml @@ -13,4 +13,5 @@ # limitations under the License. --- -krb5_kdc_type: mit +dependencies: + - role: cloudera.cluster.prereqs.license diff --git a/roles/deployment/credential/tasks/main.yml b/roles/deployment/credential/tasks/main.yml new file mode 100644 index 00000000..9c0b6817 --- /dev/null +++ b/roles/deployment/credential/tasks/main.yml @@ -0,0 +1,39 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Derive License Content + when: cloudera_manager_license_file + block: + - name: Read in License File + no_log: true + ansible.builtin.set_fact: + cloudera_manager_license: "{{ lookup('file', cloudera_manager_license_file ) }}" + + - name: Extract Username and License Name from Cloudera License Content + no_log: true + ansible.builtin.set_fact: + cloudera_manager_repo_username: "{{ cloudera_manager_license | regex_replace('(.|\n)*\"uuid\"\\s*:\\s*\"([^\"]*)\"(.|\n)*', '\\2') }}" + + - name: Generate Cloudera Repo Password from License Content + shell: "printf '{{ ''.join([__licname, cloudera_manager_repo_username]) }}' | openssl dgst -sha256 -hex | egrep -o '[a-f0-9]{12}' | head -1" + register: __repo_pword + no_log: true + vars: + __licname: "{{ cloudera_manager_license | regex_replace('(.|\n)*\"name\"\\s*:\\s*\"([^\"]*)\"(.|\n)*', '\\2') }}" + + - name: Set password from processed Cloudera License Content + no_log: true + ansible.builtin.set_fact: + cloudera_manager_repo_password: "{{ __repo_pword.stdout }}" diff --git a/roles/deployment/databases/meta/main.yml b/roles/deployment/databases/meta/main.yml index ba00be71..f63fb6e8 100644 --- a/roles/deployment/databases/meta/main.yml +++ b/roles/deployment/databases/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: deployment/definition + - role: cloudera.cluster.deployment.definition diff --git a/roles/deployment/databases/tasks/mariadb.yml b/roles/deployment/databases/tasks/mariadb.yml index 214ef67a..3067f963 100644 --- a/roles/deployment/databases/tasks/mariadb.yml +++ b/roles/deployment/databases/tasks/mariadb.yml @@ -17,8 +17,8 @@ - name: Create databases mysql_db: name: "{{ databases[service].name }}" - encoding: "{{ service | get_database_encoding_mysql }}" - collation: "{{ service | get_database_collation_mysql }}" + encoding: "{{ service | cloudera.cluster.get_database_encoding_mysql }}" + collation: "{{ service | cloudera.cluster.get_database_collation_mysql }}" become: yes loop: "{{ databases | intersect(services) }}" loop_control: diff --git a/roles/deployment/definition/vars/main.yml b/roles/deployment/definition/defaults/main.yml similarity index 67% rename from roles/deployment/definition/vars/main.yml rename to roles/deployment/definition/defaults/main.yml index d3cd43fd..ff54640f 100644 --- a/roles/deployment/definition/vars/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -13,88 +13,101 @@ # limitations under the License. --- +database_host: "{{ groups['db_server'][0] | default('localhost') }}" +database_type: postgresql database_default_password: changeme +database_version: 10 + +krb5_realm: CLOUDERA.LOCAL +krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" +krb5_kdc_admin_password: changeme +krb5_kdc_type: MIT KDC +krb5_enc_types: "aes256-cts aes128-cts" + +manual_tls_cert_distribution: false +local_temp_dir: '/tmp' + database_defaults: ACTIVITYMONITOR: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: amon user: amon password: "{{ database_default_password }}" DAS: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: das user: das password: "{{ database_default_password }}" HIVE: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: metastore user: hive password: "{{ database_default_password }}" HUE: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: hue user: hue password: "{{ database_default_password }}" NAVIGATOR: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: nav user: nav password: "{{ database_default_password }}" NAVIGATORMETASERVER: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: navms user: navms password: "{{ database_default_password }}" OOZIE: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: oozie user: oozie password: "{{ database_default_password }}" RANGER: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: ranger user: rangeradmin password: "{{ database_default_password }}" REPORTSMANAGER: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: rman user: rman password: "{{ database_default_password }}" SCHEMAREGISTRY: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: schemaregistry user: schemaregistry password: "{{ database_default_password }}" STREAMS_MESSAGING_MANAGER: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: streamsmsgmgr user: streamsmsgmgr password: "{{ database_default_password }}" SENTRY: host: "{{ database_host }}" - port: "{{ database_type | default_database_port }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" type: "{{ database_type }}" name: sentry user: sentry diff --git a/roles/deployment/definition/filter_plugins/filters.py b/roles/deployment/definition/filter_plugins/filters.py deleted file mode 100644 index c5c035ba..00000000 --- a/roles/deployment/definition/filter_plugins/filters.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/python -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import json -import re - -class FilterModule(object): - - def filters(self): - return { - 'extract_products_from_manifests': self.extract_products_from_manifests, - 'format_database_type': self.format_database_type, - 'get_product_version': self.get_product_version, - 'get_major_version': self.get_major_version, - 'append_database_port': self.append_database_port, - 'default_database_port': self.default_database_port, - 'get_database_encoding_mysql': self.get_database_encoding_mysql, - 'get_database_collation_mysql': self.get_database_collation_mysql, - 'cluster_service_role_hosts': self.cluster_service_role_hosts, - 'find_clusters': self.find_clusters - } - - - def extract_products_from_manifests(self, manifests): - products = dict() - for manifest in manifests: - for parcel in manifest['parcels']: - # take first parcel, strip off OS name and file extension - parcel_name = re.sub("-[a-z0-9]+\.parcel$", "", str(parcel['parcelName'])) - # the product name is before the first dash - product = parcel_name[:parcel_name.index("-")] - if product not in products: - # the version string is everything after the first dash - version = parcel_name[parcel_name.index("-")+1:] - products[product] = version - return products - - - def format_database_type(self, database_type): - if database_type == "mariadb": - return "mysql" - return database_type.lower() - - - def get_product_version(self, products, product_name): - for product in products: - if product['product'] == product_name: - version = product['version'] - return version[:version.index('-')] if "-" in version else version - - - def get_major_version(self, products, product_name): - version = self.get_product_version(products, product_name) - if version: - return version.split('.')[0] - - - def append_database_port(self, database_host, database_port=None): - if ":" not in database_host and database_port: - return database_host + ":" + database_port - return database_host - - - def default_database_port(self, database_type): - if database_type == "postgresql": - return 5432 - if database_type == "mysql" or database_type == "mariadb": - return 3306 - if database_type == "oracle": - return 1521 - return None - - - def get_database_encoding_mysql(self, service_name): - # workaround for https://jira.cloudera.com/browse/CDPD-9290 - if service_name == "RANGER": - database_encoding = "latin1" - else: - database_encoding = "utf8" - return database_encoding - - - def get_database_collation_mysql(self, service_name): - # workaround for https://jira.cloudera.com/browse/CDPD-9290 - if service_name == "RANGER": - database_collation = "latin1_swedish_ci" - else: - database_collation = "utf8_general_ci" - return database_collation - - - def cluster_service_role_hosts(self, cluster, hostvars, service, roles=None): - candidate_templates = [] - - if 'host_templates' in cluster: - templates = cluster['host_templates'] - - if roles: - for role in roles: - for t_name, t_services in templates.items(): - if service in t_services and role in t_services[service]: - if t_name not in candidate_templates: - candidate_templates.append(t_name) - - else: - for t_name, t_services in templates.items(): - if service in t_services: - candidate_templates.append(t_name) - - hosts = [] - for t_name in candidate_templates: - t_hosts = [ - host - for host, hostvar in hostvars.items() - if host not in hosts - if hostvar.get('host_template') == t_name] - - hosts = hosts + t_hosts - - return hosts - - - def find_clusters(self, clusters, name): - return [ - cluster - for cluster in clusters - if cluster.get('name') == name] diff --git a/roles/deployment/definition/tasks/main.yml b/roles/deployment/definition/tasks/main.yml index 59d5788e..eb49f216 100644 --- a/roles/deployment/definition/tasks/main.yml +++ b/roles/deployment/definition/tasks/main.yml @@ -14,20 +14,19 @@ --- -- name: Load cluster definition - include_vars: - file: "{{ cluster_definition }}/cluster.yml" - name: _definition - -- set_fact: +- name: Generate host_template cluster map + ansible.builtin.set_fact: _host_template_cluster_map: "{{ lookup('template', './template_cluster_map.j2') | from_yaml }}" vars: - clusters: "{{ _definition.clusters | default([]) }}" + clusters: "{{ clusters | default([]) }}" -- set_fact: - definition: - clusters: "{{ _definition.clusters | default({}) }}" - mgmt: "{{ _definition.mgmt | default({}) }}" - hosts: "{{ _definition.hosts | default({}) }}" - host_template_cluster_map: "{{ _host_template_cluster_map }}" - when: definition is not defined +- name: Generate definition canonical structure from supplied Definition details + ansible.builtin.set_fact: + definition: "{{ definition | default({}) | combine(__def_item, recursive=True) }}" + loop_control: + loop_var: __def_item + loop: + - clusters: "{{ clusters | default([]) }}" + - mgmt: "{{ mgmt | default({}) }}" + - hosts: "{{ hosts | default({}) }}" + - host_template_cluster_map: "{{ _host_template_cluster_map }}" diff --git a/tasks/group_hosts.yml b/roles/deployment/groupby/tasks/main.yml similarity index 80% rename from tasks/group_hosts.yml rename to roles/deployment/groupby/tasks/main.yml index fcc8c4ae..3e48ad0d 100644 --- a/tasks/group_hosts.yml +++ b/roles/deployment/groupby/tasks/main.yml @@ -18,11 +18,6 @@ group_by: key: "{{ 'host_template_' ~ host_template if host_template is defined else 'no_template' }}" -- name: Initial pass of cluster definition - set_fact: - _pre_template_cluster: "{{ lookup('file', cluster_definition ~ '/cluster.yml') | from_yaml }}" - run_once: true - - name: Find the correct host template set_fact: host_template_content: "{{ _pre_template_cluster | json_query(query) | first }}" @@ -33,7 +28,7 @@ - name: Group by service group_by: key: "{{ service | lower }}" - loop: "{{ host_template_content | flatten_dict_list(1) }}" + loop: "{{ host_template_content | cloudera.cluster.flatten_dict_list(1) }}" loop_control: loop_var: service when: host_template is defined @@ -41,12 +36,17 @@ - name: Group by service role group_by: key: "{{ role | lower }}" - loop: "{{ host_template_content | flatten_dict_list(2) }}" + loop: "{{ host_template_content | cloudera.cluster.flatten_dict_list(2) }}" loop_control: loop_var: role when: host_template is defined - name: (debug) Show derived groups + delegate_to: localhost debug: var: groups verbosity: 3 + +- name: Group hosts based on whether TLS flag is set in inventory + group_by: + key: "{{ 'tls' if tls | default(False) else 'no_tls' }}" diff --git a/tasks/group_hosts_by_tls.yml b/roles/deployment/repometa/meta/main.yml similarity index 80% rename from tasks/group_hosts_by_tls.yml rename to roles/deployment/repometa/meta/main.yml index d2e315e7..5925f2d6 100644 --- a/tasks/group_hosts_by_tls.yml +++ b/roles/deployment/repometa/meta/main.yml @@ -13,7 +13,5 @@ # limitations under the License. --- - -- name: Group hosts based on whether TLS flag is set in inventory - group_by: - key: "{{ 'tls' if tls | default(False) else 'no_tls' }}" +dependencies: + - role: cloudera.cluster.deployment.credential diff --git a/roles/deployment/repometa/tasks/parcels.yml b/roles/deployment/repometa/tasks/parcels.yml index 0665fc1a..a02fcd99 100644 --- a/roles/deployment/repometa/tasks/parcels.yml +++ b/roles/deployment/repometa/tasks/parcels.yml @@ -15,6 +15,7 @@ --- - name: Download parcel manifest information + delegate_to: "{{ groups.cloudera_manager[0] if 'cloudera_manager' in groups else 'localhost' }}" uri: url: "{{ repository | regex_replace('/?$','') + '/manifest.json' }}" status_code: 200 @@ -35,12 +36,12 @@ {{ manifests.results | map(attribute='json') | list - | extract_products_from_manifests + | cloudera.cluster.extract_products_from_manifests | dict2items(key_name='product', value_name='version') }} run_once: true - name: Extract runtime version from products list set_fact: - cloudera_runtime_version: "{{ products | get_product_version('CDH') }}" + cloudera_runtime_version: "{{ products | cloudera.cluster.get_product_version('CDH') }}" run_once: true diff --git a/roles/deployment/services/kms/meta/main.yml b/roles/deployment/services/kms/meta/main.yml index 5f99a9d6..20ea4909 100644 --- a/roles/deployment/services/kms/meta/main.yml +++ b/roles/deployment/services/kms/meta/main.yml @@ -13,6 +13,8 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_hosts - - role: deployment/definition + - role: cloudera.cluster.cloudera_manager.api_hosts + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.deployment.services.kts_common diff --git a/roles/deployment/services/kms/tasks/create_kms.yml b/roles/deployment/services/kms/tasks/create_kms.yml index 9b61bd27..e6b07f80 100644 --- a/roles/deployment/services/kms/tasks/create_kms.yml +++ b/roles/deployment/services/kms/tasks/create_kms.yml @@ -15,8 +15,8 @@ --- - name: Get cluster parcel details - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/parcels + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/parcels register: parcels_response - name: Extract active parcels @@ -62,8 +62,8 @@ kms_service_name: "{{ kms_service_type | lower }}" - name: Get existing services - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/services method: GET register: services @@ -78,15 +78,15 @@ name: config/services/kms - name: Create KMS service - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/services method: POST body: "{{ lookup('template', 'service.j2', convert_data=False) }}" when: not existing_kms_service - name: Add KMS dependency to HDFS - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/hdfs/config + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/services/hdfs/config method: PUT body: items: @@ -94,8 +94,8 @@ value: "{{ kms_service_name }}" - name: Wait for parcels to become active on any new hosts - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/parcels/products/{{ item.product }}/versions/{{ item.version }} + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/parcels/products/{{ item.product }}/versions/{{ item.version }} register: parcels_response until: parcels_response.json.stage == "ACTIVATED" with_items: "{{ active_parcels }}" @@ -103,16 +103,16 @@ delay: "{{ kms_parcel_poll_duration | default(20) }}" - name: Find details of KMS roles - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/{{ kms_service_name }}/roles + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/services/{{ kms_service_name }}/roles register: roles - set_fact: primary_kms_role_query: items[?hostRef.hostname==`{{ cloudera_manager_api_hosts[groups.kms_servers[0]]['hostname'] }}`].name - name: Start first KMS server instance - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/{{ kms_service_name }}/roleCommands/start + cloudera.cluster.cm_api: + endpoint: /clusters/{{ __kms_cluster.name | urlencode() }}/services/{{ kms_service_name }}/roleCommands/start method: POST body: items: "{{ roles.json | json_query(primary_kms_role_query) }}" diff --git a/roles/deployment/services/kms/tasks/main.yml b/roles/deployment/services/kms/tasks/main.yml index 1f11c4af..2a10dfef 100644 --- a/roles/deployment/services/kms/tasks/main.yml +++ b/roles/deployment/services/kms/tasks/main.yml @@ -14,13 +14,6 @@ --- -- set_fact: - keytrustee_server_conf_dir: "{{ hostvars[groups.kts_active[0]]['keytrustee_server_conf_dir'] }}" - keytrustee_server_org_name: "{{ hostvars[groups.kts_active[0]]['keytrustee_server_org_name'] }}" - -- set_fact: - auth_secret_query: "{{ keytrustee_server_org_name }}.auth_secret" - - name: Get Key Trustee organisation auth secret shell: > keytrustee-orgtool --confdir {{ keytrustee_server_conf_dir }} list @@ -32,13 +25,15 @@ - name: Extract auth secret from keytrustee-orgtool output set_fact: keytrustee_server_auth_secret: "{{ orgtool_output.stdout | from_json | json_query(auth_secret_query) }}" + vars: + auth_secret_query: "{{ keytrustee_server_org_name }}.auth_secret" - name: Create KMS services include_tasks: create_kms.yml loop: "{{ definition.clusters }}" loop_control: - loop_var: cluster + loop_var: __kms_cluster when: - - cluster.type | default('base') == 'base' - - cluster.security.hdfs_encryption | default(false) + - __kms_cluster.type | default('base') == 'base' + - __kms_cluster.security.hdfs_encryption | default(False) - '"kms_servers" in groups' diff --git a/roles/deployment/services/kms/templates/cm_api.j2 b/roles/deployment/services/kms/templates/cm_api.j2 new file mode 100644 index 00000000..88ab1447 --- /dev/null +++ b/roles/deployment/services/kms/templates/cm_api.j2 @@ -0,0 +1,90 @@ +{% macro ApiConfig(k, v, force_uppercase_keys=False) -%} + {%- set config = {} -%} + {%- if force_uppercase_keys -%} + {{ config.update({"name": k | upper}) }} + {%- else -%} + {{ config.update({"name": k}) }} + {%- endif -%} + {# the extra to_json filter renders a dict into properly escaped JSON string for the CM API call #} + {%- if v is mapping -%} + {{ config.update({"value": v | to_json}) }} + {%- else -%} + {{ config.update({"value": v}) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiConfigList(configs, force_uppercase_keys=False) -%} + {%- set config_list = { "items": [] } -%} + {%- if configs is mapping -%} + {%- for (k, v) in configs.items() -%} + {%- set config = ApiConfig(k, v, force_uppercase_keys) -%} + {{ config_list["items"].append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfig(key, service_type, role_type, value, value_type="value") -%} + {%- set config = {} -%} + {{ config.update({"name": key}) }} + {%- if value_type == "ref" or value_type == "value" -%} + {{ config.update({ value_type: value }) }} + {%- else -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {{ config.update({ "variable": var_name | upper }) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfigList(service_merged_configs, service_type, role_type) -%} + {%- set config_list = [] -%} + {%- if service_type in service_merged_configs -%} + {%- if service_merged_configs[service_type][role_type] is mapping -%} + {%- for (k, v) in service_merged_configs[service_type][role_type].items() -%} + {%- if k.endswith("_service") and not "suppression" in k -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "ref") -%} + {%- else -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%} + {%- endif -%} + {{ config_list.append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiService(service) -%} +{} +{%- endmacro %} + +{% macro ApiServiceList(services) -%} + {%- set api_service_list = { "items": [] } -%} + {%- if services -%} + {%- for service in services -%} + {%- set api_service = ApiService(service) -%} + {{ api_service_list["items"].append(api_service | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ api_service_list | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMapping(auth_type, user_mapping) %} + {%- set external_user_mapping = {'name': user_mapping.group, 'type': auth_type, 'authRoles': []} -%} + {%- for role in user_mapping.roles -%} + {{ external_user_mapping['authRoles'].append(ApiAuthRoleRef(role) | from_json) }} + {%- endfor -%} + {{ external_user_mapping | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMappingList(auth_type, auth_user_mappings) %} + {%- set external_user_mapping_list = { "items": [] } -%} + {%- for user_mapping in auth_user_mappings -%} + {{ external_user_mapping_list["items"].append(ApiExternalUserMapping(auth_type, user_mapping) | from_json) }} + {%- endfor -%} + {{ external_user_mapping_list | to_json }} +{%- endmacro %} + +{% macro ApiAuthRoleRef(name) -%} + { "uuid": {{ auth_role_uuids[name] | to_json }} } +{%- endmacro %} diff --git a/group_vars/kms_servers.yml b/roles/deployment/services/kms_ha/defaults/main.yml similarity index 95% rename from group_vars/kms_servers.yml rename to roles/deployment/services/kms_ha/defaults/main.yml index 846b37e8..8907c8c7 100644 --- a/group_vars/kms_servers.yml +++ b/roles/deployment/services/kms_ha/defaults/main.yml @@ -13,10 +13,11 @@ # limitations under the License. --- +local_temp_dir: /tmp kms_conf_dir: /var/lib/kms-keytrustee/keytrustee/.keytrustee kms_key_files: - keytrustee.conf - pubring.gpg - secring.gpg kms_user: kms -kms_group: kms +kms_group: kms \ No newline at end of file diff --git a/roles/deployment/services/kms_ha/tasks/main.yml b/roles/deployment/services/kms_ha/tasks/main.yml new file mode 100644 index 00000000..d35916a8 --- /dev/null +++ b/roles/deployment/services/kms_ha/tasks/main.yml @@ -0,0 +1,48 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Create temp directory for keys + file: + path: "{{ local_temp_dir }}/kms" + owner: root + group: root + state: directory + mode: 0777 + +- name: Fetch keys from first KMS server + delegate_to: "{{ groups.kms_servers | first }}" + ansible.builtin.fetch: + src: "{{ kms_conf_dir }}/{{ __kms_key_file }}" + dest: "{{ local_temp_dir }}/kms" + loop: "{{ kms_key_files }}" + loop_control: + loop_var: __kms_key_file + +- name: Copy keys to other KMS servers + delegate_to: "{{ __kms_host }}" + ansible.builtin.copy: + src: "{{ local_temp_dir }}/kms/{{ inventory_hostname }}{{ kms_conf_dir }}/" + dest: "{{ kms_conf_dir }}" + owner: "{{ kms_user }}" + group: "{{ kms_group }}" + mode: 0600 + loop: "{{ groups.kms_servers[1:] }}" + loop_control: + loop_var: __kms_host + +- name: Delete temp directory + file: + path: "{{ local_temp_dir }}/kms" + state: absent diff --git a/group_vars/kts_servers.yml b/roles/deployment/services/kts_common/defaults/main.yml similarity index 92% rename from group_vars/kts_servers.yml rename to roles/deployment/services/kts_common/defaults/main.yml index 8aa3d046..22ef282b 100644 --- a/group_vars/kts_servers.yml +++ b/roles/deployment/services/kts_common/defaults/main.yml @@ -13,7 +13,9 @@ # limitations under the License. --- +local_temp_dir: /tmp keytrustee_server_conf_dir: /var/lib/keytrustee/.keytrustee +keytrustee_server_db_dir: /var/lib/keytrustee/db keytrustee_server_org_name: cloudera keytrustee_server_org_email: admin@example.com keytrustee_server_key_files: diff --git a/roles/deployment/services/kts_high_availability/meta/main.yml b/roles/deployment/services/kts_high_availability/meta/main.yml index 201f8987..4713af47 100644 --- a/roles/deployment/services/kts_high_availability/meta/main.yml +++ b/roles/deployment/services/kts_high_availability/meta/main.yml @@ -13,6 +13,8 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client - - role: deployment/definition + - role: cloudera.cluster.cloudera_manager.api_client + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.deployment.services.kts_common diff --git a/roles/deployment/services/kts_high_availability/tasks/main.yml b/roles/deployment/services/kts_high_availability/tasks/main.yml index c83e4eec..1ea6b663 100644 --- a/roles/deployment/services/kts_high_availability/tasks/main.yml +++ b/roles/deployment/services/kts_high_availability/tasks/main.yml @@ -13,28 +13,73 @@ # limitations under the License. --- +- name: Query for KTS Cluster + set_fact: + kts_cluster: "{{ definition.clusters | json_query('[?type==`kts`]') | first }}" -- set_fact: - cluster: "{{ definition.clusters | json_query('[?type==`kts`]') | first }}" +- name: Check Cluster and Inventory suitable for KTS + ansible.builtin.assert: + that: + - kts_cluster.name is defined + - groups.kts_passive is defined + - groups.kts_active is defined + fail_msg: "Cluster type kts not present or expected kts_active and kts_passive not present" -- fail: - msg: No Key Trustee Server cluster was found - when: cluster.name is not defined +- name: Create temp directory for keys + file: + path: "{{ local_temp_dir }}/kts" + owner: root + group: root + state: directory + mode: 0777 + +- name: Fetch GPG keys and configs from active Key Trustee Server + delegate_to: "{{ groups.kts_active | first }}" + fetch: + src: "{{ keytrustee_server_conf_dir }}/{{ item }}" + dest: "{{ local_temp_dir }}/kts" + loop: "{{ keytrustee_server_key_files }}" + +- name: Copy to passive Key Trustee Server + delegate_to: "{{ groups.kts_passive | first }}" + copy: + src: "{{ local_temp_dir }}/kts/{{ inventory_hostname }}{{ keytrustee_server_conf_dir }}/" + dest: "{{ keytrustee_server_conf_dir }}" + owner: keytrustee + group: keytrustee + mode: 0600 + +- name: Delete temp directory + file: + path: "{{ local_temp_dir }}/kts" + state: absent + +- name: Initialize passive Key Trustee Server + delegate_to: "{{ groups.kts_passive | first }}" + shell: ktadmin init --confdir {{ keytrustee_server_conf_dir }} - name: Start Key Trustee Server service - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/keytrustee_server/commands/restart + cloudera.cluster.cm_api: + endpoint: /clusters/{{ kts_cluster.name | urlencode() }}/services/keytrustee_server/commands/restart method: POST - when: cluster.name is defined + when: kts_cluster.name is defined - name: Setup Key Trustee Server database replication - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/keytrustee_server/commands/SetupSyncReplicationCommand + cloudera.cluster.cm_api: + endpoint: /clusters/{{ kts_cluster.name | urlencode() }}/services/keytrustee_server/commands/SetupSyncReplicationCommand method: POST - when: cluster.name is defined + when: kts_cluster.name is defined - name: Restart Key Trustee Server service - cm_api: - endpoint: /clusters/{{ cluster.name | urlencode() }}/services/keytrustee_server/commands/restart + cloudera.cluster.cm_api: + endpoint: /clusters/{{ kts_cluster.name | urlencode() }}/services/keytrustee_server/commands/restart method: POST - when: cluster.name is defined + when: kts_cluster.name is defined + +- name: Create Key Trustee Server organisation + delegate_to: "{{ groups.kts_active | first }}" + shell: > + keytrustee-orgtool add + -n {{ keytrustee_server_org_name }} + -c {{ keytrustee_server_org_email }} + --confdir {{ keytrustee_server_conf_dir }} \ No newline at end of file diff --git a/roles/deployment/services/mgmt/meta/main.yml b/roles/deployment/services/mgmt/meta/main.yml index 877764a9..0e64efd4 100644 --- a/roles/deployment/services/mgmt/meta/main.yml +++ b/roles/deployment/services/mgmt/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_hosts + - role: cloudera.cluster.cloudera_manager.api_hosts diff --git a/roles/deployment/services/mgmt/tasks/main.yml b/roles/deployment/services/mgmt/tasks/main.yml index 4d6e6017..e81e8000 100644 --- a/roles/deployment/services/mgmt/tasks/main.yml +++ b/roles/deployment/services/mgmt/tasks/main.yml @@ -16,21 +16,21 @@ - name: Generate mgmt configs include_role: - name: config/services/mgmt + name: cloudera.cluster.config.services.mgmt - name: Create databases and users include_role: - name: deployment/databases + name: cloudera.cluster.deployment.databases vars: services: "{{ definition.mgmt.services }}" - name: Define target host ID for Cloudera Management Service installation set_fact: - mgmt_service_api_host_id: "{{ cloudera_manager_api_hosts[cloudera_manager_host]['id'] }}" + mgmt_service_api_host_id: "{{ cloudera_manager_api_hosts[cloudera_manager_host_local]['id'] }}" when: mgmt_service_api_host_id is not defined - name: Add Cloudera Management Service - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service method: PUT body: "{{ lookup('template', 'service.j2', convert_data=False) }}" @@ -40,7 +40,7 @@ - "'CMS instance already exists' not in api_cm_service_response.content" - name: Start Cloudera Management Service - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service/commands/restart method: POST register: api_cm_service_start diff --git a/roles/deployment/services/mgmt/templates/cm_api.j2 b/roles/deployment/services/mgmt/templates/cm_api.j2 new file mode 100644 index 00000000..88ab1447 --- /dev/null +++ b/roles/deployment/services/mgmt/templates/cm_api.j2 @@ -0,0 +1,90 @@ +{% macro ApiConfig(k, v, force_uppercase_keys=False) -%} + {%- set config = {} -%} + {%- if force_uppercase_keys -%} + {{ config.update({"name": k | upper}) }} + {%- else -%} + {{ config.update({"name": k}) }} + {%- endif -%} + {# the extra to_json filter renders a dict into properly escaped JSON string for the CM API call #} + {%- if v is mapping -%} + {{ config.update({"value": v | to_json}) }} + {%- else -%} + {{ config.update({"value": v}) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiConfigList(configs, force_uppercase_keys=False) -%} + {%- set config_list = { "items": [] } -%} + {%- if configs is mapping -%} + {%- for (k, v) in configs.items() -%} + {%- set config = ApiConfig(k, v, force_uppercase_keys) -%} + {{ config_list["items"].append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfig(key, service_type, role_type, value, value_type="value") -%} + {%- set config = {} -%} + {{ config.update({"name": key}) }} + {%- if value_type == "ref" or value_type == "value" -%} + {{ config.update({ value_type: value }) }} + {%- else -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {{ config.update({ "variable": var_name | upper }) }} + {%- endif -%} + {{ config | to_json }} +{%- endmacro %} + +{% macro ApiClusterTemplateConfigList(service_merged_configs, service_type, role_type) -%} + {%- set config_list = [] -%} + {%- if service_type in service_merged_configs -%} + {%- if service_merged_configs[service_type][role_type] is mapping -%} + {%- for (k, v) in service_merged_configs[service_type][role_type].items() -%} + {%- if k.endswith("_service") and not "suppression" in k -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "ref") -%} + {%- else -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%} + {%- endif -%} + {{ config_list.append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + +{% macro ApiService(service) -%} +{} +{%- endmacro %} + +{% macro ApiServiceList(services) -%} + {%- set api_service_list = { "items": [] } -%} + {%- if services -%} + {%- for service in services -%} + {%- set api_service = ApiService(service) -%} + {{ api_service_list["items"].append(api_service | from_json) }} + {%- endfor -%} + {%- endif -%} + {{ api_service_list | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMapping(auth_type, user_mapping) %} + {%- set external_user_mapping = {'name': user_mapping.group, 'type': auth_type, 'authRoles': []} -%} + {%- for role in user_mapping.roles -%} + {{ external_user_mapping['authRoles'].append(ApiAuthRoleRef(role) | from_json) }} + {%- endfor -%} + {{ external_user_mapping | to_json }} +{%- endmacro %} + +{% macro ApiExternalUserMappingList(auth_type, auth_user_mappings) %} + {%- set external_user_mapping_list = { "items": [] } -%} + {%- for user_mapping in auth_user_mappings -%} + {{ external_user_mapping_list["items"].append(ApiExternalUserMapping(auth_type, user_mapping) | from_json) }} + {%- endfor -%} + {{ external_user_mapping_list | to_json }} +{%- endmacro %} + +{% macro ApiAuthRoleRef(name) -%} + { "uuid": {{ auth_role_uuids[name] | to_json }} } +{%- endmacro %} diff --git a/roles/infrastructure/ca_certs/meta/main.yml b/roles/infrastructure/ca_certs/meta/main.yml new file mode 100644 index 00000000..830dd2a1 --- /dev/null +++ b/roles/infrastructure/ca_certs/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file diff --git a/roles/infrastructure/ca_certs/tasks/clean.yml b/roles/infrastructure/ca_certs/tasks/clean.yml new file mode 100644 index 00000000..bc16179e --- /dev/null +++ b/roles/infrastructure/ca_certs/tasks/clean.yml @@ -0,0 +1,19 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Clean CA Certs directory + file: + name: "{{ ca_server_root_path }}" + state: absent \ No newline at end of file diff --git a/refresh_ranger_kms_repo.yml b/roles/infrastructure/ca_certs/tasks/fetch.yml similarity index 61% rename from refresh_ranger_kms_repo.yml rename to roles/infrastructure/ca_certs/tasks/fetch.yml index 7013367a..c5dfd8db 100644 --- a/refresh_ranger_kms_repo.yml +++ b/roles/infrastructure/ca_certs/tasks/fetch.yml @@ -13,18 +13,15 @@ # limitations under the License. --- - -- name: Group hosts using host template information - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts.yml - tags: - - always - -- name: Refresh the KMS Ranger service - hosts: localhost - connection: local - gather_facts: no - roles: - - role: operations/refresh_ranger_kms_repo +- name: Fetch CA Certs + fetch: + src: "{{ cert.src }}" + dest: "{{ cert.dest }}" + flat: yes + loop: + - src: "{{ ca_server_root_cert_path }}" + dest: "{{ local_temp_dir }}/certs/cluster_rootca.pem" + - src: "{{ ca_server_intermediate_cert_path }}" + dest: "{{ local_temp_dir }}/certs/cluster_intca.pem" + loop_control: + loop_var: cert diff --git a/vars/ca_server.yml b/roles/infrastructure/ca_common/defaults/main.yml similarity index 61% rename from vars/ca_server.yml rename to roles/infrastructure/ca_common/defaults/main.yml index 5a0ab9e0..0a92165d 100644 --- a/vars/ca_server.yml +++ b/roles/infrastructure/ca_common/defaults/main.yml @@ -39,3 +39,30 @@ ca_server_intermediate_cert_path: "{{ ca_server_intermediate_path_certs }}/{{ ca ca_server_chain_cert_name: ca-chain.cert.pem ca_server_chain_cert_path: "{{ ca_server_intermediate_path_certs }}/{{ ca_server_chain_cert_name }}" + +base_dir_security: /opt/cloudera/security +base_dir_security_pki: "{{ base_dir_security }}/pki" + +tls_cert_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.pem" +tls_cert_path_generic: "{{ base_dir_security_pki }}/host.pem" +tls_chain_path: "{{ base_dir_security_pki }}/chain.pem" +tls_csr_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.csr" +tls_csr_config_path: "{{ base_dir_security_pki }}/csr.cnf" +tls_key_password: changeme +tls_key_password_file: "{{ base_dir_security_pki }}/host.key.pw" +tls_key_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.key" +tls_key_path_generic: "{{ base_dir_security_pki }}/host.key" +tls_key_path_plaintext: "{{ tls_key_path }}.unenc" +tls_key_path_plaintext_generic: "{{ tls_key_path_generic }}.unenc" +tls_keystore_password: changeme +tls_keystore_path: "{{ base_dir_security_pki }}/{{ inventory_hostname }}.jks" +tls_keystore_path_generic: "{{ base_dir_security_pki }}/host.jks" +tls_truststore_password: changeme +tls_truststore_path: "{{ base_dir_security_pki }}/truststore.jks" +tls_uber_truststore_path: "{{ base_dir_security_pki }}/full_truststore.jks" + +ca_server_attrs_general: + OU: PS + O: Cloudera, Inc. + ST: CA + C: US \ No newline at end of file diff --git a/roles/infrastructure/ca_common/meta/main.yml b/roles/infrastructure/ca_common/meta/main.yml new file mode 100644 index 00000000..5b0fc8f5 --- /dev/null +++ b/roles/infrastructure/ca_common/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.deployment.definition \ No newline at end of file diff --git a/roles/infrastructure/ca_server/meta/main.yml b/roles/infrastructure/ca_server/meta/main.yml new file mode 100644 index 00000000..830dd2a1 --- /dev/null +++ b/roles/infrastructure/ca_server/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file diff --git a/roles/infrastructure/ca_server/molecule/default/molecule.yml b/roles/infrastructure/ca_server/molecule/default/molecule.yml index e7e13a76..4851f803 100644 --- a/roles/infrastructure/ca_server/molecule/default/molecule.yml +++ b/roles/infrastructure/ca_server/molecule/default/molecule.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + dependency: name: galaxy driver: diff --git a/roles/infrastructure/ca_server/tasks/main.yml b/roles/infrastructure/ca_server/tasks/main.yml index 65d426c9..0fd32742 100644 --- a/roles/infrastructure/ca_server/tasks/main.yml +++ b/roles/infrastructure/ca_server/tasks/main.yml @@ -19,12 +19,14 @@ file: "{{ ansible_os_family }}.yml" - name: Install openssl - package: + ansible.builtin.package: + lock_timeout: 60 name: openssl state: present - name: Install PyOpenSSL - package: + ansible.builtin.package: + lock_timeout: 60 name: pyOpenSSL state: present when: ansible_os_family == "RedHat" diff --git a/roles/infrastructure/ca_server/vars/RedHat.yml b/roles/infrastructure/ca_server/vars/RedHat.yml index c01e3459..22079f89 100644 --- a/roles/infrastructure/ca_server/vars/RedHat.yml +++ b/roles/infrastructure/ca_server/vars/RedHat.yml @@ -13,4 +13,5 @@ # limitations under the License. --- + ca_server_root_key_cipher: aes256 diff --git a/roles/infrastructure/custom_repo/defaults/main.yml b/roles/infrastructure/custom_repo/defaults/main.yml index da548445..90bcb158 100644 --- a/roles/infrastructure/custom_repo/defaults/main.yml +++ b/roles/infrastructure/custom_repo/defaults/main.yml @@ -13,6 +13,11 @@ # limitations under the License. --- + local_temp_dir: /tmp repo_tar_local_dir: repo -repo_tar_files: [] +repo_tar_files: "{{ definition.repo_tar_files | default([]) }}" +keep_newer: yes + +cm_repo_tarball_url: "{{ definition.cm_repo_tarball_url | default('') }}" +custom_repo_rehost_files: "{{ definition.custom_repo_rehost_files | default([]) }}" diff --git a/roles/infrastructure/custom_repo/tasks/install_parcels_from_tars_on_controller.yml b/roles/infrastructure/custom_repo/tasks/install_parcels_from_tars_on_controller.yml new file mode 100644 index 00000000..0642fb07 --- /dev/null +++ b/roles/infrastructure/custom_repo/tasks/install_parcels_from_tars_on_controller.yml @@ -0,0 +1,52 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Work out temp and repo paths + set_fact: + temp_dir: "{{ local_temp_dir }}/{{ repo_tar_file | regex_replace('.tar.gz','') }}" + repo_dir: /var/www/html/{{ repo_tar_file | regex_replace("-([0-9\.]+)-parcels.tar.gz", "/\1/parcels") }} + +- name: Upload and extract parcel tar file + ansible.builtin.unarchive: + src: "{{ repo_tar_local_dir }}/{{ repo_tar_file }}" + dest: "{{ local_temp_dir }}" + keep_newer: "{{ keep_newer }}" + +- name: Find parcel files + find: + path: "{{ temp_dir }}" + patterns: '*.parcel,*.parcel.sha,*.parcel.sha1,.*.parcel.sha256,manifest.json' + recurse: yes + register: files + +- name: Create parcel repo directory + file: + path: "{{ repo_dir }}" + state: directory + mode: 0755 + +- name: Copy parcel files into correct location + copy: + src: "{{ item.path }}" + dest: "{{ repo_dir }}" + remote_src: yes + mode: 0644 + with_items: "{{ files.files }}" + +- name: Remove temp directory + file: + path: "{{ temp_dir }}" + state: absent diff --git a/roles/infrastructure/custom_repo/tasks/main.yml b/roles/infrastructure/custom_repo/tasks/main.yml index c7a37e0f..2c16df6f 100644 --- a/roles/infrastructure/custom_repo/tasks/main.yml +++ b/roles/infrastructure/custom_repo/tasks/main.yml @@ -19,12 +19,19 @@ file: "{{ ansible_os_family }}.yml" - name: Install {{ httpd_package }} - package: + ansible.builtin.package: + lock_timeout: 60 name: "{{ httpd_package }}" state: present -- name: Populate parcel repository contents - include_tasks: install_parcels.yml +# Runs before tars process in case you are rehosting tars +- name: Populate parcel repository contents from a Download Mirror + when: custom_repo_rehost_files + include_tasks: rehost_files_from_download.yml + +# Upload files directly from Ansible Controller to custom repo +- name: Populate parcel repository contents from tars on Ansible controller + include_tasks: install_parcels_from_tars_on_controller.yml loop: "{{ repo_tar_files }}" loop_control: loop_var: repo_tar_file diff --git a/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml b/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml new file mode 100644 index 00000000..f94dbbf8 --- /dev/null +++ b/roles/infrastructure/custom_repo/tasks/rehost_files_from_download.yml @@ -0,0 +1,78 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Populate custom_repo +- name: Ensure repo directories exist for file downloads to match required object structure + loop: "{{ custom_repo_rehost_files }}" + loop_control: + loop_var: __tmp_parcel_dir_item + ansible.builtin.file: + path: "/var/www/html/{{ __tmp_parcel_dir_item | urlsplit('path') | dirname }}" + state: directory + +- name: Request Async Download of files to tmp dir + register: __infra_download_parcels_results + loop: "{{ custom_repo_rehost_files }}" + loop_control: + loop_var: __parcel_download_item + async: 3600 + poll: 0 + ansible.builtin.get_url: + url: "{{ __parcel_download_item }}" + dest: "/var/www/html/{{ __parcel_download_item | urlsplit('path') }}" + +- name: Track async downloads to completion [ This may take a while if your files are very large or far away ] + loop: "{{ __infra_download_parcels_results.results }}" + loop_control: + loop_var: __download_async_item + register: __async_download_results + until: __async_download_results.finished is defined and __async_download_results.finished + delay: 15 + retries: 300 + async_status: + jid: "{{ __download_async_item.ansible_job_id }}" + failed_when: + - __download_async_item.failed == True + - __download_async_item.finished != 1 + +- name: Ensure paths for unpacked repo-as-tars + loop: "{{ custom_repo_rehost_files | select('search', 'tar.gz') | select('search', 'repo-as-tarball') | list }}" + loop_control: + loop_var: __tmp_tar_dir_item + ansible.builtin.file: + path: "/var/www/html{{ __tmp_tar_dir_item | urlsplit('path') | regex_replace('^(.+)repo.+-(.+)\\.tar\\.gz$', '\\1\\2' + '/yum/') }}" + state: directory + +- name: Unpack repo-as-tar archives + loop: "{{ custom_repo_rehost_files | select('search', 'tar.gz') | select('search', 'repo-as-tarball') | list }}" + loop_control: + loop_var: __tmp_unpack_item + ansible.builtin.unarchive: + extra_opts: [--strip-components=1] + remote_src: yes + src: "/var/www/html{{ __tmp_unpack_item | urlsplit('path') }}" + dest: "/var/www/html{{ __tmp_unpack_item | urlsplit('path') | regex_replace('^(.+)repo.+-(.+)\\.tar\\.gz$', '\\1\\2' + '/yum/') }}" + keep_newer: "{{ keep_newer }}" + +- name: Set Cloudera Manager Base Repo if included in rehosting list + when: "{{ custom_repo_rehost_files | select('search', 'tar.gz') | list | select('search', '/cm') | list }} | length > 0" + ansible.builtin.set_fact: + cloudera_archive_base_url: "http://{{ groups['custom_repo'] | first }}" + delegate_to: "{{ __play_host }}" + delegate_facts: true + loop: "{{ groups.cloudera_manager + groups.cluster }}" + loop_control: + loop_var: __play_host + label: __play_host diff --git a/roles/infrastructure/custom_repo/vars/RedHat.yml b/roles/infrastructure/custom_repo/vars/RedHat.yml index e3636fe1..3f6aa3e3 100644 --- a/roles/infrastructure/custom_repo/vars/RedHat.yml +++ b/roles/infrastructure/custom_repo/vars/RedHat.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + httpd_package: httpd httpd_service: httpd diff --git a/roles/infrastructure/haproxy/tasks/main.yml b/roles/infrastructure/haproxy/tasks/main.yml index f5cf47e8..d2a4597f 100644 --- a/roles/infrastructure/haproxy/tasks/main.yml +++ b/roles/infrastructure/haproxy/tasks/main.yml @@ -15,7 +15,8 @@ --- - name: Install HAProxy - package: + ansible.builtin.package: + lock_timeout: 60 name: haproxy state: latest diff --git a/roles/infrastructure/krb5_client/handlers/main.yml b/roles/infrastructure/krb5_client/handlers/main.yml new file mode 100644 index 00000000..13f8994f --- /dev/null +++ b/roles/infrastructure/krb5_client/handlers/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: restart sssd + service: + name: sssd + state: restarted \ No newline at end of file diff --git a/roles/infrastructure/krb5_client/meta/main.yml b/roles/infrastructure/krb5_client/meta/main.yml new file mode 100644 index 00000000..ea97a5b0 --- /dev/null +++ b/roles/infrastructure/krb5_client/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.krb5_common \ No newline at end of file diff --git a/prepare_security.yml b/roles/infrastructure/krb5_client/tasks/freeipa.yml similarity index 51% rename from prepare_security.yml rename to roles/infrastructure/krb5_client/tasks/freeipa.yml index 5b0ad967..4b010d11 100644 --- a/prepare_security.yml +++ b/roles/infrastructure/krb5_client/tasks/freeipa.yml @@ -13,24 +13,21 @@ # limitations under the License. --- +- name: Setup FreeIPA Client + ansible.builtin.include_role: + name: freeipa.ansible_freeipa.ipaclient + vars: + state: present + ipaserver_realm: "{{ krb5_realm }}" + ipaserver_domain: "{{ krb5_domain }}" + ipaclient_servers: "{{ groups['krb5_server'] }}" -- name: Install pre-requisite packages for Kerberos - hosts: cloudera_manager, cluster - become: yes - roles: - - role: prereqs/kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - prereqs - -- name: Configure Cloudera Manager server for Kerberos - hosts: localhost - connection: local - gather_facts: no - roles: - - role: cloudera_manager/kerberos - when: krb5_kdc_host is defined or 'krb5_server' in groups - tags: - - kerberos - - configuration +- name: Set sssd to enumerate users and groups + lineinfile: + path: /etc/sssd/sssd.conf + insertafter: "^\\[domain/.+\\]" + regexp: "^enumerate" + line: "enumerate = True" + when: "krb5_kdc_type == 'Red Hat IPA' and 'krb5_server' in groups" + notify: + - restart sssd \ No newline at end of file diff --git a/roles/infrastructure/krb5_client/tasks/main.yml b/roles/infrastructure/krb5_client/tasks/main.yml index 94aac0cc..bca64d72 100644 --- a/roles/infrastructure/krb5_client/tasks/main.yml +++ b/roles/infrastructure/krb5_client/tasks/main.yml @@ -13,19 +13,10 @@ # limitations under the License. --- +- name: Install MIT KRB5 Client setup + when: krb5_kdc_type != 'Red Hat IPA' + ansible.builtin.include_tasks: mit.yml -- name: Include variables - include_vars: - file: "{{ ansible_os_family }}.yml" - -- name: Install KRB5 client libraries - package: - name: "{{ krb5_packages }}" - state: present - -- name: Create krb5.conf - template: - src: "{{ krb5_conf_template | default('krb5.conf.j2') }}" - dest: /etc/krb5.conf - backup: yes - when: not (skip_krb5_conf_distribution | default(False)) +- name: Install FreeIPA KRB5 Client setup + when: krb5_kdc_type == 'Red Hat IPA' + ansible.builtin.include_tasks: freeipa.yml diff --git a/set_cloudera_manager_password.yml b/roles/infrastructure/krb5_client/tasks/mit.yml similarity index 58% rename from set_cloudera_manager_password.yml rename to roles/infrastructure/krb5_client/tasks/mit.yml index c82114ef..e32eba00 100644 --- a/set_cloudera_manager_password.yml +++ b/roles/infrastructure/krb5_client/tasks/mit.yml @@ -13,15 +13,19 @@ # limitations under the License. --- +- name: Include variables + include_vars: + file: "{{ ansible_os_family }}.yml" -- name: Change the Cloudera Manager admin password - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera_manager/admin_password/set +- name: Install KRB5 client libraries + ansible.builtin.package: + lock_timeout: 60 + name: "{{ krb5_packages }}" + state: present -- name: Check Cloudera Manager admin password - hosts: cloudera_manager:cluster - gather_facts: no - roles: - - cloudera_manager/api_client +- name: Create krb5.conf + template: + src: "{{ krb5_conf_template | default('krb5.conf.j2') }}" + dest: /etc/krb5.conf + backup: yes + when: not (skip_krb5_conf_distribution | default(False)) \ No newline at end of file diff --git a/roles/infrastructure/krb5_common/defaults/main.yml b/roles/infrastructure/krb5_common/defaults/main.yml new file mode 100644 index 00000000..de3b9389 --- /dev/null +++ b/roles/infrastructure/krb5_common/defaults/main.yml @@ -0,0 +1,20 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +krb5_realm: CLOUDERA.LOCAL +krb5_domain: "{{ krb5_realm | lower }}" +krb5_kdc_admin_user: "cloudera-scm/admin@{{ krb5_realm }}" +krb5_kdc_admin_password: changeme +krb5_kdc_type: MIT KDC +krb5_enc_types: "aes256-cts aes128-cts" \ No newline at end of file diff --git a/roles/infrastructure/krb5_server/meta/main.yml b/roles/infrastructure/krb5_server/meta/main.yml index d71ebbe9..87f7147e 100644 --- a/roles/infrastructure/krb5_server/meta/main.yml +++ b/roles/infrastructure/krb5_server/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: infrastructure/krb5_client + - role: cloudera.cluster.infrastructure.krb5_client diff --git a/roles/infrastructure/krb5_server/tasks/freeipa.yml b/roles/infrastructure/krb5_server/tasks/freeipa.yml new file mode 100644 index 00000000..d79adf99 --- /dev/null +++ b/roles/infrastructure/krb5_server/tasks/freeipa.yml @@ -0,0 +1,23 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Setup FreeIPA Server + ansible.builtin.include_role: + name: freeipa.ansible_freeipa.ipaserver + vars: + state: present + ipaserver_realm: "{{ krb5_realm }}" + ipaserver_domain: "{{ krb5_domain }}" + ipaserver_setup_firewalld: "no" diff --git a/roles/infrastructure/krb5_server/tasks/main.yml b/roles/infrastructure/krb5_server/tasks/main.yml index 6df08717..680e0367 100644 --- a/roles/infrastructure/krb5_server/tasks/main.yml +++ b/roles/infrastructure/krb5_server/tasks/main.yml @@ -13,40 +13,10 @@ # limitations under the License. --- +- name: Install MIT KDC Server + when: krb5_kdc_type == 'MIT KDC' + ansible.builtin.include_tasks: mit.yml -- name: Include variables - include_vars: - file: "{{ ansible_os_family }}.yml" - -- name: Install KRB5 server - package: - name: "{{ krb5_packages }}" - state: present - -- name: Set kdc.conf - template: - src: "{{ ansible_os_family }}/kdc.conf.j2" - dest: "{{ krb5_kdc_state_directory }}/kdc.conf" - backup: yes - -- name: Create KDC database - command: "/usr/sbin/kdb5_util create -s -P {{ krb5_kdc_master_password }}" - args: - creates: "{{ krb5_kdc_database }}" - -- name: Set kadm5.acl - template: - src: kadm5.acl.j2 - dest: "{{ krb5_kdc_state_directory }}/kadm5.acl" - backup: yes - -- name: Create Cloudera Manager admin principal - command: /usr/sbin/kadmin.local -q "addprinc -pw {{ krb5_kdc_admin_password }} {{ krb5_kdc_admin_user }}" - -- name: Start Kerberos KDC - service: - name: "{{ item }}" - state: restarted - enabled: yes - with_items: - - "{{ krb5_services }}" +- name: Install FreeIPA Server + when: krb5_kdc_type == 'Red Hat IPA' + ansible.builtin.include_tasks: freeipa.yml diff --git a/roles/infrastructure/krb5_server/tasks/mit.yml b/roles/infrastructure/krb5_server/tasks/mit.yml new file mode 100644 index 00000000..fdab9a2c --- /dev/null +++ b/roles/infrastructure/krb5_server/tasks/mit.yml @@ -0,0 +1,52 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Include variables + include_vars: + file: "{{ ansible_os_family }}.yml" + +- name: Install KRB5 server + ansible.builtin.package: + lock_timeout: 60 + name: "{{ krb5_packages }}" + state: present + +- name: Set kdc.conf + template: + src: "{{ ansible_os_family }}/kdc.conf.j2" + dest: "{{ krb5_kdc_state_directory }}/kdc.conf" + backup: yes + +- name: Create KDC database + command: "/usr/sbin/kdb5_util create -s -P {{ krb5_kdc_master_password }}" + args: + creates: "{{ krb5_kdc_database }}" + +- name: Set kadm5.acl + template: + src: kadm5.acl.j2 + dest: "{{ krb5_kdc_state_directory }}/kadm5.acl" + backup: yes + +- name: Create Cloudera Manager admin principal + command: /usr/sbin/kadmin.local -q "addprinc -pw {{ krb5_kdc_admin_password }} {{ krb5_kdc_admin_user }}" + +- name: Start Kerberos KDC + service: + name: "{{ item }}" + state: restarted + enabled: yes + with_items: + - "{{ krb5_services }}" \ No newline at end of file diff --git a/roles/infrastructure/rdbms/meta/main.yml b/roles/infrastructure/rdbms/meta/main.yml new file mode 100644 index 00000000..f63fb6e8 --- /dev/null +++ b/roles/infrastructure/rdbms/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.deployment.definition diff --git a/roles/infrastructure/rdbms/tasks/main.yml b/roles/infrastructure/rdbms/tasks/main.yml index d9166781..41dc0829 100644 --- a/roles/infrastructure/rdbms/tasks/main.yml +++ b/roles/infrastructure/rdbms/tasks/main.yml @@ -13,7 +13,6 @@ # limitations under the License. --- - - name: Include database type specific variables include_vars: file: "{{ database_type }}.yml" diff --git a/roles/infrastructure/rdbms/tasks/postgresql-Debian.yml b/roles/infrastructure/rdbms/tasks/postgresql-Debian.yml index bb4f696a..b13e2a0f 100644 --- a/roles/infrastructure/rdbms/tasks/postgresql-Debian.yml +++ b/roles/infrastructure/rdbms/tasks/postgresql-Debian.yml @@ -31,7 +31,8 @@ name: geerlingguy.postgresql - name: Install python-psycopg2 - package: + ansible.builtin.package: + lock_timeout: 60 name: python-psycopg2 state: present diff --git a/roles/operations/delete_cluster/meta/main.yml b/roles/operations/delete_cluster/meta/main.yml index 1aee5eba..69f6a359 100644 --- a/roles/operations/delete_cluster/meta/main.yml +++ b/roles/operations/delete_cluster/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - role: cloudera_manager/api_client diff --git a/roles/operations/delete_cluster/tasks/main.yml b/roles/operations/delete_cluster/tasks/main.yml index a723769b..997f27ae 100644 --- a/roles/operations/delete_cluster/tasks/main.yml +++ b/roles/operations/delete_cluster/tasks/main.yml @@ -15,7 +15,7 @@ --- - name: Check the cluster exists - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }} method: GET status_code: 200,404 @@ -23,7 +23,7 @@ register: cluster_before_delete - name: Stop the cluster (stop_cluster_before_delete) - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/commands/stop method: POST register: services_stop @@ -33,7 +33,7 @@ failed_when: services_stop.status | default(0) != 200 - name: Wait for the cluster services to stop - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/services method: GET status_code: 200,404 @@ -53,7 +53,7 @@ delay: "{{ teardown_stop_cluster_poll_duration | default(20) }}" - name: Get cluster services - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/services method: GET status_code: 200,404 @@ -62,7 +62,7 @@ when: cluster_before_delete.status == 200 - name: Delete data contexts of {{ cluster.name }} - cm_api: + cloudera.cluster.cm_api: endpoint: /dataContexts/{{ data_context.name | urlencode() }} method: DELETE status_code: 200,404 @@ -79,7 +79,7 @@ - set_fact: stopped_service_count: "{{ services.json | json_query('items[?(serviceState==`STOPPED` || serviceState==`NA`)]') | length }}" total_service_count: "{{ services.json | json_query('items[*]') | length }}" - - cm_api: + - cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }} method: DELETE when: stopped_service_count == total_service_count diff --git a/roles/operations/delete_cms/meta/main.yml b/roles/operations/delete_cms/meta/main.yml index 1aee5eba..3a5a0054 100644 --- a/roles/operations/delete_cms/meta/main.yml +++ b/roles/operations/delete_cms/meta/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/operations/delete_cms/tasks/main.yml b/roles/operations/delete_cms/tasks/main.yml index 480f44a9..6f6f5955 100644 --- a/roles/operations/delete_cms/tasks/main.yml +++ b/roles/operations/delete_cms/tasks/main.yml @@ -15,7 +15,7 @@ --- - name: Check cms exists - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service method: GET status_code: 200,404 @@ -23,7 +23,7 @@ register: service_before_delete - name: Stop cms (stop_cms_before_delete) - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service/commands/stop method: POST register: service_stop @@ -33,7 +33,7 @@ failed_when: service_stop.status | default(0) != 200 - name: Get the cms service - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service method: GET status_code: 200,404 @@ -42,7 +42,7 @@ when: service_before_delete.status == 200 - name: Delete cms - cm_api: + cloudera.cluster.cm_api: endpoint: /cm/service method: DELETE when: diff --git a/roles/operations/refresh_ranger_kms_repo/meta/main.yml b/roles/operations/refresh_ranger_kms_repo/meta/main.yml index 201f8987..f52ecf8c 100644 --- a/roles/operations/refresh_ranger_kms_repo/meta/main.yml +++ b/roles/operations/refresh_ranger_kms_repo/meta/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + dependencies: - - role: cloudera_manager/api_client - - role: deployment/definition + - role: cloudera.cluster.cloudera_manager.api_client + - role: cloudera.cluster.deployment.definition diff --git a/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml b/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml index 4ea4ed2a..e10a1d58 100644 --- a/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml +++ b/roles/operations/refresh_ranger_kms_repo/tasks/cluster_find_ranger.yml @@ -17,12 +17,12 @@ - name: Check a cluster has been specified fail: msg: This task list expects a cluster var. - when: cluster is not defined + when: __cluster_item is not defined - name: Check the cluster has a Ranger service fail: msg: This task list expects a the cluster to have a Ranger service. - when: "'RANGER' not in cluster.services | default([])" + when: "'RANGER' not in __cluster_item.services | default([])" - name: Identify the host template with the Ranger admin set_fact: @@ -30,7 +30,7 @@ vars: host_templates: > {{ - cluster.host_templates + __cluster_item.host_templates | dict2items(key_name='name', value_name='services') }} query: "[?services.RANGER[?contains(@, `RANGER_ADMIN`)]].name" @@ -59,13 +59,13 @@ - name: Identify Ranger service defaults set_fact: _ranger_host: "{{ _ranger_hosts | first }}" - _ranger_http_port: "{{ cluster.configs.RANGER.SERVICEWIDE.ranger_service_http_port | default(6080) }}" - _ranger_https_port: "{{ cluster.configs.RANGER.SERVICEWIDE.ranger_service_https_port | default(6182) }}" + _ranger_http_port: "{{ __cluster_item.configs.RANGER.SERVICEWIDE.ranger_service_http_port | default(6080) }}" + _ranger_https_port: "{{ __cluster_item.configs.RANGER.SERVICEWIDE.ranger_service_https_port | default(6182) }}" - name: Identify Ranger credentials set_fact: ranger_keyadmin_username: "keyadmin" - ranger_keyadmin_password: "{{ cluster.configs.RANGER.SERVICEWIDE.keyadmin_user_password | default('password123') }}" + ranger_keyadmin_password: "{{ __cluster_item.configs.RANGER.SERVICEWIDE.keyadmin_user_password | default('password123') }}" - name: Check if HTTPS is used uri: diff --git a/roles/operations/refresh_ranger_kms_repo/tasks/main.yml b/roles/operations/refresh_ranger_kms_repo/tasks/main.yml index 40d5bf73..dda5ffc5 100644 --- a/roles/operations/refresh_ranger_kms_repo/tasks/main.yml +++ b/roles/operations/refresh_ranger_kms_repo/tasks/main.yml @@ -18,10 +18,10 @@ include_tasks: setup_cluster.yml loop: "{{ definition.clusters }}" loop_control: - loop_var: cluster - label: "{{ cluster.name }}" + loop_var: __cluster_item + label: "{{ __cluster_item.name }}" when: - - cluster.type | default('base') == 'base' - - cluster.security.hdfs_encryption | default(false) - - "'RANGER' in cluster.services | default([])" + - __cluster_item.type | default('base') == 'base' + - __cluster_item.security.hdfs_encryption | default(false) + - "'RANGER' in __cluster_item.services | default([])" - '"kms_servers" in groups' diff --git a/roles/operations/refresh_ranger_kms_repo/tasks/setup_cluster.yml b/roles/operations/refresh_ranger_kms_repo/tasks/setup_cluster.yml index 75942126..eea34104 100644 --- a/roles/operations/refresh_ranger_kms_repo/tasks/setup_cluster.yml +++ b/roles/operations/refresh_ranger_kms_repo/tasks/setup_cluster.yml @@ -16,7 +16,7 @@ - fail: msg: You must pass a cluster variable to this role - when: cluster is not defined + when: __cluster_item is not defined - name: Find Ranger admin include_tasks: cluster_find_ranger.yml @@ -53,6 +53,6 @@ - "'localhost' in kms_service.configs.provider" - name: Setup Ranger Plugins - cm_api: - endpoint: "/clusters/{{ cluster.name | urlencode }}/services/ranger/commands/SetupPluginServices" + cloudera.cluster.cm_api: + endpoint: "/clusters/{{ __cluster_item.name | urlencode }}/services/ranger/commands/SetupPluginServices" method: POST diff --git a/roles/operations/restart_agents/tasks/main.yml b/roles/operations/restart_agents/tasks/main.yml new file mode 100644 index 00000000..d44056b5 --- /dev/null +++ b/roles/operations/restart_agents/tasks/main.yml @@ -0,0 +1,19 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: restart cloudera-scm-agent + service: + name: cloudera-scm-agent + state: restarted \ No newline at end of file diff --git a/roles/operations/restart_stale/meta/main.yml b/roles/operations/restart_stale/meta/main.yml index 1aee5eba..524838cc 100644 --- a/roles/operations/restart_stale/meta/main.yml +++ b/roles/operations/restart_stale/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: cloudera_manager/api_client + - role: cloudera.cluster.cloudera_manager.api_client diff --git a/roles/operations/restart_stale/tasks/main.yml b/roles/operations/restart_stale/tasks/main.yml index f67f4625..6601afed 100644 --- a/roles/operations/restart_stale/tasks/main.yml +++ b/roles/operations/restart_stale/tasks/main.yml @@ -15,7 +15,7 @@ --- - name: Get clusters - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters register: clusters diff --git a/roles/operations/restart_stale/tasks/restart.yml b/roles/operations/restart_stale/tasks/restart.yml index 5d5f17c4..bdca935a 100644 --- a/roles/operations/restart_stale/tasks/restart.yml +++ b/roles/operations/restart_stale/tasks/restart.yml @@ -15,7 +15,7 @@ --- - name: Get cluster services - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/services register: services @@ -24,13 +24,13 @@ stale_client_config_count: "{{ services.json | json_query('items[?clientConfigStalenessStatus==`STALE`]') | length }}" - name: Restart cluster - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/commands/restart method: POST when: stale_service_count | int > 0 - name: Re-deploy client configurations - cm_api: + cloudera.cluster.cm_api: endpoint: /clusters/{{ cluster.name | urlencode() }}/commands/deployClientConfig method: POST timeout: "{{ client_config_timeout | default(omit) }}" diff --git a/roles/prereqs/jdk/tasks/main.yml b/roles/prereqs/jdk/tasks/main.yml index ef842241..317ff3ec 100644 --- a/roles/prereqs/jdk/tasks/main.yml +++ b/roles/prereqs/jdk/tasks/main.yml @@ -30,7 +30,8 @@ when: jdk_package is not defined - name: Install JDK - package: + ansible.builtin.package: + lock_timeout: 60 name: - "{{ jdk_package }}" state: present diff --git a/roles/prereqs/kerberos/meta/main.yml b/roles/prereqs/kerberos/meta/main.yml index d71ebbe9..87f7147e 100644 --- a/roles/prereqs/kerberos/meta/main.yml +++ b/roles/prereqs/kerberos/meta/main.yml @@ -14,4 +14,4 @@ --- dependencies: - - role: infrastructure/krb5_client + - role: cloudera.cluster.infrastructure.krb5_client diff --git a/roles/prereqs/kerberos/tasks/main.yml b/roles/prereqs/kerberos/tasks/main.yml index 7a33f4c5..1ff5538a 100644 --- a/roles/prereqs/kerberos/tasks/main.yml +++ b/roles/prereqs/kerberos/tasks/main.yml @@ -25,7 +25,8 @@ when: ansible_os_family == "Debian" - name: Install OpenLDAP client libraries - package: + ansible.builtin.package: + lock_timeout: 60 name: "{{ ldap_packages }}" state: present when: "'cloudera_manager' in group_names" diff --git a/roles/prereqs/license/defaults/main.yml b/roles/prereqs/license/defaults/main.yml new file mode 100644 index 00000000..a4c1803a --- /dev/null +++ b/roles/prereqs/license/defaults/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Path to the license file on the Ansible controller +cloudera_manager_license_file: '' +license_local_tmp_path: /tmp/license.txt \ No newline at end of file diff --git a/enable_autotls.yml b/roles/prereqs/license/tasks/main.yml similarity index 54% rename from enable_autotls.yml rename to roles/prereqs/license/tasks/main.yml index 1f8b9689..f60a8156 100644 --- a/enable_autotls.yml +++ b/roles/prereqs/license/tasks/main.yml @@ -13,21 +13,17 @@ # limitations under the License. --- +- name: Copy Cloudera License File to temp path for Cloudera Manager + copy: + src: "{{ cloudera_manager_license_file }}" + dest: "{{ license_local_tmp_path }}" + mode: 0600 + when: + - cloudera_manager_license_file + - "'cloudera_manager' in groups" + - inventory_hostname in groups['cloudera_manager'] -- name: Enable Auto-TLS - hosts: cloudera_manager - gather_facts: no - roles: - - cloudera_manager/autotls - tags: - - security - - autotls - -- name: Restart and re-deploy client configs - hosts: localhost - connection: local - gather_facts: no - roles: - - operations/restart_stale - tags: - - restart +- name: Set license type to Enterprise if license is presented + ansible.builtin.set_fact: + cloudera_manager_license_type: enterprise + when: cloudera_manager_license_file diff --git a/vars/user_accounts.yml b/roles/prereqs/local_accounts_common/defaults/main.yml similarity index 98% rename from vars/user_accounts.yml rename to roles/prereqs/local_accounts_common/defaults/main.yml index 45dcb760..afa8e27b 100644 --- a/vars/user_accounts.yml +++ b/roles/prereqs/local_accounts_common/defaults/main.yml @@ -13,7 +13,7 @@ # limitations under the License. --- - +skip_user_group_init: false local_accounts: - user: accumulo @@ -61,7 +61,7 @@ local_accounts: - user: hdfs home: /var/lib/hadoop-hdfs comment: Hadoop HDFS - extra_groups: [hadoop] + extra_groups: [hadoop] - user: hive home: /var/lib/hive diff --git a/roles/prereqs/mysql_connector/defaults/main.yml b/roles/prereqs/mysql_connector/defaults/main.yml index 14d4021e..dc22f192 100644 --- a/roles/prereqs/mysql_connector/defaults/main.yml +++ b/roles/prereqs/mysql_connector/defaults/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + mysql_connector_url: https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.48.zip mysql_connector_checksum: md5:5da24facd99964f296ecde32abcd2384 mysql_connector_download_dir: "{{ local_temp_dir }}" diff --git a/roles/prereqs/mysql_connector/meta/main.yml b/roles/prereqs/mysql_connector/meta/main.yml new file mode 100644 index 00000000..f63fb6e8 --- /dev/null +++ b/roles/prereqs/mysql_connector/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.deployment.definition diff --git a/roles/prereqs/mysql_connector/tasks/main.yml b/roles/prereqs/mysql_connector/tasks/main.yml index a5372316..2d7a911c 100644 --- a/roles/prereqs/mysql_connector/tasks/main.yml +++ b/roles/prereqs/mysql_connector/tasks/main.yml @@ -31,7 +31,8 @@ mode: 0755 - name: Install unzip package - package: + ansible.builtin.package: + lock_timeout: 60 name: unzip state: present diff --git a/roles/prereqs/oracle_connector/defaults/main.yml b/roles/prereqs/oracle_connector/defaults/main.yml index ab6b574a..1e396b5a 100644 --- a/roles/prereqs/oracle_connector/defaults/main.yml +++ b/roles/prereqs/oracle_connector/defaults/main.yml @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- + +maven_repo: "https://repo1.maven.org/maven2" + oracle_connector_group_id: com.oracle.database.jdbc oracle_connector_artifact_id: ojdbc8 oracle_connector_version: 19.7.0.0 diff --git a/roles/prereqs/oracle_connector/meta/main.yml b/roles/prereqs/oracle_connector/meta/main.yml new file mode 100644 index 00000000..f63fb6e8 --- /dev/null +++ b/roles/prereqs/oracle_connector/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.deployment.definition diff --git a/roles/prereqs/oracle_connector/tasks/main.yml b/roles/prereqs/oracle_connector/tasks/main.yml index 5ab0b0fc..da574259 100644 --- a/roles/prereqs/oracle_connector/tasks/main.yml +++ b/roles/prereqs/oracle_connector/tasks/main.yml @@ -56,7 +56,8 @@ - oracle_instantclient_sdk_zip is defined - name: Ensure unzip is available - package: + ansible.builtin.package: + lock_timeout: 60 name: unzip state: present when: @@ -78,6 +79,7 @@ when: oracle_instantclient_sdk_zip is defined - name: Install the libaio package - package: + ansible.builtin.package: + lock_timeout: 60 name: libaio state: present diff --git a/roles/prereqs/os/tasks/main.yml b/roles/prereqs/os/tasks/main.yml index 8333b102..e6b3489a 100644 --- a/roles/prereqs/os/tasks/main.yml +++ b/roles/prereqs/os/tasks/main.yml @@ -48,7 +48,8 @@ with_items: "{{ unnecessary_services }}" - name: Install DNS utils package - package: + ansible.builtin.package: + lock_timeout: 60 name: "{{ dns_package }}" state: present @@ -57,7 +58,8 @@ manager: auto - name: Install {{ ntp_package }} service if ntp or chrony is not installed - package: + ansible.builtin.package: + lock_timeout: 60 name: "{{ ntp_package }}" state: present when: ('ntp' not in ansible_facts.packages) and ('chrony' not in ansible_facts.packages) @@ -74,7 +76,8 @@ enabled: yes - name: Install nscd service - package: + ansible.builtin.package: + lock_timeout: 60 name: "{{ nscd_package }}" state: present diff --git a/roles/prereqs/os/vars/RedHat.yml b/roles/prereqs/os/vars/RedHat.yml index 38da874b..e3176ee4 100644 --- a/roles/prereqs/os/vars/RedHat.yml +++ b/roles/prereqs/os/vars/RedHat.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + dns_package: bind-utils nscd_package: nscd nscd_service: nscd diff --git a/roles/prereqs/os/vars/Suse.yml b/roles/prereqs/os/vars/Suse.yml index c75d66eb..14677d9e 100644 --- a/roles/prereqs/os/vars/Suse.yml +++ b/roles/prereqs/os/vars/Suse.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + nscd_package: nscd nscd_service: nscd rngd_package: rng-tools diff --git a/roles/prereqs/os/vars/Ubuntu18.yml b/roles/prereqs/os/vars/Ubuntu18.yml index c48e2730..7e8863e3 100644 --- a/roles/prereqs/os/vars/Ubuntu18.yml +++ b/roles/prereqs/os/vars/Ubuntu18.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + dns_package: dnsutils nscd_package: nscd nscd_service: nscd diff --git a/roles/prereqs/user_accounts/meta/main.yml b/roles/prereqs/user_accounts/meta/main.yml new file mode 100644 index 00000000..f973edd0 --- /dev/null +++ b/roles/prereqs/user_accounts/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.prereqs.local_accounts_common \ No newline at end of file diff --git a/roles/prereqs/user_accounts/tasks/main.yml b/roles/prereqs/user_accounts/tasks/main.yml index 0647e268..16eb6880 100644 --- a/roles/prereqs/user_accounts/tasks/main.yml +++ b/roles/prereqs/user_accounts/tasks/main.yml @@ -15,10 +15,6 @@ --- - block: - - name: Include user groups vars - include_vars: - file: "user_accounts.yml" - - name: Create hadoop group group: name: hadoop @@ -34,16 +30,18 @@ run_once: true - name: Create local user accounts - user: + ansible.builtin.user: name: "{{ account.user }}" home: "{{ account.home }}" comment: "{{ account.comment | default(omit) }}" groups: "{{ account.extra_groups | default([]) }}" + uid: "{{ account.uid | default(omit) }}" shell: /sbin/nologin append: yes loop: "{{ local_accounts }}" loop_control: loop_var: account + label: "{{ account.user }}" when: account.when | default(True) - name: Set home directory permissions diff --git a/roles/security/tls_clean/meta/main.yml b/roles/security/tls_clean/meta/main.yml new file mode 100644 index 00000000..830dd2a1 --- /dev/null +++ b/roles/security/tls_clean/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file diff --git a/roles/security/tls_clean/tasks/main.yml b/roles/security/tls_clean/tasks/main.yml new file mode 100644 index 00000000..23b472d1 --- /dev/null +++ b/roles/security/tls_clean/tasks/main.yml @@ -0,0 +1,19 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Delete security artifacts + file: + name: "{{ base_dir_security }}" + state: absent \ No newline at end of file diff --git a/roles/security/tls_generate_csr/defaults/main.yml b/roles/security/tls_generate_csr/defaults/main.yml index 5805ca41..37d85dea 100644 --- a/roles/security/tls_generate_csr/defaults/main.yml +++ b/roles/security/tls_generate_csr/defaults/main.yml @@ -13,5 +13,6 @@ # limitations under the License. --- + keytool_path: /usr/bin/keytool openssl_path: /usr/bin/openssl diff --git a/roles/security/tls_generate_csr/meta/main.yml b/roles/security/tls_generate_csr/meta/main.yml new file mode 100644 index 00000000..aabb7a51 --- /dev/null +++ b/roles/security/tls_generate_csr/meta/main.yml @@ -0,0 +1,18 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.ca_common + - role: cloudera.cluster.prereqs.local_accounts_common \ No newline at end of file diff --git a/roles/security/tls_generate_csr/molecule/default/molecule.yml b/roles/security/tls_generate_csr/molecule/default/molecule.yml index 848115cf..cc1e76ea 100644 --- a/roles/security/tls_generate_csr/molecule/default/molecule.yml +++ b/roles/security/tls_generate_csr/molecule/default/molecule.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + dependency: name: galaxy driver: diff --git a/roles/security/tls_generate_csr/molecule/default/prepare.yml b/roles/security/tls_generate_csr/molecule/default/prepare.yml index 0a9829f2..af0e4507 100644 --- a/roles/security/tls_generate_csr/molecule/default/prepare.yml +++ b/roles/security/tls_generate_csr/molecule/default/prepare.yml @@ -18,8 +18,10 @@ tasks: - name: Include jdk role include_role: - name: ../prereqs/jdk + name: cloudera.cluster.prereqs.jdk + - name: Install openssl - package: + ansible.builtin.package: + lock_timeout: 60 name: openssl state: present diff --git a/roles/security/tls_generate_csr/tasks/acls.yml b/roles/security/tls_generate_csr/tasks/acls.yml index 5ab938f6..92f9c94c 100644 --- a/roles/security/tls_generate_csr/tasks/acls.yml +++ b/roles/security/tls_generate_csr/tasks/acls.yml @@ -14,12 +14,9 @@ --- -- name: Include user groups vars - include_vars: - file: "user_accounts.yml" - - name: Install acls package - package: + ansible.builtin.package: + lock_timeout: 60 name: acl state: present diff --git a/roles/security/tls_install_certs/defaults/main.yml b/roles/security/tls_install_certs/defaults/main.yml index 5ec9f280..f26cb331 100644 --- a/roles/security/tls_install_certs/defaults/main.yml +++ b/roles/security/tls_install_certs/defaults/main.yml @@ -13,6 +13,7 @@ # limitations under the License. --- + keytool_path: /usr/bin/keytool openssl_path: /usr/bin/openssl jdk_java_cacerts_paths: diff --git a/roles/security/tls_install_certs/meta/main.yml b/roles/security/tls_install_certs/meta/main.yml new file mode 100644 index 00000000..830dd2a1 --- /dev/null +++ b/roles/security/tls_install_certs/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file diff --git a/roles/security/tls_install_certs/tasks/main.yml b/roles/security/tls_install_certs/tasks/main.yml index 22ebf8a1..67f9b05f 100644 --- a/roles/security/tls_install_certs/tasks/main.yml +++ b/roles/security/tls_install_certs/tasks/main.yml @@ -28,7 +28,9 @@ - name: Check if signed cert is available become: no - local_action: stat path={{ tls_signed_certs_dir }}/{{ inventory_hostname }}.pem + delegate_to: localhost + stat: + path: "{{ tls_signed_certs_dir }}/{{ inventory_hostname }}.pem" register: signed_cert - name: Check if signed cert is available remotely diff --git a/teardown.yml b/roles/security/tls_nifi/defaults/main.yml similarity index 66% rename from teardown.yml rename to roles/security/tls_nifi/defaults/main.yml index cbbf1229..0b5073cd 100644 --- a/teardown.yml +++ b/roles/security/tls_nifi/defaults/main.yml @@ -14,15 +14,13 @@ --- -- name: Group hosts using host template information - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts.yml +nifi_dir_path: /var/lib/nifi +nifi_dir_owner: nifi +nifi_dir_group: nifi -- name: Teardown - hosts: all - gather_facts: no - any_errors_fatal: true - roles: - - teardown +nifireg_dir_path: /var/lib/nifiregistry +nifireg_dir_owner: nifiregistry +nifireg_dir_group: nifiregistry + +cm_auto_host_keystore_name: cm-auto-host_keystore.jks +cm_auto_cluster_truststore_name: cm-auto-in_cluster_truststore.jks \ No newline at end of file diff --git a/setup_kms_service.yml b/roles/security/tls_nifi/meta/main.yml similarity index 81% rename from setup_kms_service.yml rename to roles/security/tls_nifi/meta/main.yml index 80c85395..cd187111 100644 --- a/setup_kms_service.yml +++ b/roles/security/tls_nifi/meta/main.yml @@ -14,9 +14,5 @@ --- -- name: Create KMS services - hosts: localhost - connection: local - gather_facts: no - roles: - - role: deployment/services/kms +dependencies: + - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file diff --git a/roles/security/tls_nifi/tasks/main.yml b/roles/security/tls_nifi/tasks/main.yml new file mode 100644 index 00000000..a26aa200 --- /dev/null +++ b/roles/security/tls_nifi/tasks/main.yml @@ -0,0 +1,53 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +- name: Ensure the NiFi home directory exists + file: + path: "{{ nifi_dir_path }}" + owner: "{{ nifi_dir_owner }}" + group: "{{ nifi_dir_group }}" + state: directory + +- name: Ensure the link for the NiFi keystore exists + file: + src: "{{ tls_keystore_path }}" + dest: "{{ nifi_dir_path }}/{{ cm_auto_host_keystore_name }}" + state: link + +- name: Ensure the link for the NiFi truststore exists + file: + src: "{{ tls_truststore_path }}" + dest: "{{ nifi_dir_path }}/{{ cm_auto_cluster_truststore_name }}" + state: link + +- name: Ensure the NiFi Registry home directory exists + file: + path: "{{ nifireg_dir_path }}" + owner: "{{ nifireg_dir_owner }}" + group: "{{ nifireg_dir_group }}" + state: directory + +- name: Ensure the link for the NiFi Registry keystore exists + file: + src: "{{ tls_keystore_path }}" + dest: "{{ nifireg_dir_path }}/{{ cm_auto_host_keystore_name }}" + state: link + +- name: Ensure the link for the NiFi Registry truststore exists + file: + src: "{{ tls_truststore_path }}" + dest: "{{ nifireg_dir_path }}/{{ cm_auto_cluster_truststore_name }}" + state: link diff --git a/roles/security/tls_signing/meta/main.yml b/roles/security/tls_signing/meta/main.yml new file mode 100644 index 00000000..830dd2a1 --- /dev/null +++ b/roles/security/tls_signing/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.ca_common \ No newline at end of file diff --git a/roles/security/tls_signing/tasks/csr_signing_local.yml b/roles/security/tls_signing/tasks/csr_signing_local.yml index db692062..e74c29b3 100644 --- a/roles/security/tls_signing/tasks/csr_signing_local.yml +++ b/roles/security/tls_signing/tasks/csr_signing_local.yml @@ -14,9 +14,6 @@ --- -- name: Include local CA server vars - include_vars: ca_server.yml - - name: Copy CSRs to CA server copy: src: "{{ local_csrs_dir }}/{{ inventory_hostname }}.csr" diff --git a/roles/teardown/meta/main.yml b/roles/teardown/meta/main.yml index 4e3a2f89..23756888 100644 --- a/roles/teardown/meta/main.yml +++ b/roles/teardown/meta/main.yml @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- + dependencies: - - role: cloudera_manager/common - - role: deployment/definition - - role: config/cluster/common + - role: cloudera.cluster.cloudera_manager.common + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.deployment.services.kts_common diff --git a/roles/teardown/tasks/main.yml b/roles/teardown/tasks/main.yml index 1c99aeb0..c636ce11 100644 --- a/roles/teardown/tasks/main.yml +++ b/roles/teardown/tasks/main.yml @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- + +- name: Include config cluster defaults for deployment + ansible.builtin.include_role: + name: cloudera.cluster.config.cluster.common + public: yes + - name: Ensure properly configured assert: that: >- @@ -147,7 +154,8 @@ - teardown_everything | default(false) - name: Teardown Cloudera daemons - package: + ansible.builtin.package: + lock_timeout: 60 name: cloudera-manager-daemons state: absent when: diff --git a/roles/teardown/tasks/teardown_cloudera_agent.yml b/roles/teardown/tasks/teardown_cloudera_agent.yml index b38abfba..ef304810 100644 --- a/roles/teardown/tasks/teardown_cloudera_agent.yml +++ b/roles/teardown/tasks/teardown_cloudera_agent.yml @@ -23,7 +23,8 @@ ignore_errors: true - name: Remove Cloudera agent package - package: + ansible.builtin.package: + lock_timeout: 60 name: cloudera-manager-agent state: absent diff --git a/roles/teardown/tasks/teardown_cloudera_server.yml b/roles/teardown/tasks/teardown_cloudera_server.yml index 00b05729..62839532 100644 --- a/roles/teardown/tasks/teardown_cloudera_server.yml +++ b/roles/teardown/tasks/teardown_cloudera_server.yml @@ -20,7 +20,8 @@ ignore_errors: true - name: Remove Cloudera manager package - package: + ansible.builtin.package: + lock_timeout: 60 name: cloudera-manager-server state: absent diff --git a/roles/teardown/tasks/teardown_cluster.yml b/roles/teardown/tasks/teardown_cluster.yml index a5feca00..6b1c2047 100644 --- a/roles/teardown/tasks/teardown_cluster.yml +++ b/roles/teardown/tasks/teardown_cluster.yml @@ -46,12 +46,9 @@ when: - cluster.type | default('base') == 'kts' -# refactor and merge into the above task once the KTS locations are properly configured in the overlays +# refactor and merge into the above task once the KTS locations are properly configured in the overlays # https://docs.cloudera.com/cloudera-manager/7.2.0/reference/topics/cm_props_cdh710_keytrusteeserver.html - block: - - set_fact: - keytrustee_server_conf_dir: "{{ hostvars[groups.kts_active[0]]['keytrustee_server_conf_dir'] }}" - keytrustee_server_db_dir: /var/lib/keytrustee/db - name: Remove the KTS configuration directory file: path: "{{ keytrustee_server_conf_dir }}" diff --git a/roles/verify/definition/meta/main.yml b/roles/verify/definition/meta/main.yml index ba00be71..ddcd8f66 100644 --- a/roles/verify/definition/meta/main.yml +++ b/roles/verify/definition/meta/main.yml @@ -13,5 +13,7 @@ # limitations under the License. --- + dependencies: - - role: deployment/definition + - role: cloudera.cluster.deployment.definition + - role: cloudera.cluster.infrastructure.ca_common diff --git a/roles/verify/inventory/meta/main.yml b/roles/verify/inventory/meta/main.yml new file mode 100644 index 00000000..ea97a5b0 --- /dev/null +++ b/roles/verify/inventory/meta/main.yml @@ -0,0 +1,17 @@ +# Copyright 2021 Cloudera, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +dependencies: + - role: cloudera.cluster.infrastructure.krb5_common \ No newline at end of file diff --git a/setup_hdfs_encryption.yml b/setup_hdfs_encryption.yml deleted file mode 100644 index 394a3a9f..00000000 --- a/setup_hdfs_encryption.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Group hosts using host template information - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts.yml - tags: - - always - -- name: Setup Key Trustee Server HA - import_playbook: setup_kts_ha.yml - when: "'kts_passive' in groups" - tags: - - kts - -- name: Create Key Trustee Server organisation - hosts: kts_active - become: yes - tasks: - - shell: > - keytrustee-orgtool add - -n {{ keytrustee_server_org_name }} - -c {{ keytrustee_server_org_email }} - --confdir {{ keytrustee_server_conf_dir }} - tags: - - kts - -- name: Setup KMS service - import_playbook: setup_kms_service.yml - when: "'kms_servers' in groups" - tags: - - kms - -- name: Enable KMS HA - import_playbook: setup_kms_ha.yml - when: - - "'kms_servers' in groups and groups.kms_servers | length > 1" - tags: - - kms - -- name: Refresh KMS Ranger Repo - import_playbook: refresh_ranger_kms_repo.yml - when: "'kms_servers' in groups" - tags: - - kms - -- name: Restart and re-deploy client configs - hosts: localhost - connection: local - gather_facts: no - roles: - - operations/restart_stale - tags: - - restart - vars: - client_config_timeout: "{{ setup_hdfs_encryption_client_config_timeout | default(300) }}" diff --git a/setup_kms_ha.yml b/setup_kms_ha.yml deleted file mode 100644 index 529d453d..00000000 --- a/setup_kms_ha.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create temp directory for keys - hosts: localhost - connection: local - become: yes - gather_facts: no - tasks: - - file: - path: "{{ local_temp_dir }}/kms" - owner: root - group: root - state: directory - mode: 0777 - -- name: Fetch keys from first KMS server - hosts: kms_servers[0] - become: yes - gather_facts: no - tasks: - - fetch: - src: "{{ kms_conf_dir }}/{{ item }}" - dest: "{{ local_temp_dir }}/kms" - fail_on_missing: no - with_items: "{{ kms_key_files }}" - -- name: Copy keys to other KMS servers - hosts: kms_servers[1:] - become: yes - gather_facts: no - tasks: - - copy: - src: "{{ local_temp_dir }}/kms/{{ groups.kms_servers | first }}{{ kms_conf_dir }}/" - dest: "{{ kms_conf_dir }}" - owner: "{{ kms_user }}" - group: "{{ kms_group }}" - mode: 0600 - -- name: Delete temp directory - hosts: localhost - connection: local - become: yes - gather_facts: no - tasks: - - file: - path: "{{ local_temp_dir }}/kms" - state: absent diff --git a/setup_kts_ha.yml b/setup_kts_ha.yml deleted file mode 100644 index e6226733..00000000 --- a/setup_kts_ha.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Create temp directory for keys - hosts: localhost - connection: local - become: yes - gather_facts: no - tasks: - - file: - path: "{{ local_temp_dir }}/kts" - owner: root - group: root - state: directory - mode: 0777 - -- name: Fetch GPG keys and configs from active Key Trustee Server - hosts: kts_active - become: yes - gather_facts: no - tasks: - - fetch: - src: "{{ keytrustee_server_conf_dir }}/{{ item }}" - dest: "{{ local_temp_dir }}/kts" - fail_on_missing: no - when: '"kts_passive" in groups' - with_items: "{{ keytrustee_server_key_files }}" - -- name: Copy to passive Key Trustee Server - hosts: kts_passive - become: yes - gather_facts: no - tasks: - - copy: - src: "{{ local_temp_dir }}/kts/{{ groups.kts_active | first }}{{ keytrustee_server_conf_dir }}/" - dest: "{{ keytrustee_server_conf_dir }}" - owner: keytrustee - group: keytrustee - mode: 0600 - -- name: Delete temp directory - hosts: localhost - connection: local - become: yes - gather_facts: no - tasks: - - file: - path: "{{ local_temp_dir }}/kts" - state: absent - -- name: Initialize passive Key Trustee Server - hosts: kts_passive - become: yes - gather_facts: no - tasks: - - shell: ktadmin init --confdir {{ keytrustee_server_conf_dir }} - -- name: Configure Key Trustee Server HA - hosts: localhost - connection: local - become: yes - gather_facts: no - roles: - - deployment/services/kts_high_availability diff --git a/site.yml b/site.yml deleted file mode 100644 index 5c45ab13..00000000 --- a/site.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -- import_playbook: verify_inventory_and_definition.yml -- import_playbook: create_infrastructure.yml -- import_playbook: verify_parcels.yml -- import_playbook: create_freeipa.yml - when: krb5_kdc_type == 'Red Hat IPA' -- import_playbook: prepare_nodes.yml -- import_playbook: prepare_tls.yml -- import_playbook: nifi_workaround.yml -- import_playbook: install_cloudera_manager.yml -- import_playbook: set_cloudera_manager_password.yml -- import_playbook: prepare_security.yml -- import_playbook: install_cluster.yml -- import_playbook: setup_hdfs_encryption.yml diff --git a/teardown_tls.yml b/teardown_tls.yml deleted file mode 100644 index 46ce7613..00000000 --- a/teardown_tls.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Group hosts based on whether TLS flag is set in inventory - hosts: all - gather_facts: no - tasks: - - group_by: - key: "{{ 'tls' if tls | default(False) else 'no_tls' }}" - -- name: Teardown CA server - hosts: ca_server - gather_facts: no - become: yes - tasks: - - name: Delete CA data - file: - name: "{{ ca_server_root_path }}" - state: absent - vars_files: - - vars/ca_server.yml - -- name: Teardown security artifact directories - hosts: tls - gather_facts: no - become: yes - tasks: - - name: Delete security artifacts - file: - name: "{{ base_dir_security }}" - state: absent diff --git a/verify_inventory_and_definition.yml b/verify_inventory_and_definition.yml deleted file mode 100644 index 92bd4578..00000000 --- a/verify_inventory_and_definition.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Group hosts using host template information - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts.yml - -- name: Verify inventory [verify_inventory] - hosts: localhost - connection: local - gather_facts: no - roles: - - verify/inventory - tags: - - verify - - verify_inventory - -- name: Verify definition [verify_definition] - hosts: localhost - connection: local - gather_facts: no - roles: - - verify/definition - tags: - - verify - - verify_definition diff --git a/verify_parcels.yml b/verify_parcels.yml deleted file mode 100644 index 14a81647..00000000 --- a/verify_parcels.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 Cloudera, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- - -- name: Group hosts using host template information - hosts: all - gather_facts: no - tasks: - - import_tasks: tasks/group_hosts.yml - -- name: Verify definition [verify_parcels] - hosts: localhost - connection: local - gather_facts: no - roles: - - verify/parcels_and_roles - tags: - - verify - - verify_parcels