[PLAT-13893] added auto create provider via YNP in case not exist

Summary: This diff enhances the module for node-agent that auto creates the provider / register the node to the provider in YBA.

Test Plan:
Created a ubuntu VM
Ran the provision on the VM using the script.
Deployed universe - Verified that universe creation is successful.

Reviewers: anijhawan, nbhatia

Reviewed By: anijhawan

Subscribers: anijhawan, yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D36291

managed/node-agent/resources/node-agent-provision.yaml

@@ -3,16 +3,15 @@ ynp:
   yb_home_dir: /home/yugabyte
   # NTP servers for the node
   chrony_servers: ["0.pool.ntp.org", "1.pool.ntp.org"]
-  yb_user: yugabyte
   yb_user_id: 994
-  yb_user_password: password
   # Public key path for the key to be used for yugabyte user.
   # Default's to current user public key.
   public_key_filepath: ""
   # Airgapped
   is_airgap: false
   use_system_level_systemd: false
   ip_address: "127.0.0.1"
+  tmp_directory: /tmp
 
 yba:
   url: <url>
@@ -26,9 +25,10 @@ yba:
       name: region_name
       zone:
         name: zone_name
+    access_key_path: <key_content>
   instance_type:
     name: instance_name
     cores: cores
     memory_size: size
     volume_size: size
-    mount_points: []
+    mount_points: /data # Comma separated values.

managed/node-agent/resources/ynp/commands/provision_command.py

@@ -5,6 +5,7 @@
 import logging
 import pkgutil
 import jinja2
+import sys
 
 import modules.base_module as mbm
 from .base_command import Command
@@ -102,6 +103,7 @@ def _check_package(self, package_manager, package_name):
             logger.info(f"{package_name} is installed.")
         except subprocess.CalledProcessError:
             logger.info(f"{package_name} is not installed.")
+            sys.exit()
 
     def _validate_required_packages(self):
         package_manager = None

managed/node-agent/resources/ynp/configs/config.j2

@@ -13,15 +13,15 @@ chrony_servers = "{{ ynp.chrony_servers | join(', ') }}"
 
 [CreateYugabyteUser]
 yb_home_dir = {{ ynp.yb_home_dir }}
-yb_user = {{ ynp.yb_user }}
+yb_user = yugabyte
 yb_user_id = {{ ynp.yb_user_id }}
-yb_user_password = {{ ynp.yb_user_password }}
+yb_user_password =
 public_key_filepath = {{ ynp.public_key_filepath }}
 
 [ConfigureSystemd]
 yb_home_dir = {{ ynp.yb_home_dir }}
-yb_user = {{ ynp.yb_user }}
-user_name = {{ ynp.yb_user }}
+yb_user = yugabyte
+user_name = yugabyte
 use_system_level_systemd = {{ ynp.use_system_level_systemd }}
 service_files = "yb-tserver.service, yb-master.service, clock-sync.sh.j2, yb-bind_check.service, yb-clean_cores.service, yb-clean_cores.timer, yb-collect_metrics.service, yb-collect_metrics.timer, yb-controller.service, yb-zip_purge_yb_logs.service, yb-zip_purge_yb_logs.timer"
 
@@ -47,6 +47,14 @@ cpu = unlimited
 nproc = 12000
 locks = unlimited
 
+[ConfigureNodeExporter]
+prometheus_user = prometheus
+yb_home_dir = {{ ynp.yb_home_dir }}
+
+[ConfigureNetwork]
+ip_address={{ ynp.ip_address }}
+ports = 7000 7100 9000 9100 18018 22 5433 9042 9070 9300 12000 13000
+
 {% macro render_section(section, prefix="") -%}
 {%- for key, value in section.items() if value != '' -%}
     {%- if value is mapping -%}
@@ -59,15 +67,7 @@ locks = unlimited
 
 [InstallNodeAgent]
 {{ render_section(yba) -}}
-yb_user = {{ ynp.yb_user }}
-
-[ConfigureNodeExporter]
-prometheus_user = prometheus
+yb_user = yugabyte
 yb_home_dir = {{ ynp.yb_home_dir }}
+tmp_directory = {{ ynp.tmp_directory }}
 
-[ConfigureNetwork]
-ip_address={{ ynp.ip_address }}
-ports = 7000 7100 9000 9100 18018 22 5433 9042 9070 9300 12000 13000
-
-[YBA]
-{{ render_section(yba) }}

managed/node-agent/resources/ynp/modules/base_module.py

@@ -5,6 +5,8 @@
 
 class BaseYnpModule:
     registry = {}
+    run_template = "run.j2"
+    precheck_template = "precheck.j2"
 
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)

managed/node-agent/resources/ynp/modules/provision/chrony/chrony.py

@@ -2,6 +2,4 @@
 
 
 class ConfigureChrony(BaseYnpModule):
-
-    run_template = "chrony_run.j2"
-    precheck_template = "chrony_precheck.j2"
+    pass

managed/node-agent/resources/ynp/modules/provision/configure_os/os_config.py

@@ -2,6 +2,4 @@
 
 
 class ConfigureOs(BaseYnpModule):
-
-    run_template = "run.j2"
-    precheck_template = "precheck.j2"
+    pass

managed/node-agent/resources/ynp/modules/provision/network/network.py

@@ -1,11 +1,5 @@
 from ...base_module import BaseYnpModule
-import jinja2
-from jinja2 import Environment, FileSystemLoader
-from utils.filters import split_servers
-import logging
 
 
 class ConfigureNetwork(BaseYnpModule):
-
-    run_template = "run.j2"
-    precheck_template = "precheck.j2"
+    pass

managed/node-agent/resources/ynp/modules/provision/node_agent/node_agent.py

@@ -1,50 +1,227 @@
 from ...base_module import BaseYnpModule
+import logging
+import time
+import json
 import requests
+import os
 
 
 class InstallNodeAgent(BaseYnpModule):
 
-    run_template = "node_agent_run.j2"
-    precheck_template = "node_agent_precheck.j2"
-
     def _get_headers(self, token):
         return {
             'Accept': 'application/json',
-            'X-AUTH-YW-API-TOKEN': f'{token}'
+            'X-AUTH-YW-API-TOKEN': f'{token}',
+            'Content-Type': 'application/json'
+        }
+
+    def _get_provider_url(self, context):
+        return f'{context.get("url")}/api/v1/customers/{context.get("customer_uuid")}/providers?name={context.get("provider_name")}'
+
+    def _get_instance_type(self, yba_url, customer_uuid, p_uuid, code):
+        return f'{yba_url}/api/v1/customers/{customer_uuid}/providers/{p_uuid}/instance_types/{code}'
+
+    def _generate_provider_payload(self, context):
+        # Generates the body for provider payload.
+        time_stamp = int(time.time())
+        provider_data = {
+            "name": context.get("provider_name", f'onprem{time_stamp}'),
+            "code": "onprem",
+            "details": {
+                "skipProvisioning": True,
+                "cloudInfo": {
+                    "onprem": {
+                        "ybHomeDir": context.get("yb_home_dir", "/home/yugabyte")
+                    }
+                }
+            },
+            "regions": []
+        }
+        region = {
+            "name": context.get("provider_region_name"),
+            "code": context.get("provider_region_name"),
+            "zones": [{
+                "name": context.get("provider_region_zone_name"),
+                "code": context.get("provider_region_zone_name")
+            }]
+        }
+        provider_data["regions"].append(region)
+        if context.get("provider_access_key_path") is not None:
+            with open(context.get("provider_access_key_path"), 'r') as file:
+                provider_access_key = file.read().strip()
+            provider_data["allAccessKeys"] = [{
+                "keyInfo": {
+                    "keyPairName": f'onprem_key_{time_stamp}.pem',
+                    "sshPrivateKeyContent": provider_access_key,
+                    "skipKeyValidateAndUpload": False
+                }
+            }]
+
+        return provider_data
+
+    def _generate_provider_update_payload(self, context, provider):
+        regions = provider.get('regions', [])
+        region_exist = False
+        for region in regions:
+            if region['code'] == context.get('provider_region_name'):
+                region_exist = True
+                zones = region.get('zones', [])
+                zone_exist = False
+                for zone in zones:
+                    if zone['code'] == context.get('provider_region_zone_name'):
+                        zone_exist = True
+                if not zone_exist:
+                    # Append the zone in the region.
+                    zones.append({
+                        "name": context.get("provider_region_zone_name"),
+                        "code": context.get("provider_region_zone_name")
+                    })
+                region['zones'] = zones
+        if not region_exist:
+            # Append the region in the provider.
+            regions.append({
+                "name": context.get("provider_region_name"),
+                "code": context.get("provider_region_name"),
+                "zones": [{
+                    "name": context.get("provider_region_zone_name"),
+                    "code": context.get("provider_region_zone_name")
+                }]
+            })
+        provider['regions'] = regions
+        return provider
+
+    def _generate_instance_type_payload(self, context):
+        time_stamp = int(time.time())
+        instance_data = {
+            'idKey': {
+                'instanceTypeCode': context.get('instance_type_name')
+            },
+            'providerUuid': "$provider_id",
+            'providerCode': 'onprem',
+            'numCores': context.get('instance_type_cores'),
+            'memSizeGB': context.get('instance_type_memory_size'),
+            'instanceTypeDetails': {
+                'volumeDetailsList': []
+            }
+        }
+        mount_points = context.get('instance_type_mount_points').split(',')
+        for mp in mount_points:
+            volume_detail = {
+                'volumeSizeGB': context.get('instance_type_volume_size'),
+                'volumeType': 'SSD',
+                'mountPath': mp
+            }
+            instance_data['instanceTypeDetails']['volumeDetailsList'].append(volume_detail)
+
+        return instance_data
+
+    def _generate_add_node_payload(self, context):
+        node_add_payload = {
+            "nodes": [
+                {
+                    "instanceType": context.get('instance_type_name'),
+                    "ip": context.get('node_ip'),
+                    "region": context.get('provider_region_name'),
+                    "zone": context.get('provider_region_zone_name')
+                }
+            ]
         }
 
-    def _get_provider_url(self, yba_url, customer_uuid, name):
-        return f'{yba_url}/api/v1/customers/{customer_uuid}/providers?name={name}'
+        return node_add_payload
+
+    def _get_provider(self, context):
+        provider_url = self._get_provider_url(context)
+        yba_url = context.get('url')
+        skip_tls_verify = not yba_url.lower().startswith('https')
+        response = requests.get(provider_url,
+                                headers=self._get_headers(context.get('api_key')),
+                                verify=skip_tls_verify)
+        return response
 
+    def _create_instance_if_not_exists(self, context, provider):
+        yba_url = context.get('url')
+        skip_tls_verify = not yba_url.lower().startswith('https')
+        get_instance_type_url = self._get_instance_type(context.get('url'), context.get('customer_uuid'), provider.get('uuid'), context.get('instance_type_name'))
+
+        try:
+            response = requests.get(get_instance_type_url,
+                                    headers=self._get_headers(context.get('api_key')),
+                                    verify=skip_tls_verify)
+            response.raise_for_status()
+            data = response.json()
+            if not data:  # If the instance type does not exist
+                raise ValueError("Instance type does not exist")
+        except requests.exceptions.HTTPError as http_err:
+            if response.status_code == 400:
+                logging.info("Instance type does not exist, creating it.")
+                instance_data = self._generate_instance_type_payload(context, provider['uuid'])
+
+                instance_payload_file = os.path.join(context.get('tmp_directory'), 'create_instance.json')
+                with open(instance_payload_file, 'w') as f:
+                    json.dump(instance_data, f, indent=4)
+            else:
+                logging.error(f"Request error: {http_err}")
+        except requests.exceptions.RequestException as req_err:
+            logging.error(f"Request error: {req_err}")
+        except ValueError as json_err:
+            logging.error(f"Error parsing JSON response: {json_err}")
+
     def render_templates(self, context):
         node_agent_enabled = False
         yba_url = context.get('url')
         skip_tls_verify = not yba_url.lower().startswith('https')
-        provider_url = self._get_provider_url(yba_url,
-                                              context.get('customer_uuid'),
-                                              context.get('provider_name'))
 
         try:
             # Make the GET request
-            response = requests.get(provider_url,
-                                    headers=self._get_headers(context.get('api_key')),
-                                    verify=skip_tls_verify)
+            response = self._get_provider(context)
             response.raise_for_status()  # Raise an HTTPError for bad responses (4xx and 5xx)
 
             # Parse the JSON response
             try:
                 provider_data = response.json()
                 if isinstance(provider_data, list) and len(provider_data) > 0:
-                    provider_details = provider_data[0].get('details', {})
-                    context['provider_id'] = str(provider_data[0].get('uuid'))
+                    provider_data = provider_data[0]
+                    provider_details = provider_data.get('details', {})
+                    regions = provider_data.get('regions', [])
+                    region_exists = False
+                    for region in regions:
+                        if region['code'] == context.get('provider_region_name'):
+                            region_exists = True
+                            zones = region.get('zones', [])
+                            zone_exist = False
+                            for zone in zones:
+                                if zone['code'] == context.get('provider_region_zone_name'):
+                                    zone_exist = True
+                    if not region_exists or not zone_exist:
+                        update_provider_data = self._generate_provider_update_payload(context, provider_data)
+                        update_provider_data_file = os.path.join(context.get('tmp_directory'), 'update_provider.json')
+                        with open(update_provider_data_file, 'w') as f:
+                            json.dump(update_provider_data, f, indent=4)
+                    self._create_instance_if_not_exists(context, provider_data)
+                    context['provider_id'] = str(provider_data.get('uuid'))
                     node_agent_enabled = provider_details.get('enableNodeAgent', False)
                 else:
-                    print("No provider data found.")
-                    # Todo: Create provider in case it does not exist.
+                    logging.info("Generating provider create payload...")
+                    provider_payload = self._generate_provider_payload(context)
+                    provider_payload_file = os.path.join(context.get('tmp_directory'), 'create_provider.json')
+                    with open(provider_payload_file, 'w') as f:
+                        json.dump(provider_payload, f, indent=4)
+                    instance_create_payload = self._generate_instance_type_payload(context)
+                    instance_payload_file = os.path.join(context.get('tmp_directory'), 'create_instance.json')
+                    with open(instance_payload_file, 'w') as f:
+                        json.dump(instance_create_payload, f, indent=4)
+                    node_agent_enabled = True
+
             except ValueError as json_err:
-                print(f"Error parsing JSON response: {json_err}")
+                logging.error(f"Error parsing JSON response: {json_err}")
         except requests.exceptions.RequestException as req_err:
-            print(f"Request error: {req_err}")
+            logging.error(f"Request error: {req_err}")
+
+
+        add_node_payload = self._generate_add_node_payload(context)
+        add_node_payload_file = os.path.join(context.get('tmp_directory'), 'add_node_to_provider.json')
+        with open(add_node_payload_file, 'w') as f:
+            json.dump(add_node_payload, f, indent=4)
 
         if node_agent_enabled:
             return super().render_templates(context)