#120 Fixed the deployment tests

exasol_sagemaker_extension/deployment/deploy_cli.py

@@ -1,34 +1,77 @@
+from __future__ import annotations
+
 import click
 import logging
+from exasol.python_extension_common.deployment.language_container_deployer_cli import (
+    SecretParams, SECRET_DISPLAY, secret_callback)
+
 from exasol_sagemaker_extension.deployment.deploy_create_statements \
     import DeployCreateStatements
 
 
 @click.command()
-@click.option('--host', type=str, required=True, help="db host address")
-@click.option('--port', type=str, required=True, help="db host port")
-@click.option('--user', type=str, required=True, help="db user name")
-@click.option('--pass', 'pwd', required=True, help="db user password")
+@click.option('--dsn', type=str)
+@click.option('--db-user', type=str)
+@click.option(f'--{SecretParams.DB_PASSWORD.value}', type=str,
+              prompt='DB password', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--saas-url', type=str,
+              default='https://cloud.exasol.com')
+@click.option(f'--{SecretParams.SAAS_ACCOUNT_ID.value}', type=str,
+              prompt='SaaS account id', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option(f'--{SecretParams.SAAS_DATABASE_ID.value}', type=str,
+              prompt='SaaS database id', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--saas-database-name', type=str)
+@click.option(f'--{SecretParams.SAAS_TOKEN.value}', type=str,
+              prompt='SaaS token', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--ssl-cert-path', type=str, default="")
+@click.option('--ssl-client-cert-path', type=str, default="")
+@click.option('--ssl-client-private-key', type=str, default="")
+@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
 @click.option('--schema', type=str, required=True, help="schema name")
-@click.option('--print', 'verbose', type=bool, required=False,
+@click.option('--print', 'verbose', type=bool, default=False,
               is_flag=True, help="print out statements")
-@click.option('--develop', type=bool, required=False,
+@click.option('--develop', type=bool, default=False,
               is_flag=True, help="generate and execute the scripts")
-def main(host: str, port: str, user: str, pwd: str, schema: str,
-         verbose: bool = False, develop: bool = False):
+def main(
+        dsn: str | None,
+        db_user: str | None,
+        db_password: str | None,
+        saas_url: str | None,
+        saas_account_id: str | None,
+        saas_database_id: str | None,
+        saas_database_name: str | None,
+        saas_token: str | None,
+        ssl_cert_path: str,
+        ssl_client_cert_path: str,
+        ssl_client_private_key: str,
+        use_ssl_cert_validation: bool,
+        schema: str,
+        verbose: bool,
+        develop: bool):
 
     logging.basicConfig(format='%(asctime)s - %(module)s  - %(message)s',
                         level=logging.DEBUG)
 
     DeployCreateStatements.create_and_run(
-        db_host=host,
-        db_port=port,
-        db_user=user,
-        db_pass=pwd,
         schema=schema,
+        dsn=dsn,
+        db_user=db_user,
+        db_password=db_password,
+        saas_url=saas_url,
+        saas_account_id=saas_account_id,
+        saas_database_id=saas_database_id,
+        saas_database_name=saas_database_name,
+        saas_token=saas_token,
+        use_ssl_cert_validation=use_ssl_cert_validation,
+        ssl_trusted_ca=ssl_cert_path,
+        ssl_client_certificate=ssl_client_cert_path,
+        ssl_private_key=ssl_client_private_key,
         to_print=verbose,
-        develop=develop
-    )
+        develop=develop)
 
 
 if __name__ == "__main__":

exasol_sagemaker_extension/deployment/deploy_create_statements.py

@@ -1,7 +1,10 @@
+from __future__ import annotations
 import logging
-import ssl
 
 import pyexasol
+from exasol.python_extension_common.deployment.language_container_deployer import get_websocket_sslopt
+from exasol.saas.client.api_access import get_connection_params
+
 from exasol_sagemaker_extension.deployment import constants
 from exasol_sagemaker_extension.deployment. \
     generate_create_statement_autopilot_endpoint_deletion \
@@ -120,39 +123,59 @@ def create_statements():
 
     @classmethod
     def create_and_run(cls,
-               db_host: str,
-               db_port: str,
-               db_user: str,
-               db_pass: str,
-               schema: str,
-               to_print: bool,
-               develop: bool):
+                       schema: str,
+                       dsn: str | None = None,
+                       db_user: str | None = None,
+                       db_password: str | None = None,
+                       saas_url: str | None = None,
+                       saas_account_id: str | None = None,
+                       saas_database_id: str | None = None,
+                       saas_database_name: str | None = None,
+                       saas_token: str | None = None,
+                       use_ssl_cert_validation: bool = True,
+                       ssl_trusted_ca: str | None = None,
+                       ssl_client_certificate: str | None = None,
+                       ssl_private_key: str | None = None,
+                       to_print: bool = False,
+                       develop: bool = False):
         """
         Creates a database connection object based on the provided credentials
         Creates an instance of the DeployCreateStatements passing the connection
         object to it and calls its run method.
 
         Parameters:
-            db_host     - database host address
-            db_port     - database port
-            db_user     - database username
-            db_pass     - the user password
             schema      - schema where the scripts should be created
+            dsn         - database address including the port
+            db_user     - database username
+            db_password - the user password
             to_print    - if True the script creation SQL commands will be
                           printed rather than executed
             develop     - if True the scripts will be generated from scratch
         """
 
-        exasol_conn = pyexasol.connect(
-            dsn=f"{db_host}:{db_port}",
-            user=db_user,
-            password=db_pass,
-            compression=True,
-            encryption=True,
-            websocket_sslopt={
-                "cert_reqs": ssl.CERT_NONE,
-            }
-        )
+        # Infer where the database is - on-prem or SaaS.
+        if all((dsn, db_user, db_password)):
+            connection_params = {'dsn': dsn, 'user': db_user, 'password': db_password}
+        elif all((saas_url, saas_account_id, saas_token,
+                  any((saas_database_id, saas_database_name)))):
+            connection_params = get_connection_params(host=saas_url,
+                                                      account_id=saas_account_id,
+                                                      database_id=saas_database_id,
+                                                      database_name=saas_database_name,
+                                                      pat=saas_token)
+        else:
+            raise ValueError('Incomplete parameter list. '
+                             'Please either provide the parameters [dns, db_user, db_password] '
+                             'for an On-Prem database or [saas_url, saas_account_id, '
+                             'saas_database_id, saas_token] for a SaaS database.')
+
+        websocket_sslopt = get_websocket_sslopt(use_ssl_cert_validation, ssl_trusted_ca,
+                                                ssl_client_certificate, ssl_private_key)
+
+        exasol_conn = pyexasol.connect(**connection_params,
+                                       encryption=True,
+                                       websocket_sslopt=websocket_sslopt,
+                                       compression=True)
 
         deployer = cls(exasol_conn, schema, to_print, develop)
         deployer.run()

tests/ci_tests/utils/build_language_container.py

@@ -0,0 +1,92 @@
+import pyexasol
+import subprocess
+from pathlib import Path
+import exasol.bucketfs as bfs
+from exasol.python_extension_common.deployment.language_container_deployer import (
+    LanguageContainerDeployer, LanguageActivationLevel)
+from exasol.python_extension_common.deployment.language_container_validator import (
+    wait_language_container, temp_schema)
+
+from tests.integration_tests.utils.parameters import db_params
+
+LANGUAGE_ALIAS = "PYTHON3_SME"
+
+
+def find_script(script_name: str):
+    current_path = Path("../..").absolute()
+    script_path = None
+    while current_path != current_path.root:
+        script_path = Path(current_path, script_name)
+        if script_path.exists():
+            break
+        current_path = current_path.parent
+    if script_path.exists():
+        return script_path
+    else:
+        raise RuntimeError("Could not find build_language_container.sh")
+
+
+def build_language_container() -> Path:
+    """
+    Returns the path of the built container
+    """
+    script_dir = find_script("build_language_container.sh")
+    completed_process = subprocess.run(
+        [script_dir], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = completed_process.stdout.decode("UTF-8")
+    print(output)
+    completed_process.check_returncode()
+
+    lines = output.splitlines()
+    container_path_selector = "Cached container under "
+    container_path = \
+        next(line for line in lines if line.startswith(container_path_selector))
+    container_path = container_path[len(container_path_selector):]
+    return Path(container_path)
+
+
+def upload_language_container(db_conn: pyexasol.ExaConnection,
+                              bucketfs_location: bfs.path.PathLike) -> None:
+
+    container_path = build_language_container()
+    bucket_file_path = container_path.name
+    deployer = LanguageContainerDeployer(pyexasol_connection=db_conn,
+                                         language_alias=LANGUAGE_ALIAS,
+                                         bucketfs_path=bucketfs_location)
+
+    deployer.upload_container(container_file=container_path,
+                              bucket_file_path=bucket_file_path)
+    deployer.activate_container(bucket_file_path=bucket_file_path,
+                                alter_type=LanguageActivationLevel.Session)
+
+    with temp_schema(db_conn) as schema:
+        wait_language_container(db_conn, LANGUAGE_ALIAS, schema)
+
+
+def upload_language_container_onprem(db_conn: pyexasol.ExaConnection) -> None:
+
+    bucketfs_location = bfs.path.build_path(backend=bfs.path.StorageBackend.onprem,
+                                            url=f'http://{db_params.host}:{db_params.bfs_port}',
+                                            username=db_params.bfs_user,
+                                            password=db_params.bfs_password,
+                                            verify=False,
+                                            bucket_name='default',
+                                            service_name='bfsdefault',
+                                            path='container')
+
+    upload_language_container(db_conn, bucketfs_location)
+
+
+def upload_language_container_saas(db_conn: pyexasol.ExaConnection,
+                                   saas_url: str,
+                                   saas_account_id: str,
+                                   saas_database_id: str,
+                                   saas_token: str) -> None:
+
+    bucketfs_location = bfs.path.build_path(backend=bfs.path.StorageBackend.saas,
+                                            url=saas_url,
+                                            account_id=saas_account_id,
+                                            database_id=saas_database_id,
+                                            pat=saas_token)
+
+    upload_language_container(db_conn, bucketfs_location)

tests/conftest.py

@@ -1,8 +1,8 @@
 pytest_plugins = [
     "tests.fixtures.aws_connection_fixture",
     "tests.fixtures.database_connection_fixture",
-    "tests.fixtures.build_language_container_fixture",
     "tests.fixtures.setup_database_fixture",
     "tests.fixtures.prepare_environment_fixture",
+    "tests.fixtures.script_deployment_fixture",
     "tests.fixtures.setup_ci_test_environment"
 ]

tests/deployment/test_deploy_cli.py

@@ -1,6 +1,11 @@
+from __future__ import annotations
+from typing import Any
+
+import pytest
 from click.testing import CliRunner
+import exasol.bucketfs as bfs
+
 from exasol_sagemaker_extension.deployment import deploy_cli
-from tests.integration_tests.utils.parameters import db_params
 
 DB_SCHEMA = "TEST_CLI_SCHEMA"
 AUTOPILOT_TRAINING_LUA_SCRIPT_NAME = \
@@ -34,14 +39,17 @@ def get_all_scripts(db_conn):
     return list(map(lambda x: x[0], all_scripts))
 
 
-def test_deploy_cli_main(db_conn, register_language_container):
-    args_list = [
-        "--host", db_params.host,
-        "--port", db_params.port,
-        "--user", db_params.user,
-        "--pass", db_params.password,
-        "--schema", DB_SCHEMA
-    ]
+@pytest.mark.parametrize("db_conn,deploy_params", [
+    (bfs.path.StorageBackend.onprem, bfs.path.StorageBackend.onprem),
+    (bfs.path.StorageBackend.saas, bfs.path.StorageBackend.saas)
+], indirect=True)
+def test_deploy_cli_main(db_conn, deploy_params):
+
+    args_list: list[Any] = []
+    for param_name, param_value in deploy_params.items():
+        args_list.append(f'--{param_name}')
+        args_list.append(param_value)
+
     runner = CliRunner()
     result = runner.invoke(deploy_cli.main, args_list)
     assert result.exit_code == 0

tests/deployment/test_deploy_create_statements.py

@@ -1,6 +1,8 @@
+import pytest
+import exasol.bucketfs as bfs
+
 from exasol_sagemaker_extension.deployment.deploy_create_statements import \
     DeployCreateStatements
-from tests.integration_tests.utils.parameters import db_params
 
 DB_SCHEMA = "TEST_DEPLOY_SCHEMA"
 AUTOPILOT_TRAINING_LUA_SCRIPT_NAME = \
@@ -26,16 +28,13 @@ def get_all_scripts(db_conn):
     return list(map(lambda x: x[0], all_scripts))
 
 
-def test_deploy_create_statements(db_conn, register_language_container):
-    DeployCreateStatements.create_and_run(
-        db_host=db_params.host,
-        db_port=db_params.port,
-        db_user=db_params.user,
-        db_pass=db_params.password,
-        schema=DB_SCHEMA,
-        to_print=False,
-        develop=False
-    )
+@pytest.mark.parametrize("db_conn,deploy_params", [
+    (bfs.path.StorageBackend.onprem, bfs.path.StorageBackend.onprem),
+    (bfs.path.StorageBackend.saas, bfs.path.StorageBackend.saas)
+], indirect=True)
+def test_deploy_create_statements(db_conn, deploy_params):
+
+    DeployCreateStatements.create_and_run(**deploy_params)
 
     all_schemas = get_all_schemas(db_conn)
     all_scripts = get_all_scripts(db_conn)