Added TDE Migration cmdlet (#6173)

* Added TDE Migration cmdlet

* Added logic to extract console app from nuget

* Removed TODO

* Renamed variable

* Download console app NuGet from NuGet.org

* Updated README.md and changelog

* ran azdev style

* Updated help and README.md

* Added prompt for inputs

* Set values

* Fixed build issue

* Fixed linter issues

* Raise error

src/datamigration/HISTORY.rst

@@ -3,6 +3,11 @@
 Release History
 ===============
 
+=======
+0.5.0
+++++++
+* [NEW COMMAND] `az datamigration tde-migration` : Migrate TDE certificate from source SQL Server to the target Azure SQL Server.
+
 0.4.1
 ++++++
 * Bug fix for list-logins parameter in command "az datamigration login-migration".

src/datamigration/README.md

@@ -34,6 +34,11 @@ az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutpu
 az datamigration login-migration --src-sql-connection-str  "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --tgt-sql-connection-str  "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --csv-file-path "C:\\CSVFile" --list-of-login "loginname1" "loginname2" --output-folder "C:\\OutputFolder" --aad-domain-name "AADDomainName" --display-result --overwrite
 ```
 
+##### TDE-migration #####
+```
+az datamigration tde-migration --source-sql-connection-string "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --target-subscription-id "00000000-0000-0000-0000-000000000000" --target-resource-group-name "ResourceGroupName" --target-managed-instance-name "TargetManagedInstanceName" --network-share-path "\\NetworkShare\Folder" --network-share-domain "" --network-share-user-name "NetworkShareUserName" --network-share-password "" --database-name "TdeDb_0" "TdeDb_1" "TdeDb_2"
+```
+
 #### datamigration sql-managed-instance ####
 ##### Create (Backup source Fileshare) #####
 ```

src/datamigration/azext_datamigration/manual/_help.py

@@ -69,6 +69,15 @@
                az datamigration login-migration --config-file-path "C:\\Users\\user\\document\\config.json"
 """
 
+helps['datamigration tde-migration'] = """
+    type: command
+    short-summary: Migrate TDE certificate from source SQL Server to the target Azure SQL Server.
+    examples:
+      - name: Migrate TDE certificate from source SQL Server to the target Azure SQL Server.
+        text: |-
+               az datamigration tde-migration --source-sql-connection-string "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --target-subscription-id "00000000-0000-0000-0000-000000000000" --target-resource-group-name "ResourceGroupName" --target-managed-instance-name "TargetManagedInstanceName" --network-share-path "\\NetworkShare\Folder" --network-share-domain "NetworkShare" --network-share-user-name "NetworkShareUserName" --network-share-password "" --database-name "TdeDb_0" "TdeDb_1" "TdeDb_2"
+"""
+
 helps['datamigration register-integration-runtime'] = """
     type: command
     short-summary: Register Database Migration Service on Integration Runtime

src/datamigration/azext_datamigration/manual/_params.py

@@ -63,6 +63,17 @@ def load_arguments(self, _):
         c.argument('aad_domain_name', type=str, help='Required if Windows logins are included in the list of logins to be migrated.')
         c.argument('config_file_path', type=file_type, completer=FilesCompleter(), help='Path of the ConfigFile')
 
+    with self.argument_context('datamigration tde-migration') as c:
+        c.argument('source_sql_connection_string', options_list=["--source-sql-connection-string", "--srcsqlcs"], type=str, help='Connection string for the source SQL instance, using the formal connection string format.')
+        c.argument('target_subscription_id', options_list=["--target-subscription-id", "--tgtsubscription"], type=str, help='Subscription Id of the target Azure SQL server.')
+        c.argument('target_resource_group_name', options_list=["--target-resource-group-name", "--tgtrg"], type=str, help='Resource group name of the target Azure SQL server.')
+        c.argument('target_managed_instance_name', options_list=["--target-managed-instance-name", "--tgtname"], type=str, help='Name of the Azure SQL Server.')
+        c.argument('network_share_path', options_list=["--network-share-path", "--networkpath"], type=str, help='Network share path.')
+        c.argument('network_share_domain', options_list=["--network-share-domain", "--networkdomain"], type=str, help='Network share domain.')
+        c.argument('network_share_user_name', options_list=["--network-share-user-name", "--networkuser"], type=str, help='Network share user name.')
+        c.argument('network_share_password', options_list=["--network-share-password", "--networkpw"], type=str, help='Network share password.')
+        c.argument('database_name', nargs='+', options_list=["--database-name", "--dbname"], help='Source database name.')
+
     with self.argument_context('datamigration register-integration-runtime') as c:
         c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service')
         c.argument('ir_path', type=str, help='Path of Integration Runtime MSI')

src/datamigration/azext_datamigration/manual/commands.py

@@ -20,3 +20,4 @@ def load_command_table(self, _):
         g.custom_command('get-sku-recommendation', 'datamigration_get_sku_recommendation')
         g.custom_command('register-integration-runtime', 'datamigration_register_ir')
         g.custom_command('login-migration', 'datamigration_login_migration')
+        g.custom_command('tde-migration', 'datamigration_tde_migration')

src/datamigration/azext_datamigration/manual/custom.py

@@ -14,6 +14,7 @@
 import os
 import signal
 import subprocess
+from knack.prompting import prompt_pass
 from azure.cli.core.azclierror import MutuallyExclusiveArgumentError
 from azure.cli.core.azclierror import RequiredArgumentMissingError
 from azure.cli.core.azclierror import UnclassifiedUserFault
@@ -299,3 +300,44 @@ def datamigration_login_migration(src_sql_connection_str=None,
 
     except Exception as e:
         raise e
+
+
+# -----------------------------------------------------------------------------------------------------------------
+#  Migrate TDE certificate from source SQL Server to the target Azure SQL Server.
+# -----------------------------------------------------------------------------------------------------------------
+def datamigration_tde_migration(source_sql_connection_string=None,
+                                target_subscription_id=None,
+                                target_resource_group_name=None,
+                                target_managed_instance_name=None,
+                                network_share_path=None,
+                                network_share_domain=None,
+                                network_share_user_name=None,
+                                network_share_password=None,
+                                database_name=None):
+
+    if source_sql_connection_string is None:
+        source_sql_connection_string = prompt_pass('Connection String:', confirm=False)
+
+    if network_share_password is None:
+        network_share_password = prompt_pass('Network Share Password:', confirm=False)
+
+    try:
+        # Setup the console app
+        exePath = helper.tdeMigration_console_app_setup()
+
+        if exePath is None:
+            return
+
+        if os.path.exists(exePath) is False:
+            print("Failed to locate executable.")
+            return
+
+        cmd = f'{exePath} --sourceSqlConnectionString "{source_sql_connection_string}" --targetSubscriptionId "{target_subscription_id}" --targetResourceGroupName "{target_resource_group_name}" --targetManagedInstanceName "{target_managed_instance_name}" --networkSharePath "{network_share_path}" --networkShareDomain "{network_share_domain}" --networkShareUserName "{network_share_user_name}" --networkSharePassword "{network_share_password}" --databaseName'
+
+        for db in database_name:
+            cmd += " \"" + db + "\""
+
+        subprocess.call(cmd, shell=False)
+
+    except Exception as e:
+        raise e

src/datamigration/azext_datamigration/manual/helper.py

@@ -19,6 +19,9 @@
 import time
 import urllib.request
 from zipfile import ZipFile
+import shutil
+import requests
+from knack.util import CLIError
 from azure.cli.core.azclierror import CLIInternalError
 from azure.cli.core.azclierror import FileOperationError
 from azure.cli.core.azclierror import InvalidArgumentValueError
@@ -102,6 +105,30 @@ def loginMigration_console_app_setup():
     return defaultOutputFolder, exePath
 
 
+# -----------------------------------------------------------------------------------------------------------------
+# TdeMigration helper function to do console app setup (mkdir, download and extract)
+# -----------------------------------------------------------------------------------------------------------------
+def tdeMigration_console_app_setup():
+    validate_os_env()
+
+    # Create downloads directory if it doesn't already exists
+    default_output_folder = get_tdeMigration_default_output_folder()
+    downloads_folder = os.path.join(default_output_folder, "Downloads")
+    os.makedirs(downloads_folder, exist_ok=True)
+
+    # check and download console app
+    console_app_version = check_and_download_tdeMigration_console_app(downloads_folder)
+    if console_app_version is None:
+        raise CLIError("Connection to NuGet.org required. Please check connection and try again.")
+
+    # Assigning base folder path\TdeConsoleApp\console
+    console_app_location = os.path.join(downloads_folder, console_app_version)
+
+    exePath = os.path.join(console_app_location, "tools", "Microsoft.SqlServer.Migration.Tde.ConsoleApp.exe")
+
+    return exePath
+
+
 # -----------------------------------------------------------------------------------------------------------------
 # Assessment helper function to return the default output folder path depending on OS environment.
 # -----------------------------------------------------------------------------------------------------------------
@@ -136,6 +163,23 @@ def get_loginMigration_default_output_folder():
     return defaultOutputPath
 
 
+# -----------------------------------------------------------------------------------------------------------------
+# TdeMigration helper function to return the default output folder path depending on OS environment.
+# -----------------------------------------------------------------------------------------------------------------
+def get_tdeMigration_default_output_folder():
+
+    osPlatform = platform.system()
+
+    if osPlatform.__contains__('Linux'):
+        defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), ".config", "Microsoft", "SqlTdeMigrations")
+    elif osPlatform.__contains__('Darwin'):
+        defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), "Library", "Application Support", "Microsoft", "SqlTdeMigrations")
+    else:
+        defaultOutputPath = os.path.join(os.getenv('LOCALAPPDATA'), "Microsoft", "SqlTdeMigrations")
+
+    return defaultOutputPath
+
+
 # -----------------------------------------------------------------------------------------------------------------
 # Assessment helper function to check if console app exists, if not download it.
 # -----------------------------------------------------------------------------------------------------------------
@@ -170,6 +214,102 @@ def check_and_download_loginMigration_console_app(exePath, baseFolder):
             zipFile.extractall(path=baseFolder)
 
 
+# -----------------------------------------------------------------------------------------------------------------
+# Get latest version of TDE Console App on NuGet.org
+# -----------------------------------------------------------------------------------------------------------------
+def get_latest_nuget_org_version(package_id):
+    # Get Nuget.org service index
+    service_index_response = None
+    try:
+        service_index_response = requests.get("https://api.nuget.org/v3/index.json")
+    except Exception:
+        print("Unable to connect to NuGet.org to check for updates.")
+
+    if(service_index_response is None or
+       service_index_response.status_code != 200 or
+       len(service_index_response.content) > 999999):
+        return None
+
+    json_response = json.loads(service_index_response.content)
+    nuget_org_resources = json_response["resources"]
+
+    package_base_address_type = "PackageBaseAddress/3.0.0"
+    package_base_address_url = None
+
+    for resource in nuget_org_resources:
+        if resource["@type"] == package_base_address_type:
+            package_base_address_url = resource["@id"]
+            break
+
+    if package_base_address_url is None:
+        return None
+
+    package_versions_response = None
+    package_versions_response = requests.get(f"{package_base_address_url}{package_id.lower()}/index.json")
+    if(package_versions_response.status_code != 200 or len(package_versions_response.content) > 999999):
+        return None
+
+    package_versions_json = json.loads(package_versions_response.content)
+    package_versions_array = package_versions_json["versions"]
+    return package_versions_array[len(package_versions_array) - 1]
+
+
+# -----------------------------------------------------------------------------------------------------------------
+# Get latest local version of TDE Console App
+# -----------------------------------------------------------------------------------------------------------------
+def get_latest_local_name_and_version(downloads_folder):
+    nuget_versions = os.listdir(downloads_folder)
+    if len(nuget_versions) == 0:
+        return None
+
+    nuget_versions.sort(reverse=True)
+    return nuget_versions[0]
+
+
+# -----------------------------------------------------------------------------------------------------------------
+# TdeMigration helper function to check if console app exists, if not download it.
+# -----------------------------------------------------------------------------------------------------------------
+def check_and_download_tdeMigration_console_app(downloads_folder):
+    # Get latest TdeConsoleApp name from nuget.org
+    package_id = "Microsoft.SqlServer.Migration.TdeConsoleApp"
+
+    latest_nuget_org_version = get_latest_nuget_org_version(package_id)
+    latest_nuget_org_name_and_version = f"{package_id}.{latest_nuget_org_version}"
+    latest_local_name_and_version = get_latest_local_name_and_version(downloads_folder)
+
+    if latest_nuget_org_version is None:
+        # Cannot retrieve latest version on NuGet.org, return latest local version
+        return latest_local_name_and_version
+
+    if latest_local_name_and_version is not None and latest_local_name_and_version >= latest_nuget_org_name_and_version:
+        # Console app is up to date, do not download update
+        return latest_local_name_and_version
+
+    # Download latest version of NuGet
+    latest_nuget_org_download_directory = os.path.join(downloads_folder, latest_nuget_org_name_and_version)
+    os.makedirs(latest_nuget_org_download_directory, exist_ok=True)
+
+    latest_nuget_org_download_name = f"{latest_nuget_org_download_directory}/{latest_nuget_org_name_and_version}.nupkg"
+    download_url = f"https://www.nuget.org/api/v2/package/{package_id}/{latest_nuget_org_version}"
+
+    # Extract downloaded NuGet
+    urllib.request.urlretrieve(download_url, filename=latest_nuget_org_download_name)
+    with ZipFile(latest_nuget_org_download_name, 'r') as zipFile:
+        zipFile.extractall(path=latest_nuget_org_download_directory)
+
+    exe_path = os.path.join(latest_nuget_org_download_directory, "tools", "Microsoft.SqlServer.Migration.Tde.ConsoleApp.exe")
+
+    if os.path.exists(exe_path):
+        for nuget_version in os.listdir(downloads_folder):
+            if nuget_version != latest_nuget_org_name_and_version:
+                shutil.rmtree(os.path.join(downloads_folder, nuget_version))
+
+    else:
+        return latest_local_name_and_version
+
+    return latest_nuget_org_name_and_version
+
+
 # -----------------------------------------------------------------------------------------------------------------
 # Assessment helper function to check if baseFolder exists, if not create it.
 # -----------------------------------------------------------------------------------------------------------------

src/datamigration/setup.py

@@ -10,7 +10,7 @@
 from setuptools import setup, find_packages
 
 # HISTORY.rst entry.
-VERSION = '0.4.1'
+VERSION = '0.5.0'
 try:
     from azext_datamigration.manual.version import VERSION
 except ImportError: