From 3cb59b7390ff6002c13b59431c601e9c504e7234 Mon Sep 17 00:00:00 2001 From: Steven Marturano <71411280+smartura@users.noreply.github.com> Date: Tue, 23 May 2023 21:49:00 -0400 Subject: [PATCH] Added TDE Migration cmdlet (#6173) * Added TDE Migration cmdlet * Added logic to extract console app from nuget * Removed TODO * Renamed variable * Download console app NuGet from NuGet.org * Updated README.md and changelog * ran azdev style * Updated help and README.md * Added prompt for inputs * Set values * Fixed build issue * Fixed linter issues * Raise error --- src/datamigration/HISTORY.rst | 5 + src/datamigration/README.md | 5 + .../azext_datamigration/manual/_help.py | 9 ++ .../azext_datamigration/manual/_params.py | 11 ++ .../azext_datamigration/manual/commands.py | 1 + .../azext_datamigration/manual/custom.py | 42 ++++++ .../azext_datamigration/manual/helper.py | 140 ++++++++++++++++++ src/datamigration/setup.py | 2 +- 8 files changed, 214 insertions(+), 1 deletion(-) diff --git a/src/datamigration/HISTORY.rst b/src/datamigration/HISTORY.rst index 696e1860862..92ea6e76fe5 100644 --- a/src/datamigration/HISTORY.rst +++ b/src/datamigration/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +======= +0.5.0 +++++++ +* [NEW COMMAND] `az datamigration tde-migration` : Migrate TDE certificate from source SQL Server to the target Azure SQL Server. + 0.4.1 ++++++ * Bug fix for list-logins parameter in command "az datamigration login-migration". diff --git a/src/datamigration/README.md b/src/datamigration/README.md index 5b78810c848..1313735a345 100644 --- a/src/datamigration/README.md +++ b/src/datamigration/README.md @@ -34,6 +34,11 @@ az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutpu az datamigration login-migration --src-sql-connection-str "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --tgt-sql-connection-str "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --csv-file-path "C:\\CSVFile" --list-of-login "loginname1" "loginname2" --output-folder "C:\\OutputFolder" --aad-domain-name "AADDomainName" --display-result --overwrite ``` +##### TDE-migration ##### +``` +az datamigration tde-migration --source-sql-connection-string "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --target-subscription-id "00000000-0000-0000-0000-000000000000" --target-resource-group-name "ResourceGroupName" --target-managed-instance-name "TargetManagedInstanceName" --network-share-path "\\NetworkShare\Folder" --network-share-domain "" --network-share-user-name "NetworkShareUserName" --network-share-password "" --database-name "TdeDb_0" "TdeDb_1" "TdeDb_2" +``` + #### datamigration sql-managed-instance #### ##### Create (Backup source Fileshare) ##### ``` diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index 671b382f040..3b0ea931db6 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -69,6 +69,15 @@ az datamigration login-migration --config-file-path "C:\\Users\\user\\document\\config.json" """ +helps['datamigration tde-migration'] = """ + type: command + short-summary: Migrate TDE certificate from source SQL Server to the target Azure SQL Server. + examples: + - name: Migrate TDE certificate from source SQL Server to the target Azure SQL Server. + text: |- + az datamigration tde-migration --source-sql-connection-string "data source=servername;user id=userid;password=;initial catalog=master;TrustServerCertificate=True" --target-subscription-id "00000000-0000-0000-0000-000000000000" --target-resource-group-name "ResourceGroupName" --target-managed-instance-name "TargetManagedInstanceName" --network-share-path "\\NetworkShare\Folder" --network-share-domain "NetworkShare" --network-share-user-name "NetworkShareUserName" --network-share-password "" --database-name "TdeDb_0" "TdeDb_1" "TdeDb_2" +""" + helps['datamigration register-integration-runtime'] = """ type: command short-summary: Register Database Migration Service on Integration Runtime diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index e89f1d0033a..c8856b2beb8 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -63,6 +63,17 @@ def load_arguments(self, _): c.argument('aad_domain_name', type=str, help='Required if Windows logins are included in the list of logins to be migrated.') c.argument('config_file_path', type=file_type, completer=FilesCompleter(), help='Path of the ConfigFile') + with self.argument_context('datamigration tde-migration') as c: + c.argument('source_sql_connection_string', options_list=["--source-sql-connection-string", "--srcsqlcs"], type=str, help='Connection string for the source SQL instance, using the formal connection string format.') + c.argument('target_subscription_id', options_list=["--target-subscription-id", "--tgtsubscription"], type=str, help='Subscription Id of the target Azure SQL server.') + c.argument('target_resource_group_name', options_list=["--target-resource-group-name", "--tgtrg"], type=str, help='Resource group name of the target Azure SQL server.') + c.argument('target_managed_instance_name', options_list=["--target-managed-instance-name", "--tgtname"], type=str, help='Name of the Azure SQL Server.') + c.argument('network_share_path', options_list=["--network-share-path", "--networkpath"], type=str, help='Network share path.') + c.argument('network_share_domain', options_list=["--network-share-domain", "--networkdomain"], type=str, help='Network share domain.') + c.argument('network_share_user_name', options_list=["--network-share-user-name", "--networkuser"], type=str, help='Network share user name.') + c.argument('network_share_password', options_list=["--network-share-password", "--networkpw"], type=str, help='Network share password.') + c.argument('database_name', nargs='+', options_list=["--database-name", "--dbname"], help='Source database name.') + with self.argument_context('datamigration register-integration-runtime') as c: c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service') c.argument('ir_path', type=str, help='Path of Integration Runtime MSI') diff --git a/src/datamigration/azext_datamigration/manual/commands.py b/src/datamigration/azext_datamigration/manual/commands.py index 6b88bdc1b0d..bdd25e72c82 100644 --- a/src/datamigration/azext_datamigration/manual/commands.py +++ b/src/datamigration/azext_datamigration/manual/commands.py @@ -20,3 +20,4 @@ def load_command_table(self, _): g.custom_command('get-sku-recommendation', 'datamigration_get_sku_recommendation') g.custom_command('register-integration-runtime', 'datamigration_register_ir') g.custom_command('login-migration', 'datamigration_login_migration') + g.custom_command('tde-migration', 'datamigration_tde_migration') diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index 46561c202d9..0be8e2a7a1d 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -14,6 +14,7 @@ import os import signal import subprocess +from knack.prompting import prompt_pass from azure.cli.core.azclierror import MutuallyExclusiveArgumentError from azure.cli.core.azclierror import RequiredArgumentMissingError from azure.cli.core.azclierror import UnclassifiedUserFault @@ -299,3 +300,44 @@ def datamigration_login_migration(src_sql_connection_str=None, except Exception as e: raise e + + +# ----------------------------------------------------------------------------------------------------------------- +# Migrate TDE certificate from source SQL Server to the target Azure SQL Server. +# ----------------------------------------------------------------------------------------------------------------- +def datamigration_tde_migration(source_sql_connection_string=None, + target_subscription_id=None, + target_resource_group_name=None, + target_managed_instance_name=None, + network_share_path=None, + network_share_domain=None, + network_share_user_name=None, + network_share_password=None, + database_name=None): + + if source_sql_connection_string is None: + source_sql_connection_string = prompt_pass('Connection String:', confirm=False) + + if network_share_password is None: + network_share_password = prompt_pass('Network Share Password:', confirm=False) + + try: + # Setup the console app + exePath = helper.tdeMigration_console_app_setup() + + if exePath is None: + return + + if os.path.exists(exePath) is False: + print("Failed to locate executable.") + return + + cmd = f'{exePath} --sourceSqlConnectionString "{source_sql_connection_string}" --targetSubscriptionId "{target_subscription_id}" --targetResourceGroupName "{target_resource_group_name}" --targetManagedInstanceName "{target_managed_instance_name}" --networkSharePath "{network_share_path}" --networkShareDomain "{network_share_domain}" --networkShareUserName "{network_share_user_name}" --networkSharePassword "{network_share_password}" --databaseName' + + for db in database_name: + cmd += " \"" + db + "\"" + + subprocess.call(cmd, shell=False) + + except Exception as e: + raise e diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index a1c702bde57..8936430487e 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -19,6 +19,9 @@ import time import urllib.request from zipfile import ZipFile +import shutil +import requests +from knack.util import CLIError from azure.cli.core.azclierror import CLIInternalError from azure.cli.core.azclierror import FileOperationError from azure.cli.core.azclierror import InvalidArgumentValueError @@ -102,6 +105,30 @@ def loginMigration_console_app_setup(): return defaultOutputFolder, exePath +# ----------------------------------------------------------------------------------------------------------------- +# TdeMigration helper function to do console app setup (mkdir, download and extract) +# ----------------------------------------------------------------------------------------------------------------- +def tdeMigration_console_app_setup(): + validate_os_env() + + # Create downloads directory if it doesn't already exists + default_output_folder = get_tdeMigration_default_output_folder() + downloads_folder = os.path.join(default_output_folder, "Downloads") + os.makedirs(downloads_folder, exist_ok=True) + + # check and download console app + console_app_version = check_and_download_tdeMigration_console_app(downloads_folder) + if console_app_version is None: + raise CLIError("Connection to NuGet.org required. Please check connection and try again.") + + # Assigning base folder path\TdeConsoleApp\console + console_app_location = os.path.join(downloads_folder, console_app_version) + + exePath = os.path.join(console_app_location, "tools", "Microsoft.SqlServer.Migration.Tde.ConsoleApp.exe") + + return exePath + + # ----------------------------------------------------------------------------------------------------------------- # Assessment helper function to return the default output folder path depending on OS environment. # ----------------------------------------------------------------------------------------------------------------- @@ -136,6 +163,23 @@ def get_loginMigration_default_output_folder(): return defaultOutputPath +# ----------------------------------------------------------------------------------------------------------------- +# TdeMigration helper function to return the default output folder path depending on OS environment. +# ----------------------------------------------------------------------------------------------------------------- +def get_tdeMigration_default_output_folder(): + + osPlatform = platform.system() + + if osPlatform.__contains__('Linux'): + defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), ".config", "Microsoft", "SqlTdeMigrations") + elif osPlatform.__contains__('Darwin'): + defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), "Library", "Application Support", "Microsoft", "SqlTdeMigrations") + else: + defaultOutputPath = os.path.join(os.getenv('LOCALAPPDATA'), "Microsoft", "SqlTdeMigrations") + + return defaultOutputPath + + # ----------------------------------------------------------------------------------------------------------------- # Assessment helper function to check if console app exists, if not download it. # ----------------------------------------------------------------------------------------------------------------- @@ -170,6 +214,102 @@ def check_and_download_loginMigration_console_app(exePath, baseFolder): zipFile.extractall(path=baseFolder) +# ----------------------------------------------------------------------------------------------------------------- +# Get latest version of TDE Console App on NuGet.org +# ----------------------------------------------------------------------------------------------------------------- +def get_latest_nuget_org_version(package_id): + # Get Nuget.org service index + service_index_response = None + try: + service_index_response = requests.get("https://api.nuget.org/v3/index.json") + except Exception: + print("Unable to connect to NuGet.org to check for updates.") + + if(service_index_response is None or + service_index_response.status_code != 200 or + len(service_index_response.content) > 999999): + return None + + json_response = json.loads(service_index_response.content) + nuget_org_resources = json_response["resources"] + + package_base_address_type = "PackageBaseAddress/3.0.0" + package_base_address_url = None + + for resource in nuget_org_resources: + if resource["@type"] == package_base_address_type: + package_base_address_url = resource["@id"] + break + + if package_base_address_url is None: + return None + + package_versions_response = None + package_versions_response = requests.get(f"{package_base_address_url}{package_id.lower()}/index.json") + if(package_versions_response.status_code != 200 or len(package_versions_response.content) > 999999): + return None + + package_versions_json = json.loads(package_versions_response.content) + package_versions_array = package_versions_json["versions"] + return package_versions_array[len(package_versions_array) - 1] + + +# ----------------------------------------------------------------------------------------------------------------- +# Get latest local version of TDE Console App +# ----------------------------------------------------------------------------------------------------------------- +def get_latest_local_name_and_version(downloads_folder): + nuget_versions = os.listdir(downloads_folder) + if len(nuget_versions) == 0: + return None + + nuget_versions.sort(reverse=True) + return nuget_versions[0] + + +# ----------------------------------------------------------------------------------------------------------------- +# TdeMigration helper function to check if console app exists, if not download it. +# ----------------------------------------------------------------------------------------------------------------- +def check_and_download_tdeMigration_console_app(downloads_folder): + # Get latest TdeConsoleApp name from nuget.org + package_id = "Microsoft.SqlServer.Migration.TdeConsoleApp" + + latest_nuget_org_version = get_latest_nuget_org_version(package_id) + latest_nuget_org_name_and_version = f"{package_id}.{latest_nuget_org_version}" + latest_local_name_and_version = get_latest_local_name_and_version(downloads_folder) + + if latest_nuget_org_version is None: + # Cannot retrieve latest version on NuGet.org, return latest local version + return latest_local_name_and_version + + if latest_local_name_and_version is not None and latest_local_name_and_version >= latest_nuget_org_name_and_version: + # Console app is up to date, do not download update + return latest_local_name_and_version + + # Download latest version of NuGet + latest_nuget_org_download_directory = os.path.join(downloads_folder, latest_nuget_org_name_and_version) + os.makedirs(latest_nuget_org_download_directory, exist_ok=True) + + latest_nuget_org_download_name = f"{latest_nuget_org_download_directory}/{latest_nuget_org_name_and_version}.nupkg" + download_url = f"https://www.nuget.org/api/v2/package/{package_id}/{latest_nuget_org_version}" + + # Extract downloaded NuGet + urllib.request.urlretrieve(download_url, filename=latest_nuget_org_download_name) + with ZipFile(latest_nuget_org_download_name, 'r') as zipFile: + zipFile.extractall(path=latest_nuget_org_download_directory) + + exe_path = os.path.join(latest_nuget_org_download_directory, "tools", "Microsoft.SqlServer.Migration.Tde.ConsoleApp.exe") + + if os.path.exists(exe_path): + for nuget_version in os.listdir(downloads_folder): + if nuget_version != latest_nuget_org_name_and_version: + shutil.rmtree(os.path.join(downloads_folder, nuget_version)) + + else: + return latest_local_name_and_version + + return latest_nuget_org_name_and_version + + # ----------------------------------------------------------------------------------------------------------------- # Assessment helper function to check if baseFolder exists, if not create it. # ----------------------------------------------------------------------------------------------------------------- diff --git a/src/datamigration/setup.py b/src/datamigration/setup.py index fd2be45c94d..b06c4fad77f 100644 --- a/src/datamigration/setup.py +++ b/src/datamigration/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages # HISTORY.rst entry. -VERSION = '0.4.1' +VERSION = '0.5.0' try: from azext_datamigration.manual.version import VERSION except ImportError: