From 5761a3663adb261f49e0e7fc2301b327ee833c12 Mon Sep 17 00:00:00 2001 From: Jacob Freck Date: Fri, 23 Mar 2018 13:42:30 -0700 Subject: [PATCH] Bug: set explicit file open encoding (#448) * explicit file encoding * crlf->lf --- aztk/internal/cluster_data/node_data.py | 6 +++--- aztk/models/plugins/plugin_file.py | 2 +- aztk/node_scripts/install/create_user.py | 5 +++-- aztk/node_scripts/install/plugins.py | 4 ++-- aztk/node_scripts/install/scripts.py | 2 +- aztk/node_scripts/install/spark.py | 4 ++-- aztk/node_scripts/job_submission.py | 2 +- aztk/node_scripts/submit.py | 5 +++-- aztk/utils/get_ssh_key.py | 2 +- aztk_cli/config.py | 8 ++++---- aztk_cli/spark/endpoints/init.py | 4 ++-- setup.py | 2 +- 12 files changed, 24 insertions(+), 22 deletions(-) diff --git a/aztk/internal/cluster_data/node_data.py b/aztk/internal/cluster_data/node_data.py index 552b803d..6975d07b 100644 --- a/aztk/internal/cluster_data/node_data.py +++ b/aztk/internal/cluster_data/node_data.py @@ -47,7 +47,7 @@ def add_file(self, file: str, zip_dir: str, binary: bool = True): return if isinstance(file, (str, bytes)): full_file_path = Path(file) - with io.open(file, 'r') as f: + with io.open(file, 'r', encoding='UTF-8') as f: if binary: self.zipf.write(file, os.path.join(zip_dir, full_file_path.name)) else: @@ -70,7 +70,7 @@ def add_dir(self, path: str, dest: str = None, exclude: List[str] = []): relative_folder = os.path.relpath(base, path) for file in files: if self._includeFile(file, exclude): - with io.open(os.path.join(base, file), 'r') as f: + with io.open(os.path.join(base, file), 'r', encoding='UTF-8') as f: self.zipf.writestr(os.path.join(dest, relative_folder, file), f.read().replace('\r\n', '\n')) def _add_custom_scripts(self): @@ -83,7 +83,7 @@ def _add_custom_scripts(self): new_file_name = str(index) + '_' + os.path.basename(custom_script.script) data.append(dict(script=new_file_name, runOn=str(custom_script.run_on))) try: - with io.open(custom_script.script, 'r') as f: + with io.open(custom_script.script, 'r', encoding='UTF-8') as f: self.zipf.writestr( os.path.join(CUSTOM_SCRIPT_FOLDER, new_file_name), f.read().replace('\r\n', '\n')) diff --git a/aztk/models/plugins/plugin_file.py b/aztk/models/plugins/plugin_file.py index 93ab80cd..f5e756f0 100644 --- a/aztk/models/plugins/plugin_file.py +++ b/aztk/models/plugins/plugin_file.py @@ -13,7 +13,7 @@ def __init__(self, target: str, local_path: str): # TODO handle folders? def content(self): - with open(self.local_path, "r") as f: + with open(self.local_path, "r", encoding='UTF-8') as f: return f.read() diff --git a/aztk/node_scripts/install/create_user.py b/aztk/node_scripts/install/create_user.py index 9f20261e..5c595d45 100644 --- a/aztk/node_scripts/install/create_user.py +++ b/aztk/node_scripts/install/create_user.py @@ -16,7 +16,7 @@ def create_user(batch_client): print("No user to create.") return - with open(path) as file: + with open(path, 'r', encoding='UTF-8') as file: user_conf = yaml.load(file.read()) try: @@ -43,7 +43,8 @@ def decrypt_password(user_conf): tag = user_conf['tag'] # Read private key - private_key = RSA.import_key(open(os.path.join(os.environ['DOCKER_WORKING_DIR'], 'id_rsa')).read()) + with open(os.path.join(os.environ['DOCKER_WORKING_DIR'], 'id_rsa'), encoding='UTF-8') as f: + private_key = RSA.import_key(f.read()) # Decrypt the session key with the public RSA key cipher_rsa = PKCS1_OAEP.new(private_key) session_key = cipher_rsa.decrypt(encrypted_aes_session_key) diff --git a/aztk/node_scripts/install/plugins.py b/aztk/node_scripts/install/plugins.py index bac6fd97..02dd057b 100644 --- a/aztk/node_scripts/install/plugins.py +++ b/aztk/node_scripts/install/plugins.py @@ -12,7 +12,7 @@ def _read_manifest_file(path=None): if not os.path.isfile(path): print("Plugins manifest file doesn't exist at {0}".format(path)) else: - with open(path, 'r') as stream: + with open(path, 'r', encoding='UTF-8') as stream: try: custom_scripts = yaml.load(stream) except json.JSONDecodeError as err: @@ -86,7 +86,7 @@ def _run_script(name: str, script_path: str = None, args: dict = None, env: dict if args is None: args = [] - out_file = open(os.path.join(log_folder, '{0}.txt'.format(name)), 'w') + out_file = open(os.path.join(log_folder, '{0}.txt'.format(name)), 'w', encoding='UTF-8') try: subprocess.call( [script_path] + args, diff --git a/aztk/node_scripts/install/scripts.py b/aztk/node_scripts/install/scripts.py index 6bb942ea..c3534824 100644 --- a/aztk/node_scripts/install/scripts.py +++ b/aztk/node_scripts/install/scripts.py @@ -9,7 +9,7 @@ def _read_yaml_file(path=None): if not os.path.isfile(path): print("Configuration file doesn't exist at {0}".format(path)) else: - with open(path, 'r') as stream: + with open(path, 'r', encoding='UTF-8') as stream: try: custom_scripts = yaml.load(stream) except yaml.YAMLError as err: diff --git a/aztk/node_scripts/install/spark.py b/aztk/node_scripts/install/spark.py index 11ed2342..423489c5 100644 --- a/aztk/node_scripts/install/spark.py +++ b/aztk/node_scripts/install/spark.py @@ -43,7 +43,7 @@ def setup_connection(): master_node = get_node(master_node_id) master_config_file = os.path.join(spark_conf_folder, "master") - master_file = open(master_config_file, 'w') + master_file = open(master_config_file, 'w', encoding='UTF-8') print("Adding master node ip {0} to config file '{1}'".format( master_node.ip_address, master_config_file)) @@ -195,7 +195,7 @@ def copy_jars(): def parse_configuration_file(path_to_file: str): try: - file = open(path_to_file, 'r') + file = open(path_to_file, 'r', encoding='UTF-8') properties = {} for line in file: if (not line.startswith('#') and len(line) > 1): diff --git a/aztk/node_scripts/job_submission.py b/aztk/node_scripts/job_submission.py index 8393192f..434b7345 100644 --- a/aztk/node_scripts/job_submission.py +++ b/aztk/node_scripts/job_submission.py @@ -27,7 +27,7 @@ def schedule_tasks(tasks_path): blob_client = config.blob_client for task_definition in tasks_path: - with open(task_definition, 'r') as stream: + with open(task_definition, 'r', encoding='UTF-8') as stream: try: task = yaml.load(stream) except yaml.YAMLError as exc: diff --git a/aztk/node_scripts/submit.py b/aztk/node_scripts/submit.py index 7911bb69..b8b99633 100644 --- a/aztk/node_scripts/submit.py +++ b/aztk/node_scripts/submit.py @@ -124,7 +124,7 @@ def load_application(application_file_path): ''' Read and parse the application from file ''' - with open(application_file_path) as f: + with open(application_file_path, encoding='UTF-8') as f: application = yaml.load(f) return application @@ -176,7 +176,8 @@ def upload_error_log(error, application_file_path): application = load_application(application_file_path) blob_client = config.blob_client - with open(os.path.join(os.environ["AZ_BATCH_TASK_WORKING_DIR"], "error.log"), "w") as error_log: + error_log_path = os.path.join(os.environ["AZ_BATCH_TASK_WORKING_DIR"], "error.log") + with open(error_log_path, "w", encoding='UTF-8') as error_log: error_log.write(error) upload_file_to_container( diff --git a/aztk/utils/get_ssh_key.py b/aztk/utils/get_ssh_key.py index 523b2de0..601d005e 100644 --- a/aztk/utils/get_ssh_key.py +++ b/aztk/utils/get_ssh_key.py @@ -28,6 +28,6 @@ def __read_ssh_key_from_file(path: str) -> str: """ Read the content of the given file """ - with open(os.path.expanduser(path), 'r') as content_file: + with open(os.path.expanduser(path), 'r', encoding='UTF-8') as content_file: content = content_file.read() return content diff --git a/aztk_cli/config.py b/aztk_cli/config.py index 9af49bec..716e2e48 100644 --- a/aztk_cli/config.py +++ b/aztk_cli/config.py @@ -46,7 +46,7 @@ def _load_secrets_config( if not os.path.isfile(path): return None - with open(path, 'r') as stream: + with open(path, 'r', encoding='UTF-8') as stream: try: return yaml.load(stream) except yaml.YAMLError as err: @@ -129,7 +129,7 @@ def read_cluster_config( if not os.path.isfile(path): return - with open(path, 'r') as stream: + with open(path, 'r', encoding='UTF-8') as stream: try: config_dict = yaml.load(stream) except yaml.YAMLError as err: @@ -224,7 +224,7 @@ def _read_config_file( if not os.path.isfile(path): return - with open(path, 'r') as stream: + with open(path, 'r', encoding='UTF-8') as stream: try: config = yaml.load(stream) except yaml.YAMLError as err: @@ -359,7 +359,7 @@ def _read_config_file( if not path or not os.path.isfile(path): return - with open(path, 'r') as stream: + with open(path, 'r', encoding='UTF-8') as stream: try: config = yaml.load(stream) except yaml.YAMLError as err: diff --git a/aztk_cli/spark/endpoints/init.py b/aztk_cli/spark/endpoints/init.py index 27a424c5..4b7b9465 100644 --- a/aztk_cli/spark/endpoints/init.py +++ b/aztk_cli/spark/endpoints/init.py @@ -48,8 +48,8 @@ def create_directory(dest_path: str, docker_repo: str): cluster_path = os.path.join(dest_path, 'cluster.yaml') if os.path.isfile(cluster_path): - with open(cluster_path, 'r') as stream: + with open(cluster_path, 'r', encoding='UTF-8') as stream: cluster_yaml = stream.read() cluster_yaml = cluster_yaml.replace("docker_repo: \n", "docker_repo: {}\n".format(docker_repo)) - with open(cluster_path, 'w') as file: + with open(cluster_path, 'w', encoding='UTF-8') as file: file.write(cluster_yaml) diff --git a/setup.py b/setup.py index c925bcc2..23a23208 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def find_package_files(root, directory, dest=""): return paths -with open('README.md') as fd: +with open('README.md', encoding='UTF-8') as fd: long_description = fd.read() setup(