diff --git a/cli/config.py b/cli/config.py index 20184c7c..6bd924ed 100644 --- a/cli/config.py +++ b/cli/config.py @@ -325,7 +325,7 @@ def _merge_dict(self, config): self.spark_defaults_conf = self.__convert_to_path(spark_configuration.get('spark_defaults_conf')) self.spark_env_sh = self.__convert_to_path(spark_configuration.get('spark_env_sh')) self.core_site_xml = self.__convert_to_path(spark_configuration.get('core_site_xml')) - self.jars = [self.__convert_to_path(jar) for jar in spark_configuration.get('jars')] + self.jars = [self.__convert_to_path(jar) for jar in spark_configuration.get('jars') or []] def __convert_to_path(self, str_path): if str_path: diff --git a/config/spark-defaults.conf b/config/spark-defaults.conf index 683b41aa..89e178e7 100644 --- a/config/spark-defaults.conf +++ b/config/spark-defaults.conf @@ -35,4 +35,8 @@ spark.jars /home/spark-current/jars/azure-storage-2.0.0.jar spark.deploy.recoveryMode FILESYSTEM spark.deploy.recoveryDirectory /root/ +# enable spark shuffle service spark.shuffle.service.enabled true + +# enable dynamic allocation +spark.dynamicAllocation.enabled true