Feature: enable dynamic allocation by default (#386)

Azure · Feb 9, 2018 · 9bd3490 · 9bd3490
1 parent d75ae44
commit 9bd3490
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/cli/config.py b/cli/config.py
@@ -325,7 +325,7 @@ def _merge_dict(self, config):
             self.spark_defaults_conf = self.__convert_to_path(spark_configuration.get('spark_defaults_conf'))
             self.spark_env_sh = self.__convert_to_path(spark_configuration.get('spark_env_sh'))
             self.core_site_xml = self.__convert_to_path(spark_configuration.get('core_site_xml'))
-            self.jars = [self.__convert_to_path(jar) for jar in spark_configuration.get('jars')]
+            self.jars = [self.__convert_to_path(jar) for jar in spark_configuration.get('jars') or []]
 
     def __convert_to_path(self, str_path):
         if str_path:

diff --git a/config/spark-defaults.conf b/config/spark-defaults.conf
@@ -35,4 +35,8 @@ spark.jars                      /home/spark-current/jars/azure-storage-2.0.0.jar
 spark.deploy.recoveryMode       FILESYSTEM
 spark.deploy.recoveryDirectory  /root/
 
+# enable spark shuffle service
 spark.shuffle.service.enabled   true
+
+# enable dynamic allocation
+spark.dynamicAllocation.enabled true