Update init_spark_on_yarn (intel-analytics#2587)

* update * meet review * update doc * style
sgwhat · Jul 21, 2020 · 1c38205 · 1c38205
1 parent c4c5281
commit 1c38205
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 8 deletions.
diff --git a/python/orca/example/learn/horovod/pytorch_estimator.py b/python/orca/example/learn/horovod/pytorch_estimator.py
@@ -127,7 +127,7 @@ def train_example():
         sc = init_spark_on_yarn(
             hadoop_conf=args.hadoop_conf,
             conda_name=args.conda_name,
-            num_executor=args.slave_num,
+            num_executors=args.slave_num,
             executor_cores=args.executor_cores,
             executor_memory=args.executor_memory,
             driver_memory=args.driver_memory,

diff --git a/python/orca/example/learn/horovod/simple_horovod_pytorch.py b/python/orca/example/learn/horovod/simple_horovod_pytorch.py
@@ -219,7 +219,7 @@ def test():
         sc = init_spark_on_yarn(
             hadoop_conf=args.hadoop_conf,
             conda_name=args.conda_name,
-            num_executor=args.slave_num,
+            num_executors=args.slave_num,
             executor_cores=args.executor_cores,
             executor_memory=args.executor_memory,
             driver_memory=args.driver_memory,

diff --git a/python/orca/example/learn/mxnet/lenet_mnist.py b/python/orca/example/learn/mxnet/lenet_mnist.py
@@ -117,7 +117,7 @@ def get_metrics(config):
         sc = init_spark_on_yarn(
             hadoop_conf=opt.hadoop_conf,
             conda_name=opt.conda_name,
-            num_executor=opt.num_workers,
+            num_executors=opt.num_workers,
             executor_cores=opt.executor_cores)
     else:
         sc = init_spark_on_local(cores="*")

diff --git a/python/orca/example/torchmodel/train/mnist/main.py b/python/orca/example/torchmodel/train/mnist/main.py
@@ -93,14 +93,14 @@ def main():
         sc = init_spark_on_yarn(
             hadoop_conf=hadoop_conf_dir,
             conda_name=zoo_conda_name,
-            num_executor=num_executors,
+            num_executors=num_executors,
             executor_cores=num_cores_per_executor,
             executor_memory="2g",
             driver_memory="10g",
             driver_cores=1,
-            spark_conf={"spark.rpc.message.maxSize": "1024",
-                        "spark.task.maxFailures":  "1",
-                        "spark.driver.extraJavaOptions": "-Dbigdl.failure.retryTimes=1"})
+            conf={"spark.rpc.message.maxSize": "1024",
+                  "spark.task.maxFailures":  "1",
+                  "spark.driver.extraJavaOptions": "-Dbigdl.failure.retryTimes=1"})
 
     model = Net()
     model.train()

diff --git a/python/orca/example/torchmodel/train/resnet_finetune/resnet_finetune.py b/python/orca/example/torchmodel/train/resnet_finetune/resnet_finetune.py
@@ -58,7 +58,7 @@ def forward(self, x):
         sc = init_spark_on_yarn(
             hadoop_conf=hadoop_conf_dir,
             conda_name=os.environ["ZOO_CONDA_NAME"],  # The name of the created conda-env
-            num_executor=num_executors,
+            num_executors=num_executors,
             executor_cores=num_cores_per_executor,
             executor_memory="8g",
             driver_memory="2g",