Skip to content

Commit

Permalink
Add init_spark_standalone for local node (intel-analytics#2685)
Browse files Browse the repository at this point in the history
* initial version

* remove

* update to local

* remove

* style

* fix path

* fix

* update pythonhome
  • Loading branch information
hkvision committed Aug 10, 2020
1 parent 864618f commit 2e8919a
Showing 1 changed file with 14 additions and 4 deletions.
18 changes: 14 additions & 4 deletions python/orca/src/bigdl/orca/ray/raycontext.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,19 @@ class RayServiceFuncGenerator(object):
"""
def _prepare_env(self):
modified_env = os.environ.copy()
cwd = os.getcwd()
modified_env["PATH"] = "{}/{}:{}".format(cwd, "/".join(self.python_loc.split("/")[:-1]),
os.environ["PATH"])
if self.python_loc == "python_env/bin/python":
# In this case the executor is using the conda yarn archive under the current
# working directory. Need to get the full path.
executor_python_path = "{}/{}".format(
os.getcwd(), "/".join(self.python_loc.split("/")[:-1]))
else:
executor_python_path = "/".join(self.python_loc.split("/")[:-1])
if "PATH" in os.environ:
modified_env["PATH"] = "{}:{}".format(executor_python_path, os.environ["PATH"])
else:
modified_env["PATH"] = executor_python_path
modified_env["LC_ALL"] = "C.UTF-8"
modified_env["LANG"] = "C.UTF-8"
modified_env.pop("MALLOC_ARENA_MAX", None)
modified_env.pop("RAY_BACKEND_LOG_LEVEL", None)
# Unset all MKL setting as Analytics Zoo would give default values when init env.
Expand Down Expand Up @@ -95,7 +105,7 @@ def __init__(self, python_loc, redis_port, ray_node_cpu_cores,
# This is useful to allocate workers and servers in the cluster.
# Leave some reserved custom resources free to avoid unknown crash due to resources.
self.labels = \
"""--resources='{"_mxnet_worker": %s, "_mxnet_server": %s, "_reserved": %s}' """ \
"""--resources '{"_mxnet_worker": %s, "_mxnet_server": %s, "_reserved": %s}'""" \
% (1, 1, 2)

def gen_stop(self):
Expand Down

0 comments on commit 2e8919a

Please sign in to comment.