Skip to content

Commit

Permalink
Add init_spark_standalone for local node (intel-analytics#2685)
Browse files Browse the repository at this point in the history
* initial version

* remove

* update to local

* remove

* style

* fix path

* fix

* update pythonhome
  • Loading branch information
hkvision authored and yangw1234 committed Sep 26, 2021
1 parent d7f469e commit e56f112
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 28 deletions.
2 changes: 1 addition & 1 deletion python/orca/src/bigdl/orca/data/shard.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ def to_ray(self):
def put_to_plasma(ids):
def f(index, iterator):
import pyarrow.plasma as plasma
from zoo.orca.data.utils import get_node_ip
from zoo.util.utils import get_node_ip
res = list(iterator)
client = plasma.connect(object_store_address)
target_id = ids[index]
Expand Down
27 changes: 0 additions & 27 deletions python/orca/src/bigdl/orca/data/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,33 +237,6 @@ def get_class_name(obj):
return obj.__class__.__name__


def get_node_ip():
"""
This function is ported from ray to get the ip of the current node. In the settings where
Ray is not involved, calling ray.services.get_node_ip_address would introduce Ray overhead.
"""
import socket
import errno
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# This command will raise an exception if there is no internet connection.
s.connect(("8.8.8.8", 80))
node_ip_address = s.getsockname()[0]
except OSError as e:
node_ip_address = "127.0.0.1"
# [Errno 101] Network is unreachable
if e.errno == errno.ENETUNREACH:
try:
# try get node ip address from host name
host_name = socket.getfqdn(socket.gethostname())
node_ip_address = socket.gethostbyname(host_name)
except Exception:
pass
finally:
s.close()
return node_ip_address


def _convert_list_tuple(data, allow_tuple, allow_list):
if not allow_list and not allow_tuple:
raise ValueError("value of x and y should be a ndarray, but get a " +
Expand Down

0 comments on commit e56f112

Please sign in to comment.