From 1157a4285e9d4fdc7432562e1dcd16343251a07c Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Mon, 10 Aug 2020 18:27:29 +0800 Subject: [PATCH] Add init_spark_standalone for local node (#2685) * initial version * remove * update to local * remove * style * fix path * fix * update pythonhome --- python/orca/src/bigdl/orca/data/shard.py | 2 +- python/orca/src/bigdl/orca/data/utils.py | 27 ------------------------ 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/python/orca/src/bigdl/orca/data/shard.py b/python/orca/src/bigdl/orca/data/shard.py index 21f27e7fc36..c8f856c1803 100644 --- a/python/orca/src/bigdl/orca/data/shard.py +++ b/python/orca/src/bigdl/orca/data/shard.py @@ -301,7 +301,7 @@ def to_ray(self): def put_to_plasma(ids): def f(index, iterator): import pyarrow.plasma as plasma - from zoo.orca.data.utils import get_node_ip + from zoo.util.utils import get_node_ip res = list(iterator) client = plasma.connect(object_store_address) target_id = ids[index] diff --git a/python/orca/src/bigdl/orca/data/utils.py b/python/orca/src/bigdl/orca/data/utils.py index 95398330a73..907fe652b99 100644 --- a/python/orca/src/bigdl/orca/data/utils.py +++ b/python/orca/src/bigdl/orca/data/utils.py @@ -237,33 +237,6 @@ def get_class_name(obj): return obj.__class__.__name__ -def get_node_ip(): - """ - This function is ported from ray to get the ip of the current node. In the settings where - Ray is not involved, calling ray.services.get_node_ip_address would introduce Ray overhead. - """ - import socket - import errno - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # This command will raise an exception if there is no internet connection. - s.connect(("8.8.8.8", 80)) - node_ip_address = s.getsockname()[0] - except OSError as e: - node_ip_address = "127.0.0.1" - # [Errno 101] Network is unreachable - if e.errno == errno.ENETUNREACH: - try: - # try get node ip address from host name - host_name = socket.getfqdn(socket.gethostname()) - node_ip_address = socket.gethostbyname(host_name) - except Exception: - pass - finally: - s.close() - return node_ip_address - - def _convert_list_tuple(data, allow_tuple, allow_list): if not allow_list and not allow_tuple: raise ValueError("value of x and y should be a ndarray, but get a " +