From f151a7ceab1469b455871b01434d76a2d07af9c3 Mon Sep 17 00:00:00 2001 From: Le-Zheng <30695225+Le-Zheng@users.noreply.github.com> Date: Sun, 27 Sep 2020 02:10:54 +0100 Subject: [PATCH] [WIP] spark 3.0 (#3054) * spark 3.0 --- python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py index d333cb1772f..8e42ee311b8 100644 --- a/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py +++ b/python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py @@ -19,7 +19,6 @@ import gc from tempfile import NamedTemporaryFile -from pyspark.cloudpickle import print_exec from pyspark.broadcast import Broadcast from pyspark.broadcast import _from_id from bigdl.nn.layer import Model @@ -49,6 +48,10 @@ def dump(self, value, f): value.saveModel(f.name, over_write=True) except Exception as e: msg = "Could not serialize broadcast: %s" % e.__class__.__name__ + if not self.sc.version.startswith("2.1"): + from pyspark.cloudpickle import print_exec + else: + from pyspark.util import print_exec print_exec(sys.stderr) raise ValueError(msg) f.close()