Skip to content

Commit

Permalink
[WIP] spark 3.0 (intel-analytics#3054)
Browse files Browse the repository at this point in the history
* spark 3.0
  • Loading branch information
Le-Zheng committed Sep 27, 2020
1 parent 947a028 commit f151a7c
Showing 1 changed file with 4 additions and 1 deletion.
5 changes: 4 additions & 1 deletion python/dllib/src/bigdl/dllib/models/utils/model_broadcast.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import gc
from tempfile import NamedTemporaryFile

from pyspark.cloudpickle import print_exec
from pyspark.broadcast import Broadcast
from pyspark.broadcast import _from_id
from bigdl.nn.layer import Model
Expand Down Expand Up @@ -49,6 +48,10 @@ def dump(self, value, f):
value.saveModel(f.name, over_write=True)
except Exception as e:
msg = "Could not serialize broadcast: %s" % e.__class__.__name__
if not self.sc.version.startswith("2.1"):
from pyspark.cloudpickle import print_exec
else:
from pyspark.util import print_exec
print_exec(sys.stderr)
raise ValueError(msg)
f.close()
Expand Down

0 comments on commit f151a7c

Please sign in to comment.