Skip to content

Commit

Permalink
Databricks remove unneeded files and fix build to not fail on rm when…
Browse files Browse the repository at this point in the history
… file missing (NVIDIA#451)

* remove unneeded files and use rm -rf

* Fix scalastyle for databricks shim

Co-authored-by: Thomas Graves <tgraves@nvidia.com>
  • Loading branch information
tgravescs and tgravescs authored Jul 28, 2020
1 parent 60c87b0 commit d3a674d
Show file tree
Hide file tree
Showing 4 changed files with 3 additions and 231 deletions.
2 changes: 1 addition & 1 deletion jenkins/Jenkinsfile.databricksnightly
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ pipeline {
steps {
script {
sshagent(credentials : ['svcngcc_pubpriv']) {
sh "rm spark-rapids-ci.tgz"
sh "rm -rf spark-rapids-ci.tgz"
sh "tar -zcvf spark-rapids-ci.tgz *"
sh "python3.6 ./jenkins/databricks/run-tests.py -c $CLUSTER_ID -z ./spark-rapids-ci.tgz -t $DATABRICKS_TOKEN -p /home/svcngcc/.ssh/id_rsa -l ./jenkins/databricks/build.sh -j $CI_RAPIDS_JAR -b $DATABRICKS_VERSION -k $SPARK_VERSION -a $SCALA_VERSION -f $CUDF_VERSION -u $CUDA_VERSION -m $CI_CUDF_JAR"
sh "./jenkins/databricks/deploy.sh"
Expand Down
110 changes: 0 additions & 110 deletions jenkins/Jenkinsfile.databricksrelease

This file was deleted.

118 changes: 0 additions & 118 deletions jenkins/databricks/dbimports.patch

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class Spark300dbShims extends Spark300Shims {

override def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] = {
Seq(
GpuOverrides.exec[FileSourceScanExec](
GpuOverrides.exec[FileSourceScanExec](
"Reading data from files, often from Hive tables",
(fsse, conf, p, r) => new SparkPlanMeta[FileSourceScanExec](fsse, conf, p, r) {
// partition filters and data filters are not run on the GPU
Expand Down Expand Up @@ -104,7 +104,7 @@ class Spark300dbShims extends Spark300Shims {
(join, conf, p, r) => new GpuBroadcastHashJoinMeta(join, conf, p, r)),
GpuOverrides.exec[ShuffledHashJoinExec](
"Implementation of join using hashed shuffled data",
(join, conf, p, r) => new GpuShuffledHashJoinMeta(join, conf, p, r)),
(join, conf, p, r) => new GpuShuffledHashJoinMeta(join, conf, p, r))
).map(r => (r.getClassFor.asSubclass(classOf[SparkPlan]), r)).toMap
}

Expand Down

0 comments on commit d3a674d

Please sign in to comment.