diff --git a/pom.xml b/pom.xml index 7a8ad2d7ff9..57d9f64840f 100644 --- a/pom.xml +++ b/pom.xml @@ -163,6 +163,7 @@ 1.7.30 3.0.0 3.0.0-amzn + 3.0.1-amzn diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml index 15659bb5a03..92b87779137 100644 --- a/shims/aggregator/pom.xml +++ b/shims/aggregator/pom.xml @@ -91,6 +91,12 @@ ${project.version} compile + + com.nvidia + rapids-4-spark-shims-spark301emr_${scala.binary.version} + ${project.version} + compile + com.nvidia rapids-4-spark-shims-spark301_${scala.binary.version} diff --git a/shims/pom.xml b/shims/pom.xml index da8c923363b..3177f83d960 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -54,6 +54,7 @@ spark300 spark300emr + spark301emr spark301 aggregator diff --git a/shims/spark301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/SparkShimServiceProvider.scala b/shims/spark301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/SparkShimServiceProvider.scala index 6708d460c55..24668123c21 100644 --- a/shims/spark301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/SparkShimServiceProvider.scala +++ b/shims/spark301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/SparkShimServiceProvider.scala @@ -20,7 +20,7 @@ import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion} object SparkShimServiceProvider { val VERSION = SparkShimVersion(3, 0, 1) - val VERSIONNAMES = Seq(s"$VERSION", s"$VERSION-SNAPSHOT") + val VERSIONNAMES = Seq(s"$VERSION") } class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { diff --git a/shims/spark301emr/pom.xml b/shims/spark301emr/pom.xml new file mode 100644 index 00000000000..5815f6895b6 --- /dev/null +++ b/shims/spark301emr/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-shims_2.12 + 0.3.0-SNAPSHOT + ../pom.xml + + com.nvidia + rapids-4-spark-shims-spark301emr_2.12 + RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 EMR Shim + The RAPIDS SQL plugin for Apache Spark 3.0.1 EMR Shim + 0.3.0-SNAPSHOT + + + + + + + maven-antrun-plugin + + + dependency + generate-resources + + + + + + + + + + + + + run + + + + + + + + + + ${project.build.directory}/extra-resources + + + src/main/resources + + + + + + + com.nvidia + rapids-4-spark-shims-spark301_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark301.version} + provided + + + diff --git a/shims/spark301emr/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider b/shims/spark301emr/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider new file mode 100644 index 00000000000..3f866d00236 --- /dev/null +++ b/shims/spark301emr/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider @@ -0,0 +1 @@ +com.nvidia.spark.rapids.shims.spark301emr.SparkShimServiceProvider diff --git a/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/Spark301EMRShims.scala b/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/Spark301EMRShims.scala new file mode 100644 index 00000000000..022c530d82c --- /dev/null +++ b/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/Spark301EMRShims.scala @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.shims.spark301emr + +import java.lang.reflect.Constructor + +import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.shims.spark301.Spark301Shims +import com.nvidia.spark.rapids.spark301emr.RapidsShuffleManager + +class Spark301EMRShims extends Spark301Shims { + + override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION + + override def getRapidsShuffleManagerClass: String = { + classOf[RapidsShuffleManager].getCanonicalName + } +} diff --git a/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/SparkShimServiceProvider.scala b/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/SparkShimServiceProvider.scala new file mode 100644 index 00000000000..2731873bc1f --- /dev/null +++ b/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/SparkShimServiceProvider.scala @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.shims.spark301emr + +import com.nvidia.spark.rapids.{EMRShimVersion, SparkShims} + +object SparkShimServiceProvider { + val VERSION = EMRShimVersion(3, 0, 1) +} + +class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { + + def matchesVersion(version: String): Boolean = { + // EMR version looks like 3.0.1-amzn-0 + val baseVersion = SparkShimServiceProvider.VERSION.toString + raw"(-\d+)" + val amznVersion = baseVersion.r + val amznVersionSnapshot = (baseVersion + "-SNAPSHOT").r + version match { + case amznVersion(_*) => true + case amznVersionSnapshot(_*) => true + case _ => false + } + } + + def buildShim: SparkShims = { + new Spark301EMRShims() + } +} diff --git a/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/spark301emr/RapidsShuffleManager.scala b/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/spark301emr/RapidsShuffleManager.scala new file mode 100644 index 00000000000..8cd0279f106 --- /dev/null +++ b/shims/spark301emr/src/main/scala/com/nvidia/spark/rapids/spark301emr/RapidsShuffleManager.scala @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids.spark301emr + +import org.apache.spark.SparkConf +import org.apache.spark.sql.rapids.shims.spark301.RapidsShuffleInternalManager + +/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */ +sealed class RapidsShuffleManager( + conf: SparkConf, + isDriver: Boolean) extends RapidsShuffleInternalManager(conf, isDriver) { +}