Add shim for EMR 3.0.1 and EMR 3.0.1-SNAPSHOT (NVIDIA#995)

* Add emr301 shim Signed-off-by: Thomas Graves <tgraves@nvidia.com> * remove snapshot from spark 3.0.1 Signed-off-by: Thomas Graves <tgraves@nvidia.com> * Cleanupu imports and unused var Signed-off-by: Thomas Graves <tgraves@nvidia.com>
nartal1 · Oct 21, 2020 · b2daceb · b2daceb
1 parent be9285c
commit b2daceb
Show file tree

Hide file tree

Showing 9 changed files with 197 additions and 1 deletion.
diff --git a/pom.xml b/pom.xml
@@ -163,6 +163,7 @@
         <slf4j.version>1.7.30</slf4j.version>
         <spark300.version>3.0.0</spark300.version>
         <spark300emr.version>3.0.0-amzn</spark300emr.version>
+        <spark301emr.version>3.0.1-amzn</spark301emr.version>
         <!--
          If you update a dependendy version so it is no longer a SNAPSHOT
          please update the snapshot-shims profile as well so it is accurate -->

diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml
@@ -91,6 +91,12 @@
             <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.nvidia</groupId>
+            <artifactId>rapids-4-spark-shims-spark301emr_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <groupId>com.nvidia</groupId>
             <artifactId>rapids-4-spark-shims-spark301_${scala.binary.version}</artifactId>

diff --git a/shims/pom.xml b/shims/pom.xml
@@ -54,6 +54,7 @@
     <modules>
         <module>spark300</module>
         <module>spark300emr</module>
+        <module>spark301emr</module>
         <module>spark301</module>
         <module>aggregator</module>
     </modules>

diff --git a/...k301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/SparkShimServiceProvider.scala b/...k301/src/main/scala/com/nvidia/spark/rapids/shims/spark301/SparkShimServiceProvider.scala
@@ -20,7 +20,7 @@ import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion}
 
 object SparkShimServiceProvider {
   val VERSION = SparkShimVersion(3, 0, 1)
-  val VERSIONNAMES = Seq(s"$VERSION", s"$VERSION-SNAPSHOT")
+  val VERSIONNAMES = Seq(s"$VERSION")
 }
 class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
 

diff --git a/shims/spark301emr/pom.xml b/shims/spark301emr/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Copyright (c) 2020, NVIDIA CORPORATION.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.nvidia</groupId>
+        <artifactId>rapids-4-spark-shims_2.12</artifactId>
+        <version>0.3.0-SNAPSHOT</version>
+	<relativePath>../pom.xml</relativePath>
+    </parent>
+    <groupId>com.nvidia</groupId>
+    <artifactId>rapids-4-spark-shims-spark301emr_2.12</artifactId>
+    <name>RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.0.1 EMR Shim</name>
+    <description>The RAPIDS SQL plugin for Apache Spark 3.0.1 EMR Shim</description>
+    <version>0.3.0-SNAPSHOT</version>
+
+    <!-- Set 'spark.version' for the shims layer -->
+    <!-- Create a separate file 'SPARK_VER.properties' in the jar to save cudf & spark version info -->
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>dependency</id>
+                        <phase>generate-resources</phase>
+                        <configuration>
+                            <target>
+                                <mkdir dir="${project.build.directory}/extra-resources"/>
+                                <exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark301emr.version}-info.properties">
+                                    <arg value="${user.dir}/build/dependency-info.sh"/>
+                                    <arg value="${cudf.version}"/>
+                                    <arg value="${cuda.version}"/>
+                                    <arg value="${spark301emr.version}"/>
+                                </exec>
+                            </target>
+                        </configuration>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+
+        <resources>
+          <resource>
+            <!-- Include the properties file to provide the build information. -->
+            <directory>${project.build.directory}/extra-resources</directory>
+          </resource>
+          <resource>
+            <directory>src/main/resources</directory>
+          </resource>
+        </resources>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.nvidia</groupId>
+            <artifactId>rapids-4-spark-shims-spark301_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
+            <version>${spark301.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/...emr/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider b/...emr/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider
@@ -0,0 +1 @@
+com.nvidia.spark.rapids.shims.spark301emr.SparkShimServiceProvider
diff --git a/...ark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/Spark301EMRShims.scala b/...ark301emr/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/Spark301EMRShims.scala
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids.shims.spark301emr
+
+import java.lang.reflect.Constructor
+
+import com.nvidia.spark.rapids._
+import com.nvidia.spark.rapids.shims.spark301.Spark301Shims
+import com.nvidia.spark.rapids.spark301emr.RapidsShuffleManager
+
+class Spark301EMRShims extends Spark301Shims {
+
+  override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION
+
+  override def getRapidsShuffleManagerClass: String = {
+    classOf[RapidsShuffleManager].getCanonicalName
+  }
+}
diff --git a/...r/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/SparkShimServiceProvider.scala b/...r/src/main/scala/com/nvidia/spark/rapids/shims/spark301emr/SparkShimServiceProvider.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids.shims.spark301emr
+
+import com.nvidia.spark.rapids.{EMRShimVersion, SparkShims}
+
+object SparkShimServiceProvider {
+  val VERSION = EMRShimVersion(3, 0, 1)
+}
+
+class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
+
+  def matchesVersion(version: String): Boolean = {
+    // EMR version looks like 3.0.1-amzn-0
+    val baseVersion = SparkShimServiceProvider.VERSION.toString + raw"(-\d+)"
+    val amznVersion = baseVersion.r
+    val amznVersionSnapshot = (baseVersion + "-SNAPSHOT").r
+    version match {
+        case amznVersion(_*) => true
+        case amznVersionSnapshot(_*) => true
+        case _ => false
+    }
+  }
+
+  def buildShim: SparkShims = {
+    new Spark301EMRShims()
+  }
+}
diff --git a/...spark301emr/src/main/scala/com/nvidia/spark/rapids/spark301emr/RapidsShuffleManager.scala b/...spark301emr/src/main/scala/com/nvidia/spark/rapids/spark301emr/RapidsShuffleManager.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids.spark301emr
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.rapids.shims.spark301.RapidsShuffleInternalManager
+
+/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */
+sealed class RapidsShuffleManager(
+    conf: SparkConf,
+    isDriver: Boolean) extends RapidsShuffleInternalManager(conf, isDriver) {
+}