Merge pull request NVIDIA#1848 from jlowe/fix-merge

Fix merge conflict with branch-0.4

docs/FAQ.md

@@ -10,10 +10,10 @@ nav_order: 11
 
 ### What versions of Apache Spark does the RAPIDS Accelerator for Apache Spark support?
 
-The RAPIDS Accelerator for Apache Spark requires version 3.0.0 or 3.0.1 of Apache Spark. Because the
-plugin replaces parts of the physical plan that Apache Spark considers to be internal the code for
-those plans can change even between bug fix releases. As a part of our process, we try to stay on
-top of these changes and release updates as quickly as possible.
+The RAPIDS Accelerator for Apache Spark requires version 3.0.0, 3.0.1, 3.0.2 or 3.1.1 of Apache
+Spark. Because the plugin replaces parts of the physical plan that Apache Spark considers to be
+internal the code for those plans can change even between bug fix releases. As a part of our
+process, we try to stay on top of these changes and release updates as quickly as possible.
 
 ### Which distributions are supported?
 

docs/download.md

@@ -21,8 +21,8 @@ This release includes additional performance improvements, including
 * Instructions on how to use [Alluxio caching](get-started/getting-started-alluxio.md) with Spark to
   leverage caching.
 
-The release is supported on Apache Spark 3.0.0, 3.0.1, 3.1.1, Databricks 7.3 ML LTS and Google Cloud
-Platform Dataproc 2.0.
+The release is supported on Apache Spark 3.0.0, 3.0.1, 3.0.2, 3.1.1, Databricks 7.3 ML LTS and
+Google Cloud Platform Dataproc 2.0.
 
 The list of all supported operations is provided [here](supported_ops.md).
 

pom.xml

@@ -205,7 +205,7 @@
         <spark301.version>3.0.1</spark301.version>
         <spark301db.version>3.0.1-databricks</spark301db.version>
         <spark302.version>3.0.2</spark302.version>
-        <spark311.version>3.1.1-SNAPSHOT</spark311.version>
+        <spark311.version>3.1.1</spark311.version>
         <spark320.version>3.2.0-SNAPSHOT</spark320.version>
         <mockito.version>3.6.0</mockito.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>

shims/aggregator/pom.xml

@@ -68,12 +68,6 @@
                     <version>${project.version}</version>
                     <scope>compile</scope>
                 </dependency>
-                <dependency>
-                    <groupId>com.nvidia</groupId>
-                    <artifactId>rapids-4-spark-shims-spark311_${scala.binary.version}</artifactId>
-                    <version>${project.version}</version>
-                    <scope>compile</scope>
-                </dependency>
             </dependencies>
         </profile>
     </profiles>
@@ -109,5 +103,11 @@
             <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.nvidia</groupId>
+            <artifactId>rapids-4-spark-shims-spark311_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 </project>

shims/pom.xml

@@ -45,7 +45,6 @@
                 <activeByDefault>true</activeByDefault>
             </activation>
             <modules>
-                <module>spark311</module>
                 <module>spark320</module>
             </modules>
         </profile>
@@ -57,6 +56,7 @@
         <module>spark301emr</module>
         <module>spark301</module>
         <module>spark302</module>
+        <module>spark311</module>
         <module>aggregator</module>
     </modules>
     <dependencies>

shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala

@@ -19,10 +19,8 @@ package com.nvidia.spark.rapids.shims.spark311
 import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion}
 
 object SparkShimServiceProvider {
-  // keep 3.1.0 snapshot version for now until 3.1.1 is released
-  val VERSION310 = SparkShimVersion(3, 1, 0)
   val VERSION = SparkShimVersion(3, 1, 1)
-  val VERSIONNAMES = Seq(s"$VERSION310-SNAPSHOT", s"$VERSION", s"$VERSION-SNAPSHOT")
+  val VERSIONNAMES = Seq(s"$VERSION")
 }
 
 class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {

sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionPathString
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
@@ -204,12 +204,7 @@ case class GpuInsertIntoHadoopFsRelationCommand(
       committer: FileCommitProtocol): Unit = {
     val staticPartitionPrefix = if (staticPartitions.nonEmpty) {
       "/" + partitionColumns.flatMap { p =>
-        staticPartitions.get(p.name) match {
-          case Some(value) =>
-            Some(escapePathName(p.name) + "=" + escapePathName(value))
-          case None =>
-            None
-        }
+        staticPartitions.get(p.name).map(getPartitionPathString(p.name, _))
       }.mkString("/")
     } else {
       ""

tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids
+
+import org.scalatest.BeforeAndAfterEach
+
+class InsertPartition311Suite extends SparkQueryCompareTestSuite with BeforeAndAfterEach {
+  var tableNr = 0
+
+  override def afterEach(): Unit = {
+    List(1, 2).foreach { tnr =>
+      SparkSessionHolder.sparkSession.sql(s"DROP TABLE IF EXISTS t$tnr")
+    }
+  }
+
+  testSparkResultsAreEqual(
+    testName ="Insert null-value partition ",
+    spark => {
+      tableNr += 1
+      spark.sql(s"""CREATE TABLE t${tableNr}(i STRING, c STRING)
+                   |USING PARQUET PARTITIONED BY (c)""".stripMargin)
+      spark.sql(s"""INSERT OVERWRITE t${tableNr} PARTITION (c=null)
+                   |VALUES ('1')""".stripMargin)})(
+    _.sparkSession.sql(s"SELECT * FROM t$tableNr"))
+}