diff --git a/docs/FAQ.md b/docs/FAQ.md
index 16b888e9da7..4e6498b6134 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -10,10 +10,10 @@ nav_order: 11
### What versions of Apache Spark does the RAPIDS Accelerator for Apache Spark support?
-The RAPIDS Accelerator for Apache Spark requires version 3.0.0 or 3.0.1 of Apache Spark. Because the
-plugin replaces parts of the physical plan that Apache Spark considers to be internal the code for
-those plans can change even between bug fix releases. As a part of our process, we try to stay on
-top of these changes and release updates as quickly as possible.
+The RAPIDS Accelerator for Apache Spark requires version 3.0.0, 3.0.1, 3.0.2 or 3.1.1 of Apache
+Spark. Because the plugin replaces parts of the physical plan that Apache Spark considers to be
+internal the code for those plans can change even between bug fix releases. As a part of our
+process, we try to stay on top of these changes and release updates as quickly as possible.
### Which distributions are supported?
diff --git a/docs/download.md b/docs/download.md
index cae33496dae..f743eddfb38 100644
--- a/docs/download.md
+++ b/docs/download.md
@@ -21,8 +21,8 @@ This release includes additional performance improvements, including
* Instructions on how to use [Alluxio caching](get-started/getting-started-alluxio.md) with Spark to
leverage caching.
-The release is supported on Apache Spark 3.0.0, 3.0.1, 3.1.1, Databricks 7.3 ML LTS and Google Cloud
-Platform Dataproc 2.0.
+The release is supported on Apache Spark 3.0.0, 3.0.1, 3.0.2, 3.1.1, Databricks 7.3 ML LTS and
+Google Cloud Platform Dataproc 2.0.
The list of all supported operations is provided [here](supported_ops.md).
diff --git a/pom.xml b/pom.xml
index 0f8139e4055..caa35293b89 100644
--- a/pom.xml
+++ b/pom.xml
@@ -205,7 +205,7 @@
3.0.1
3.0.1-databricks
3.0.2
- 3.1.1-SNAPSHOT
+ 3.1.1
3.2.0-SNAPSHOT
3.6.0
4.3.0
diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml
index efd44b9a4c2..e05bdae7f91 100644
--- a/shims/aggregator/pom.xml
+++ b/shims/aggregator/pom.xml
@@ -68,12 +68,6 @@
${project.version}
compile
-
- com.nvidia
- rapids-4-spark-shims-spark311_${scala.binary.version}
- ${project.version}
- compile
-
@@ -109,5 +103,11 @@
${project.version}
compile
+
+ com.nvidia
+ rapids-4-spark-shims-spark311_${scala.binary.version}
+ ${project.version}
+ compile
+
diff --git a/shims/pom.xml b/shims/pom.xml
index 1dca15fdd09..7d0490f8406 100644
--- a/shims/pom.xml
+++ b/shims/pom.xml
@@ -45,7 +45,6 @@
true
- spark311
spark320
@@ -57,6 +56,7 @@
spark301emr
spark301
spark302
+ spark311
aggregator
diff --git a/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
index 8ce22923520..3e7e694564a 100644
--- a/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
@@ -19,10 +19,8 @@ package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion}
object SparkShimServiceProvider {
- // keep 3.1.0 snapshot version for now until 3.1.1 is released
- val VERSION310 = SparkShimVersion(3, 1, 0)
val VERSION = SparkShimVersion(3, 1, 1)
- val VERSIONNAMES = Seq(s"$VERSION310-SNAPSHOT", s"$VERSION", s"$VERSION-SNAPSHOT")
+ val VERSIONNAMES = Seq(s"$VERSION")
}
class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala
index 7cd815179e9..39d6cfb6442 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ import org.apache.spark.internal.io.FileCommitProtocol
import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition}
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionPathString
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
@@ -204,12 +204,7 @@ case class GpuInsertIntoHadoopFsRelationCommand(
committer: FileCommitProtocol): Unit = {
val staticPartitionPrefix = if (staticPartitions.nonEmpty) {
"/" + partitionColumns.flatMap { p =>
- staticPartitions.get(p.name) match {
- case Some(value) =>
- Some(escapePathName(p.name) + "=" + escapePathName(value))
- case None =>
- None
- }
+ staticPartitions.get(p.name).map(getPartitionPathString(p.name, _))
}.mkString("/")
} else {
""
diff --git a/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala
new file mode 100644
index 00000000000..4865c9aa910
--- /dev/null
+++ b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids
+
+import org.scalatest.BeforeAndAfterEach
+
+class InsertPartition311Suite extends SparkQueryCompareTestSuite with BeforeAndAfterEach {
+ var tableNr = 0
+
+ override def afterEach(): Unit = {
+ List(1, 2).foreach { tnr =>
+ SparkSessionHolder.sparkSession.sql(s"DROP TABLE IF EXISTS t$tnr")
+ }
+ }
+
+ testSparkResultsAreEqual(
+ testName ="Insert null-value partition ",
+ spark => {
+ tableNr += 1
+ spark.sql(s"""CREATE TABLE t${tableNr}(i STRING, c STRING)
+ |USING PARQUET PARTITIONED BY (c)""".stripMargin)
+ spark.sql(s"""INSERT OVERWRITE t${tableNr} PARTITION (c=null)
+ |VALUES ('1')""".stripMargin)})(
+ _.sparkSession.sql(s"SELECT * FROM t$tableNr"))
+}