diff --git a/docs/FAQ.md b/docs/FAQ.md index 16b888e9da7..4e6498b6134 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -10,10 +10,10 @@ nav_order: 11 ### What versions of Apache Spark does the RAPIDS Accelerator for Apache Spark support? -The RAPIDS Accelerator for Apache Spark requires version 3.0.0 or 3.0.1 of Apache Spark. Because the -plugin replaces parts of the physical plan that Apache Spark considers to be internal the code for -those plans can change even between bug fix releases. As a part of our process, we try to stay on -top of these changes and release updates as quickly as possible. +The RAPIDS Accelerator for Apache Spark requires version 3.0.0, 3.0.1, 3.0.2 or 3.1.1 of Apache +Spark. Because the plugin replaces parts of the physical plan that Apache Spark considers to be +internal the code for those plans can change even between bug fix releases. As a part of our +process, we try to stay on top of these changes and release updates as quickly as possible. ### Which distributions are supported? diff --git a/docs/download.md b/docs/download.md index cae33496dae..f743eddfb38 100644 --- a/docs/download.md +++ b/docs/download.md @@ -21,8 +21,8 @@ This release includes additional performance improvements, including * Instructions on how to use [Alluxio caching](get-started/getting-started-alluxio.md) with Spark to leverage caching. -The release is supported on Apache Spark 3.0.0, 3.0.1, 3.1.1, Databricks 7.3 ML LTS and Google Cloud -Platform Dataproc 2.0. +The release is supported on Apache Spark 3.0.0, 3.0.1, 3.0.2, 3.1.1, Databricks 7.3 ML LTS and +Google Cloud Platform Dataproc 2.0. The list of all supported operations is provided [here](supported_ops.md). diff --git a/pom.xml b/pom.xml index 0f8139e4055..caa35293b89 100644 --- a/pom.xml +++ b/pom.xml @@ -205,7 +205,7 @@ 3.0.1 3.0.1-databricks 3.0.2 - 3.1.1-SNAPSHOT + 3.1.1 3.2.0-SNAPSHOT 3.6.0 4.3.0 diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml index efd44b9a4c2..e05bdae7f91 100644 --- a/shims/aggregator/pom.xml +++ b/shims/aggregator/pom.xml @@ -68,12 +68,6 @@ ${project.version} compile - - com.nvidia - rapids-4-spark-shims-spark311_${scala.binary.version} - ${project.version} - compile - @@ -109,5 +103,11 @@ ${project.version} compile + + com.nvidia + rapids-4-spark-shims-spark311_${scala.binary.version} + ${project.version} + compile + diff --git a/shims/pom.xml b/shims/pom.xml index 1dca15fdd09..7d0490f8406 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -45,7 +45,6 @@ true - spark311 spark320 @@ -57,6 +56,7 @@ spark301emr spark301 spark302 + spark311 aggregator diff --git a/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala index 8ce22923520..3e7e694564a 100644 --- a/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala @@ -19,10 +19,8 @@ package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion} object SparkShimServiceProvider { - // keep 3.1.0 snapshot version for now until 3.1.1 is released - val VERSION310 = SparkShimVersion(3, 1, 0) val VERSION = SparkShimVersion(3, 1, 1) - val VERSIONNAMES = Seq(s"$VERSION310-SNAPSHOT", s"$VERSION", s"$VERSION-SNAPSHOT") + val VERSIONNAMES = Seq(s"$VERSION") } class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala index 7cd815179e9..39d6cfb6442 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuInsertIntoHadoopFsRelationCommand.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. + * Copyright (c) 2019-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, CatalogTablePartition} import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName +import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.getPartitionPathString import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap @@ -204,12 +204,7 @@ case class GpuInsertIntoHadoopFsRelationCommand( committer: FileCommitProtocol): Unit = { val staticPartitionPrefix = if (staticPartitions.nonEmpty) { "/" + partitionColumns.flatMap { p => - staticPartitions.get(p.name) match { - case Some(value) => - Some(escapePathName(p.name) + "=" + escapePathName(value)) - case None => - None - } + staticPartitions.get(p.name).map(getPartitionPathString(p.name, _)) }.mkString("/") } else { "" diff --git a/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala new file mode 100644 index 00000000000..4865c9aa910 --- /dev/null +++ b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/InsertPartition311Suite.scala @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids + +import org.scalatest.BeforeAndAfterEach + +class InsertPartition311Suite extends SparkQueryCompareTestSuite with BeforeAndAfterEach { + var tableNr = 0 + + override def afterEach(): Unit = { + List(1, 2).foreach { tnr => + SparkSessionHolder.sparkSession.sql(s"DROP TABLE IF EXISTS t$tnr") + } + } + + testSparkResultsAreEqual( + testName ="Insert null-value partition ", + spark => { + tableNr += 1 + spark.sql(s"""CREATE TABLE t${tableNr}(i STRING, c STRING) + |USING PARQUET PARTITIONED BY (c)""".stripMargin) + spark.sql(s"""INSERT OVERWRITE t${tableNr} PARTITION (c=null) + |VALUES ('1')""".stripMargin)})( + _.sparkSession.sql(s"SELECT * FROM t$tableNr")) +}