Added Spark-3.4.2 Shims (#9967)

* added 342 shims

* Signing off

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

* missed from upmerge

* Fixed Sequence size limit check

* Fixed the overflow check for addition and subtraction

* Updated test conditions

* removed unused file

* Added MockContextBase to 342

---------

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

aggregator/pom.xml

@@ -699,6 +699,23 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>release342</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>342</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.nvidia</groupId>
+                    <artifactId>rapids-4-spark-delta-24x_${scala.binary.version}</artifactId>
+                    <version>${project.version}</version>
+                    <classifier>${spark.version.classifier}</classifier>
+                </dependency>
+            </dependencies>
+        </profile>
         <profile>
             <id>release350</id>
             <activation>

datagen/src/main/spark320/scala/org/apache/spark/sql/tests/datagen/datagen/DataGenExprBase.scala

@@ -33,6 +33,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

integration_tests/src/main/python/collection_ops_test.py

@@ -20,7 +20,7 @@
 from string_test import mk_str_gen
 import pyspark.sql.functions as f
 import pyspark.sql.utils
-from spark_session import with_cpu_session, with_gpu_session, is_before_spark_351
+from spark_session import with_cpu_session, with_gpu_session, is_spark_350, is_before_spark_342, is_before_spark_340
 from conftest import get_datagen_seed
 from marks import allow_non_gpu
 
@@ -326,7 +326,8 @@ def test_sequence_illegal_boundaries(start_gen, stop_gen, step_gen):
 @pytest.mark.parametrize('stop_gen', sequence_too_long_length_gens, ids=idfn)
 @allow_non_gpu(*non_utc_allow)
 def test_sequence_too_long_sequence(stop_gen):
-    msg = "Too long sequence" if is_before_spark_351() else "Unsuccessful try to create array with"
+    msg = "Too long sequence" if is_before_spark_342() \
+    or is_spark_350() else "Unsuccessful try to create array with"
     assert_gpu_and_cpu_error(
         # To avoid OOM, reduce the row number to 1, it is enough to verify this case.
         lambda spark:unary_op_df(spark, stop_gen, 1).selectExpr(

integration_tests/src/main/python/spark_session.py

@@ -193,6 +193,9 @@ def is_before_spark_340():
 def is_before_spark_341():
     return spark_version() < "3.4.1"
 
+def is_before_spark_342():
+    return spark_version() < "3.4.2"
+
 def is_before_spark_350():
     return spark_version() < "3.5.0"
 
@@ -217,6 +220,9 @@ def is_spark_350_or_later():
 def is_spark_330():
     return spark_version() == "3.3.0"
 
+def is_spark_350():
+    return spark_version() == "3.5.0"
+
 def is_spark_33X():
     return "3.3.0" <= spark_version() < "3.4.0"
 

pom.xml

@@ -411,6 +411,26 @@
                 <module>delta-lake/delta-24x</module>
             </modules>
         </profile>
+        <profile>
+            <id>release342</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>342</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>342</buildver>
+                <spark.version>${spark342.version}</spark.version>
+                <spark.test.version>${spark342.version}</spark.test.version>
+                <parquet.hadoop.version>1.12.3</parquet.hadoop.version>
+                <iceberg.version>${spark330.iceberg.version}</iceberg.version>
+                <slf4j.version>2.0.6</slf4j.version>
+            </properties>
+            <modules>
+                <module>delta-lake/delta-24x</module>
+            </modules>
+        </profile>
         <profile>
             <id>release330cdh</id>
             <activation>
@@ -732,6 +752,7 @@
         <spark333.version>3.3.3</spark333.version>
         <spark340.version>3.4.0</spark340.version>
         <spark341.version>3.4.1</spark341.version>
+        <spark342.version>3.4.2</spark342.version>
         <spark330cdh.version>3.3.0.3.3.7180.0-274</spark330cdh.version>
         <spark332cdh.version>3.3.2.3.3.7190.0-91</spark332cdh.version>
         <spark330db.version>3.3.0-databricks</spark330db.version>
@@ -785,6 +806,7 @@
             332cdh,
             340,
             341,
+            342,
             350
         </noSnapshot.buildvers>
         <snapshot.buildvers>

scala2.13/aggregator/pom.xml

@@ -699,6 +699,23 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>release342</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>342</value>
+                </property>
+            </activation>
+            <dependencies>
+                <dependency>
+                    <groupId>com.nvidia</groupId>
+                    <artifactId>rapids-4-spark-delta-24x_${scala.binary.version}</artifactId>
+                    <version>${project.version}</version>
+                    <classifier>${spark.version.classifier}</classifier>
+                </dependency>
+            </dependencies>
+        </profile>
         <profile>
             <id>release350</id>
             <activation>

scala2.13/pom.xml

@@ -411,6 +411,26 @@
                 <module>delta-lake/delta-24x</module>
             </modules>
         </profile>
+        <profile>
+            <id>release342</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>342</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>342</buildver>
+                <spark.version>${spark342.version}</spark.version>
+                <spark.test.version>${spark342.version}</spark.test.version>
+                <parquet.hadoop.version>1.12.3</parquet.hadoop.version>
+                <iceberg.version>${spark330.iceberg.version}</iceberg.version>
+                <slf4j.version>2.0.6</slf4j.version>
+            </properties>
+            <modules>
+                <module>delta-lake/delta-24x</module>
+            </modules>
+        </profile>
         <profile>
             <id>release330cdh</id>
             <activation>
@@ -732,6 +752,7 @@
         <spark333.version>3.3.3</spark333.version>
         <spark340.version>3.4.0</spark340.version>
         <spark341.version>3.4.1</spark341.version>
+        <spark342.version>3.4.2</spark342.version>
         <spark330cdh.version>3.3.0.3.3.7180.0-274</spark330cdh.version>
         <spark332cdh.version>3.3.2.3.3.7190.0-91</spark332cdh.version>
         <spark330db.version>3.3.0-databricks</spark330db.version>
@@ -785,6 +806,7 @@
             332cdh,
             340,
             341,
+            342,
             350
         </noSnapshot.buildvers>
         <snapshot.buildvers>

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AQEUtils.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/AggregationTagging.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/DeltaLakeUtils.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuAggregateInPandasExecMeta.scala

@@ -35,6 +35,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims
 

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuFileFormatDataWriterShim.scala

@@ -34,6 +34,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/GpuWindowInPandasExec.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala

@@ -36,6 +36,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims
 

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala

@@ -36,6 +36,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims
 

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/PythonUDFShim.scala

@@ -35,6 +35,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims
 

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ReaderUtils.scala

@@ -33,6 +33,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/ShimLeafExecNode.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/com/nvidia/spark/rapids/shims/extractValueShims.scala

@@ -33,6 +33,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/org/apache/spark/rapids/shims/GpuShuffleExchangeExec.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/execution/rapids/shims/FilePartitionShims.scala

@@ -36,6 +36,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.execution.rapids.shims
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/FileSinkDescShim.scala

@@ -36,6 +36,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.hive.rapids.shims
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuRowBasedHiveGenericUDFShim.scala

@@ -34,6 +34,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.hive.rapids.shims
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/HiveInspectorsShim.scala

@@ -36,6 +36,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.hive.rapids.shims
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/GpuShuffleMeta.scala

@@ -33,6 +33,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.rapids.execution
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala

@@ -35,6 +35,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuBasePythonRunner.scala

@@ -35,6 +35,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.rapids.execution.python.shims
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala

@@ -35,6 +35,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala

@@ -32,6 +32,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/ArrowUtilsShim.scala

@@ -36,6 +36,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.rapids.shims
 

sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/shims/DataTypeUtilsShim.scala

@@ -36,6 +36,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.rapids.shims
 

sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/ShimSupportsRuntimeFiltering.java

@@ -33,6 +33,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark320/java/com/nvidia/spark/rapids/shims/XxHash64Shims.scala

@@ -33,6 +33,7 @@
 {"spark": "340"}
 {"spark": "341"}
 {"spark": "341db"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/

sql-plugin/src/main/spark320/scala/com/nvidia/spark/rapids/shims/FileSourceScanExecMeta.scala

@@ -29,6 +29,7 @@
 {"spark": "333"}
 {"spark": "340"}
 {"spark": "341"}
+{"spark": "342"}
 {"spark": "350"}
 {"spark": "351"}
 spark-rapids-shim-json-lines ***/