diff --git a/docs/get-started/getting-started-workload-qualification.md b/docs/get-started/getting-started-workload-qualification.md index a35f48d1187..4c173aeed1e 100644 --- a/docs/get-started/getting-started-workload-qualification.md +++ b/docs/get-started/getting-started-workload-qualification.md @@ -64,7 +64,12 @@ the other is to modify your existing Spark application code to call a function d Please note that if using adaptive execution in Spark the explain output may not be perfect as the plan could have changed along the way in a way that we wouldn't see by looking at just the CPU plan. The same applies if you are using an older version of Spark. Spark planning -may be slightly different if you go up to a newer version of Spark. +may be slightly different when you go up to a newer version of Spark. One example where we have +seen Spark 2.4.X plan differently is in the use of the EqualNullSafe expression. We have seen Spark 2.4.X +use EqualNullSafe but in Spark 3.X it used other expressions to do the same thing. In this case +it shows up as GPU doesn't support EqualNullSafe in the Spark 2.X explain output but when you +go to Spark 3.X those parts would run on the GPU because it is using different operators. This +is something to keep in mind when doing the analysis. ### Using the Configuration Flag for Explain Only Mode diff --git a/scripts/rundiffspark2.sh b/scripts/rundiffspark2.sh index 9d31fb61a1f..2bb09caf7a7 100755 --- a/scripts/rundiffspark2.sh +++ b/scripts/rundiffspark2.sh @@ -112,6 +112,11 @@ sed -n '/class GpuStringSplitMeta/,/override def convertToGpu/{/override def co diff $tmp_dir/GpuStringSplitMeta_new.out $tmp_dir/GpuStringSplitMeta_old.out > $tmp_dir/GpuStringSplitMeta.newdiff || true diff -c spark2diffs/GpuStringSplitMeta.diff $tmp_dir/GpuStringSplitMeta.newdiff +sed -n '/class GpuStringToMapMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringMeta.scala > $tmp_dir/GpuStringToMapMeta_new.out +sed -n '/class GpuStringToMapMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala > $tmp_dir/GpuStringToMapMeta_old.out +diff $tmp_dir/GpuStringToMapMeta_new.out $tmp_dir/GpuStringToMapMeta_old.out > $tmp_dir/GpuStringToMapMeta.newdiff || true +diff -c spark2diffs/GpuStringToMapMeta.diff $tmp_dir/GpuStringToMapMeta.newdiff + sed -n '/object GpuOrcFileFormat/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuOrcFileFormat.scala > $tmp_dir/GpuOrcFileFormat_new.out sed -n '/object GpuOrcFileFormat/,/^}/{/^}/!p}' ../sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuOrcFileFormat.scala > $tmp_dir/GpuOrcFileFormat_old.out diff $tmp_dir/GpuOrcFileFormat_new.out $tmp_dir/GpuOrcFileFormat_old.out > $tmp_dir/GpuOrcFileFormat.newdiff || true @@ -181,54 +186,50 @@ diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.s diff -c spark2diffs/GpuOverrides.diff $tmp_dir/GpuOverrides.newdiff sed -n '/GpuOverrides.expr\[Cast\]/,/doFloatToIntCheck/p' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/cast_new.out -sed -n '/GpuOverrides.expr\[Cast\]/,/doFloatToIntCheck/p' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/cast_old.out +sed -n '/GpuOverrides.expr\[Cast\]/,/doFloatToIntCheck/p' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/cast_old.out diff $tmp_dir/cast_new.out $tmp_dir/cast_old.out > $tmp_dir/cast.newdiff || true diff -c spark2diffs/cast.diff $tmp_dir/cast.newdiff sed -n '/GpuOverrides.expr\[Average\]/,/GpuOverrides.expr\[Abs/p' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/average_new.out -sed -n '/GpuOverrides.expr\[Average\]/,/GpuOverrides.expr\[Abs/p' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/average_old.out -diff $tmp_dir/average_new.out $tmp_dir/average_old.out > $tmp_dir/average.newdiff || true +sed -n '/GpuOverrides.expr\[Average\]/,/GpuOverrides.expr\[Abs/p' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/average_old.out +diff -w $tmp_dir/average_new.out $tmp_dir/average_old.out > $tmp_dir/average.newdiff || true diff -c spark2diffs/average.diff $tmp_dir/average.newdiff sed -n '/GpuOverrides.expr\[Abs\]/,/GpuOverrides.expr\[RegExpReplace/p' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/abs_new.out -sed -n '/GpuOverrides.expr\[Abs\]/,/GpuOverrides.expr\[RegExpReplace/p' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/abs_old.out -diff $tmp_dir/abs_new.out $tmp_dir/abs_old.out > $tmp_dir/abs.newdiff || true +sed -n '/GpuOverrides.expr\[Abs\]/,/GpuOverrides.expr\[RegExpReplace/p' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/abs_old.out +diff -w $tmp_dir/abs_new.out $tmp_dir/abs_old.out > $tmp_dir/abs.newdiff || true diff -c spark2diffs/abs.diff $tmp_dir/abs.newdiff -sed -n '/GpuOverrides.expr\[RegExpReplace\]/,/GpuOverrides.expr\[TimeSub/{/GpuOverrides.expr\[TimeSub/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/regexreplace_new.out -sed -n '/GpuOverrides.expr\[RegExpReplace\]/,/GpuScalaUDFMeta.exprMeta/{/GpuScalaUDFMeta.exprMeta/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/regexreplace_old.out -diff -c $tmp_dir/regexreplace_new.out $tmp_dir/regexreplace_old.out - -sed -n '/GpuOverrides.expr\[TimeSub\]/,/GpuOverrides.expr\[ScalaUDF/{/GpuOverrides.expr\[ScalaUDF/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/TimeSub_new.out -sed -n '/GpuOverrides.expr\[TimeSub\]/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/TimeSub_old.out -diff -w $tmp_dir/TimeSub_new.out $tmp_dir/TimeSub_old.out > $tmp_dir/TimeSub.newdiff || true -diff -c spark2diffs/TimeSub.diff $tmp_dir/TimeSub.newdiff +sed -n '/GpuOverrides.expr\[RegExpReplace\]/,/GpuOverrides.expr\[ScalaUDF/{/GpuOverrides.expr\[ScalaUDF/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/regexreplace_new.out +sed -n '/GpuOverrides.expr\[RegExpReplace\]/,/GpuOverrides.expr\[Lead/{/GpuOverrides.expr\[Lead/!p}' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/regexreplace_old.out +diff -w $tmp_dir/regexreplace_new.out $tmp_dir/regexreplace_old.out > $tmp_dir/regexreplace.newdiff || true +diff -c spark2diffs/regexreplace.diff $tmp_dir/regexreplace.newdiff sed -n '/GpuOverrides.expr\[ScalaUDF\]/,/})/{/})/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/ScalaUDF_new.out -sed -n '/GpuOverrides.expr\[ScalaUDF\]/,/})/{/})/!p}' ../sql-plugin/src/main/301until310-all/scala/com/nvidia/spark/rapids/shims/v2/GpuRowBasedScalaUDF.scala > $tmp_dir/ScalaUDF_old.out +sed -n '/GpuOverrides.expr\[ScalaUDF\]/,/})/{/})/!p}' ../sql-plugin/src/main/311+-all/scala/com/nvidia/spark/rapids/shims/GpuRowBasedScalaUDF.scala > $tmp_dir/ScalaUDF_old.out diff -w $tmp_dir/ScalaUDF_new.out $tmp_dir/ScalaUDF_old.out > $tmp_dir/ScalaUDF.newdiff || true diff -c spark2diffs/ScalaUDF.diff $tmp_dir/ScalaUDF.newdiff sed -n '/GpuOverrides.exec\[FileSourceScanExec\]/,/})/{/})/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/FileSourceScanExec_new.out -sed -n '/GpuOverrides.exec\[FileSourceScanExec\]/,/override def convertToCpu/{/override def convertToCpu/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/FileSourceScanExec_old.out +sed -n '/GpuOverrides.exec\[FileSourceScanExec\]/,/override def convertToCpu/{/override def convertToCpu/!p}' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/FileSourceScanExec_old.out diff -w $tmp_dir/FileSourceScanExec_new.out $tmp_dir/FileSourceScanExec_old.out > $tmp_dir/FileSourceScanExec.newdiff || true diff -c spark2diffs/FileSourceScanExec.diff $tmp_dir/FileSourceScanExec.newdiff sed -n '/GpuOverrides.exec\[ArrowEvalPythonExec\]/,/})/{/})/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/ArrowEvalPythonExec_new.out -sed -n '/GpuOverrides.exec\[ArrowEvalPythonExec\]/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/ArrowEvalPythonExec_old.out +sed -n '/GpuOverrides.exec\[ArrowEvalPythonExec\]/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/ArrowEvalPythonExec_old.out diff -w $tmp_dir/ArrowEvalPythonExec_new.out $tmp_dir/ArrowEvalPythonExec_old.out > $tmp_dir/ArrowEvalPythonExec.newdiff || true diff -c spark2diffs/ArrowEvalPythonExec.diff $tmp_dir/ArrowEvalPythonExec.newdiff sed -n '/GpuOverrides.exec\[FlatMapGroupsInPandasExec\]/,/GpuOverrides.exec\[WindowInPandasExec/{/GpuOverrides.exec\[WindowInPandasExec/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/FlatMapGroupsInPandasExec_new.out -sed -n '/GpuOverrides.exec\[FlatMapGroupsInPandasExec\]/,/GpuOverrides.exec\[AggregateInPandasExec/{/GpuOverrides.exec\[AggregateInPandasExec/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/FlatMapGroupsInPandasExec_old.out +sed -n '/GpuOverrides.exec\[FlatMapGroupsInPandasExec\]/,/GpuOverrides.exec\[AggregateInPandasExec/{/GpuOverrides.exec\[AggregateInPandasExec/!p}' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/FlatMapGroupsInPandasExec_old.out diff -c -w $tmp_dir/FlatMapGroupsInPandasExec_new.out $tmp_dir/FlatMapGroupsInPandasExec_old.out sed -n '/GpuOverrides.exec\[WindowInPandasExec\]/,/})/{/})/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/WindowInPandasExec_new.out -sed -n '/GpuOverrides.exec\[WindowInPandasExec\]/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/WindowInPandasExec_old.out +sed -n '/GpuOverrides.exec\[WindowInPandasExec\]/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/WindowInPandasExec_old.out diff -c -w --ignore-blank-lines $tmp_dir/WindowInPandasExec_new.out $tmp_dir/WindowInPandasExec_old.out sed -n '/GpuOverrides.exec\[AggregateInPandasExec\]/,/)\.collect/{/)\.collect/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala > $tmp_dir/AggregateInPandasExec_new.out -sed -n '/GpuOverrides.exec\[AggregateInPandasExec\]/,/)\.map/{/)\.map/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/Spark30XShims.scala > $tmp_dir/AggregateInPandasExec_old.out +sed -n '/GpuOverrides.exec\[AggregateInPandasExec\]/,/)\.map/{/)\.map/!p}' ../sql-plugin/src/main/311until320-nondb/scala/com/nvidia/spark/rapids/shims/Spark31XShims.scala > $tmp_dir/AggregateInPandasExec_old.out diff -c -w $tmp_dir/AggregateInPandasExec_new.out $tmp_dir/AggregateInPandasExec_old.out sed -n '/object GpuOrcScanBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOrcScanBase.scala > $tmp_dir/GpuOrcScanBase_new.out @@ -241,7 +242,7 @@ sed -n '/class LiteralExprMeta/,/^}/{/^}/!p}' ../sql-plugin/src/main/scala/com diff $tmp_dir/LiteralExprMeta_new.out $tmp_dir/LiteralExprMeta_old.out > $tmp_dir/LiteralExprMeta.newdiff || true diff -c spark2diffs/LiteralExprMeta.diff $tmp_dir/LiteralExprMeta.newdiff -# 2.x doesn't have a base aggregate class so this is much different, check the revision for now +# 2.x doesn't have a base aggregate class so this is much different diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/aggregateMeta.scala ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/aggregate.scala > $tmp_dir/aggregate.newdiff || true diff -c spark2diffs/aggregate.diff $tmp_dir/aggregate.newdiff @@ -276,85 +277,85 @@ diff -c spark2diffs/GpuSortMeta.diff $tmp_dir/GpuSortMeta.newdiff diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/InputFileBlockRule.scala ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/InputFileBlockRule.scala > $tmp_dir/InputFileBlockRule.newdiff || true diff -c spark2diffs/InputFileBlockRule.diff $tmp_dir/InputFileBlockRule.newdiff -sed -n '/abstract class GpuWindowInPandasExecMetaBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuPandasMeta.scala > $tmp_dir/GpuWindowInPandasExecMetaBase_new.out +sed -n '/abstract class GpuWindowInPandasExecMetaBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuPandasMeta.scala > $tmp_dir/GpuWindowInPandasExecMetaBase_new.out sed -n '/abstract class GpuWindowInPandasExecMetaBase/,/^}/{/^}/!p}' ../sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuWindowInPandasExecBase.scala > $tmp_dir/GpuWindowInPandasExecMetaBase_old.out diff $tmp_dir/GpuWindowInPandasExecMetaBase_new.out $tmp_dir/GpuWindowInPandasExecMetaBase_old.out > $tmp_dir/GpuWindowInPandasExecMetaBase.newdiff || true diff -c spark2diffs/GpuWindowInPandasExecMetaBase.diff $tmp_dir/GpuWindowInPandasExecMetaBase.newdiff -sed -n '/class GpuAggregateInPandasExecMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuPandasMeta.scala > $tmp_dir/GpuAggregateInPandasExecMeta_new.out +sed -n '/class GpuAggregateInPandasExecMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuPandasMeta.scala > $tmp_dir/GpuAggregateInPandasExecMeta_new.out sed -n '/class GpuAggregateInPandasExecMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/python/GpuAggregateInPandasExec.scala > $tmp_dir/GpuAggregateInPandasExecMeta_old.out diff $tmp_dir/GpuAggregateInPandasExecMeta_new.out $tmp_dir/GpuAggregateInPandasExecMeta_old.out > $tmp_dir/GpuAggregateInPandasExecMeta.newdiff || true diff -c spark2diffs/GpuAggregateInPandasExecMeta.diff $tmp_dir/GpuAggregateInPandasExecMeta.newdiff -sed -n '/class GpuFlatMapGroupsInPandasExecMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuPandasMeta.scala > $tmp_dir/GpuFlatMapGroupsInPandasExecMeta_new.out -sed -n '/class GpuFlatMapGroupsInPandasExecMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/301+-nondb/scala/org/apache/spark/sql/rapids/execution/python/shims/v2/GpuFlatMapGroupsInPandasExec.scala > $tmp_dir/GpuFlatMapGroupsInPandasExecMeta_old.out +sed -n '/class GpuFlatMapGroupsInPandasExecMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuPandasMeta.scala > $tmp_dir/GpuFlatMapGroupsInPandasExecMeta_new.out +sed -n '/class GpuFlatMapGroupsInPandasExecMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/311+-nondb/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuFlatMapGroupsInPandasExec.scala > $tmp_dir/GpuFlatMapGroupsInPandasExecMeta_old.out diff $tmp_dir/GpuFlatMapGroupsInPandasExecMeta_new.out $tmp_dir/GpuFlatMapGroupsInPandasExecMeta_old.out > $tmp_dir/GpuFlatMapGroupsInPandasExecMeta.newdiff || true diff -c spark2diffs/GpuFlatMapGroupsInPandasExecMeta.diff $tmp_dir/GpuFlatMapGroupsInPandasExecMeta.newdiff -sed -n '/class GpuShuffledHashJoinMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/GpuShuffledHashJoinExecMeta.scala > $tmp_dir/GpuShuffledHashJoinMeta_new.out +sed -n '/class GpuShuffledHashJoinMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuShuffledHashJoinExecMeta.scala > $tmp_dir/GpuShuffledHashJoinMeta_new.out sed -n '/class GpuShuffledHashJoinMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuShuffledHashJoinExec.scala > $tmp_dir/GpuShuffledHashJoinMeta_old.out diff $tmp_dir/GpuShuffledHashJoinMeta_new.out $tmp_dir/GpuShuffledHashJoinMeta_old.out > $tmp_dir/GpuShuffledHashJoinMeta.newdiff || true diff -c spark2diffs/GpuShuffledHashJoinMeta.diff $tmp_dir/GpuShuffledHashJoinMeta.newdiff -diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/TreeNode.scala ../sql-plugin/src/main/pre320-treenode/scala/com/nvidia/spark/rapids/shims/v2/TreeNode.scala > $tmp_dir/TreeNode.newdiff || true +diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/TreeNode.scala ../sql-plugin/src/main/pre320-treenode/scala/com/nvidia/spark/rapids/shims/TreeNode.scala > $tmp_dir/TreeNode.newdiff || true diff -c spark2diffs/TreeNode.diff $tmp_dir/TreeNode.newdiff -diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/GpuSortMergeJoinMeta.scala ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinMeta.scala > $tmp_dir/GpuSortMergeJoinMeta.newdiff || true +diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuSortMergeJoinMeta.scala ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuSortMergeJoinMeta.scala > $tmp_dir/GpuSortMergeJoinMeta.newdiff || true diff -c spark2diffs/GpuSortMergeJoinMeta.diff $tmp_dir/GpuSortMergeJoinMeta.newdiff -diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/GpuJoinUtils.scala ../sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/shims/v2/GpuJoinUtils.scala > $tmp_dir/GpuJoinUtils.newdiff || true +diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuJoinUtils.scala ../sql-plugin/src/main/311+-all/scala/com/nvidia/spark/rapids/shims/GpuJoinUtils.scala > $tmp_dir/GpuJoinUtils.newdiff || true diff -c spark2diffs/GpuJoinUtils.diff $tmp_dir/GpuJoinUtils.newdiff -diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/TypeSigUtil.scala ../sql-plugin/src/main/301until320-all/scala/com/nvidia/spark/rapids/shims/v2/TypeSigUtil.scala > $tmp_dir/TypeSigUtil.newdiff || true +diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala ../sql-plugin/src/main/311until320-all/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala > $tmp_dir/TypeSigUtil.newdiff || true diff -c spark2diffs/TypeSigUtil.diff $tmp_dir/TypeSigUtil.newdiff -sed -n '/class GpuBroadcastHashJoinMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/GpuBroadcastHashJoinExecMeta.scala > $tmp_dir/GpuBroadcastHashJoinMeta_new.out +sed -n '/class GpuBroadcastHashJoinMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuBroadcastHashJoinExecMeta.scala > $tmp_dir/GpuBroadcastHashJoinMeta_new.out sed -n '/class GpuBroadcastHashJoinMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuBroadcastHashJoinExec.scala > $tmp_dir/GpuBroadcastHashJoinMeta_old.out diff $tmp_dir/GpuBroadcastHashJoinMeta_new.out $tmp_dir/GpuBroadcastHashJoinMeta_old.out > $tmp_dir/GpuBroadcastHashJoinMeta.newdiff || true diff -c spark2diffs/GpuBroadcastHashJoinMeta.diff $tmp_dir/GpuBroadcastHashJoinMeta.newdiff -sed -n '/object GpuCSVScan/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/GpuCSVScan.scala > $tmp_dir/GpuCSVScan_new.out +sed -n '/object GpuCSVScan/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuCSVScan.scala > $tmp_dir/GpuCSVScan_new.out sed -n '/object GpuCSVScan/,/^}/{/^}/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuBatchScanExec.scala > $tmp_dir/GpuCSVScan_old.out diff $tmp_dir/GpuCSVScan_new.out $tmp_dir/GpuCSVScan_old.out > $tmp_dir/GpuCSVScan.newdiff || true diff -c spark2diffs/GpuCSVScan.diff $tmp_dir/GpuCSVScan.newdiff -diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/OffsetWindowFunctionMeta.scala ../sql-plugin/src/main/301until310-all/scala/com/nvidia/spark/rapids/shims/v2/OffsetWindowFunctionMeta.scala > $tmp_dir/OffsetWindowFunctionMeta.newdiff || true +diff ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala ../sql-plugin/src/main/311until320-all/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala > $tmp_dir/OffsetWindowFunctionMeta.newdiff || true diff -c spark2diffs/OffsetWindowFunctionMeta.diff $tmp_dir/OffsetWindowFunctionMeta.newdiff -sed -n '/class GpuRegExpReplaceMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/GpuRegExpReplaceMeta.scala > $tmp_dir/GpuRegExpReplaceMeta_new.out -sed -n '/class GpuRegExpReplaceMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/301until310-nondb/scala/com/nvidia/spark/rapids/shims/v2/GpuRegExpReplaceMeta.scala > $tmp_dir/GpuRegExpReplaceMeta_old.out +sed -n '/class GpuRegExpReplaceMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuRegExpReplaceMeta.scala > $tmp_dir/GpuRegExpReplaceMeta_new.out +sed -n '/class GpuRegExpReplaceMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/311+-nondb/scala/com/nvidia/spark/rapids/shims/GpuRegExpReplaceExec.scala > $tmp_dir/GpuRegExpReplaceMeta_old.out diff $tmp_dir/GpuRegExpReplaceMeta_new.out $tmp_dir/GpuRegExpReplaceMeta_old.out > $tmp_dir/GpuRegExpReplaceMeta.newdiff || true diff -c spark2diffs/GpuRegExpReplaceMeta.diff $tmp_dir/GpuRegExpReplaceMeta.newdiff -sed -n '/class GpuWindowExpressionMetaBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuWindowExpressionMetaBase_new.out +sed -n '/class GpuWindowExpressionMetaBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuWindowExpressionMetaBase_new.out sed -n '/class GpuWindowExpressionMetaBase/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuWindowExpression.scala > $tmp_dir/GpuWindowExpressionMetaBase_old.out diff $tmp_dir/GpuWindowExpressionMetaBase_new.out $tmp_dir/GpuWindowExpressionMetaBase_old.out > $tmp_dir/GpuWindowExpressionMetaBase.newdiff || true diff -c spark2diffs/GpuWindowExpressionMetaBase.diff $tmp_dir/GpuWindowExpressionMetaBase.newdiff -sed -n '/abstract class GpuSpecifiedWindowFrameMetaBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuSpecifiedWindowFrameMetaBase_new.out +sed -n '/abstract class GpuSpecifiedWindowFrameMetaBase/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuSpecifiedWindowFrameMetaBase_new.out sed -n '/abstract class GpuSpecifiedWindowFrameMetaBase/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuWindowExpression.scala > $tmp_dir/GpuSpecifiedWindowFrameMetaBase_old.out diff $tmp_dir/GpuSpecifiedWindowFrameMetaBase_new.out $tmp_dir/GpuSpecifiedWindowFrameMetaBase_old.out > $tmp_dir/GpuSpecifiedWindowFrameMetaBase.newdiff || true diff -c spark2diffs/GpuSpecifiedWindowFrameMetaBase.diff $tmp_dir/GpuSpecifiedWindowFrameMetaBase.newdiff -sed -n '/class GpuSpecifiedWindowFrameMeta(/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuSpecifiedWindowFrameMeta_new.out -sed -n '/class GpuSpecifiedWindowFrameMeta(/,/^}/{/^}/!p}' ../sql-plugin/src/main/301until320-all/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuSpecifiedWindowFrameMeta_old.out +sed -n '/class GpuSpecifiedWindowFrameMeta(/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuSpecifiedWindowFrameMeta_new.out +sed -n '/class GpuSpecifiedWindowFrameMeta(/,/^}/{/^}/!p}' ../sql-plugin/src/main/311until320-all/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuSpecifiedWindowFrameMeta_old.out diff $tmp_dir/GpuSpecifiedWindowFrameMeta_new.out $tmp_dir/GpuSpecifiedWindowFrameMeta_old.out > $tmp_dir/GpuSpecifiedWindowFrameMeta.newdiff || true diff -c spark2diffs/GpuSpecifiedWindowFrameMeta.diff $tmp_dir/GpuSpecifiedWindowFrameMeta.newdiff -sed -n '/class GpuWindowExpressionMeta(/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuWindowExpressionMeta_new.out -sed -n '/class GpuWindowExpressionMeta(/,/^}/{/^}/!p}' ../sql-plugin/src/main/301until320-all/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuWindowExpressionMeta_old.out +sed -n '/class GpuWindowExpressionMeta(/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuWindowExpressionMeta_new.out +sed -n '/class GpuWindowExpressionMeta(/,/^}/{/^}/!p}' ../sql-plugin/src/main/311until320-all/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuWindowExpressionMeta_old.out diff $tmp_dir/GpuWindowExpressionMeta_new.out $tmp_dir/GpuWindowExpressionMeta_old.out > $tmp_dir/GpuWindowExpressionMeta.newdiff || true diff -c spark2diffs/GpuWindowExpressionMeta.diff $tmp_dir/GpuWindowExpressionMeta.newdiff -sed -n '/object GpuWindowUtil/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuWindowUtil_new.out -sed -n '/object GpuWindowUtil/,/^}/{/^}/!p}' ../sql-plugin/src/main/301until320-all/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuWindowUtil_old.out +sed -n '/object GpuWindowUtil/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuWindowUtil_new.out +sed -n '/object GpuWindowUtil/,/^}/{/^}/!p}' ../sql-plugin/src/main/311until320-all/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuWindowUtil_old.out diff -c $tmp_dir/GpuWindowUtil_new.out $tmp_dir/GpuWindowUtil_old.out -sed -n '/case class ParsedBoundary/p' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/ParsedBoundary_new.out +sed -n '/case class ParsedBoundary/p' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/ParsedBoundary_new.out sed -n '/case class ParsedBoundary/p' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuWindowExec.scala > $tmp_dir/ParsedBoundary_old.out diff -c $tmp_dir/ParsedBoundary_new.out $tmp_dir/ParsedBoundary_old.out -sed -n '/class GpuWindowSpecDefinitionMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/v2/gpuWindows.scala > $tmp_dir/GpuWindowSpecDefinitionMeta_new.out +sed -n '/class GpuWindowSpecDefinitionMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/gpuWindows.scala > $tmp_dir/GpuWindowSpecDefinitionMeta_new.out sed -n '/class GpuWindowSpecDefinitionMeta/,/override def convertToGpu/{/override def convertToGpu/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuWindowExpression.scala > $tmp_dir/GpuWindowSpecDefinitionMeta_old.out diff $tmp_dir/GpuWindowSpecDefinitionMeta_new.out $tmp_dir/GpuWindowSpecDefinitionMeta_old.out > $tmp_dir/GpuWindowSpecDefinitionMeta.newdiff || true diff -c spark2diffs/GpuWindowSpecDefinitionMeta.diff $tmp_dir/GpuWindowSpecDefinitionMeta.newdiff @@ -427,19 +428,14 @@ sed -n '/def optionallyAsDecimalType/,/^ }/{/^ }/!p}' ../sql-plugin/src/main/ diff $tmp_dir/optionallyAsDecimalType_new.out $tmp_dir/optionallyAsDecimalType_old.out > $tmp_dir/optionallyAsDecimalType.newdiff || true diff -c spark2diffs/optionallyAsDecimalType.diff $tmp_dir/optionallyAsDecimalType.newdiff -sed -n '/def getPrecisionForIntegralType/,/^ }/{/^ }/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/getPrecisionForIntegralType_new.out -sed -n '/def getPrecisionForIntegralType/,/^ }/{/^ }/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/getPrecisionForIntegralType_old.out -diff $tmp_dir/getPrecisionForIntegralType_new.out $tmp_dir/getPrecisionForIntegralType_old.out > $tmp_dir/getPrecisionForIntegralType.newdiff || true -diff -c spark2diffs/getPrecisionForIntegralType.diff $tmp_dir/getPrecisionForIntegralType.newdiff - # not sure this diff works very well due to java vs scala and quite a bit different but should find any changes in those functions sed -n '/def toRapidsStringOrNull/,/^ }/{/^ }/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/toRapidsStringOrNull_new.out -sed -n '/private static DType/,/^ }/{/^ }/!p}' ../sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVector.java > $tmp_dir/toRapidsStringOrNull_old.out +sed -n '/private static DType toRapidsOrNullCommon/,/^ }/{/^ }/!p}' ../sql-plugin/src/main/java/com/nvidia/spark/rapids/GpuColumnVector.java > $tmp_dir/toRapidsStringOrNull_old.out diff $tmp_dir/toRapidsStringOrNull_new.out $tmp_dir/toRapidsStringOrNull_old.out > $tmp_dir/toRapidsStringOrNull.newdiff || true diff -c spark2diffs/toRapidsStringOrNull.diff $tmp_dir/toRapidsStringOrNull.newdiff -sed -n '/def createCudfDecimal/,/^ }/{/^ }/!p}' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/createCudfDecimal_new.out -sed -n '/def createCudfDecimal/,/^ }/{/^ }/!p}' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/createCudfDecimal_old.out +sed -n '/def createCudfDecimal/,/^ /p' ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/createCudfDecimal_new.out +sed -n '/def createCudfDecimal/,/^ /p' ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala > $tmp_dir/createCudfDecimal_old.out diff $tmp_dir/createCudfDecimal_new.out $tmp_dir/createCudfDecimal_old.out > $tmp_dir/createCudfDecimal.newdiff || true diff -c spark2diffs/createCudfDecimal.diff $tmp_dir/createCudfDecimal.newdiff @@ -491,5 +487,20 @@ sed -n '/object GpuJsonScan/,/^}/{/^}/!p}' ../sql-plugin/src/main/scala/org/apa diff $tmp_dir/GpuJsonScan_new.out $tmp_dir/GpuJsonScan_old.out > $tmp_dir/GpuJsonScan.newdiff || true diff -c spark2diffs/GpuJsonScan.diff $tmp_dir/GpuJsonScan.newdiff +sed -n '/object GpuCsvUtils/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala > $tmp_dir/GpuCsvUtils_new.out +sed -n '/object GpuCsvUtils/,/^}/{/^}/!p}' ../sql-plugin/src/main/311until330-nondb/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala > $tmp_dir/GpuCsvUtils_old.out +diff $tmp_dir/GpuCsvUtils_new.out $tmp_dir/GpuCsvUtils_old.out > $tmp_dir/GpuCsvUtils.newdiff || true +diff -c spark2diffs/GpuCsvUtils.diff $tmp_dir/GpuCsvUtils.newdiff + +sed -n '/abstract class StringSplitRegExpMeta/,/^}/{/^}/!p}' ../spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringMeta.scala > $tmp_dir/StringSplitRegExpMeta_new.out +sed -n '/abstract class StringSplitRegExpMeta/,/^}/{/^}/!p}' ../sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala > $tmp_dir/StringSplitRegExpMeta_old.out +diff $tmp_dir/StringSplitRegExpMeta_new.out $tmp_dir/StringSplitRegExpMeta_old.out > $tmp_dir/StringSplitRegExpMeta.newdiff || true +diff -c spark2diffs/StringSplitRegExpMeta.diff $tmp_dir/StringSplitRegExpMeta.newdiff + +diff -c ../spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataTypes.scala ../sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataTypes.scala + +diff ../spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/expressions/rapids/Timestamp.scala ../sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/expressions/rapids/Timestamp.scala > $tmp_dir/Timestamp.newdiff || true +diff -c spark2diffs/Timestamp.diff $tmp_dir/Timestamp.newdiff + echo "Done running Diffs of spark2.x files" rm -r $tmp_dir diff --git a/scripts/spark2diffs/CastExprMeta.diff b/scripts/spark2diffs/CastExprMeta.diff index 97fb9b3dcac..366bb4c4cfd 100644 --- a/scripts/spark2diffs/CastExprMeta.diff +++ b/scripts/spark2diffs/CastExprMeta.diff @@ -10,7 +10,7 @@ < // 2.x doesn't have the SQLConf.LEGACY_COMPLEX_TYPES_TO_STRING config, so set it to true < val legacyCastToString: Boolean = true --- -> val legacyCastToString: Boolean = ShimLoader.getSparkShims.getLegacyComplexTypeToString() +> val legacyCastToString: Boolean = SparkShimImpl.getLegacyComplexTypeToString() 46c45 < if (dt.precision > GpuOverrides.DECIMAL128_MAX_PRECISION) { --- @@ -27,13 +27,13 @@ > YearParseUtil.tagParseStringAsDate(conf, this) 85,91c85 < // Spark 2.x: removed check for -< // !ShimLoader.getSparkShims.isCastingStringToNegDecimalScaleSupported +< // !SparkShimImpl.isCastingStringToNegDecimalScaleSupported < // this dealt with handling a bug fix that is only in newer versions of Spark < // (https://issues.apache.org/jira/browse/SPARK-37451) < // Since we don't know what version of Spark 3 they will be using < // just always say it won't work and they can hopefully figure it out from warning. < if (dt.scale < 0) { --- -> if (dt.scale < 0 && !ShimLoader.getSparkShims.isCastingStringToNegDecimalScaleSupported) { -120a115 +> if (dt.scale < 0 && !SparkShimImpl.isCastingStringToNegDecimalScaleSupported) { +124a119 > diff --git a/scripts/spark2diffs/DateUtils.diff b/scripts/spark2diffs/DateUtils.diff index 38fed34ab05..727295f9baf 100644 --- a/scripts/spark2diffs/DateUtils.diff +++ b/scripts/spark2diffs/DateUtils.diff @@ -1,7 +1,7 @@ 2c2 < * Copyright (c) 2022, NVIDIA CORPORATION. --- -> * Copyright (c) 2020-2021, NVIDIA CORPORATION. +> * Copyright (c) 2020-2022, NVIDIA CORPORATION. 19c19 < import java.time._ --- @@ -12,35 +12,35 @@ > 23a27 > import org.apache.spark.sql.catalyst.util.DateTimeUtils.localDateToDays -59,60c63,65 +61,62c65,67 < // Spark 2.x - removed isSpark320orlater checks < def specialDatesDays: Map[String, Int] = { --- > def specialDatesDays: Map[String, Int] = if (isSpark320OrLater) { > Map.empty > } else { -71c76,78 +73c78,80 < def specialDatesSeconds: Map[String, Long] = { --- > def specialDatesSeconds: Map[String, Long] = if (isSpark320OrLater) { > Map.empty > } else { -73,74c80 +75,76c82 < // spark 2.4 Date utils are different < val now = DateTimeUtils.instantToMicros(Instant.now()) --- > val now = DateTimeUtils.currentTimestamp() -84c90,92 +86c92,94 < def specialDatesMicros: Map[String, Long] = { --- > def specialDatesMicros: Map[String, Long] = if (isSpark320OrLater) { > Map.empty > } else { -86c94 +88c96 < val now = DateTimeUtils.instantToMicros(Instant.now()) --- > val now = DateTimeUtils.currentTimestamp() -96c104,121 +98c106,123 < def currentDate(): Int = Math.toIntExact(LocalDate.now().toEpochDay) --- > def fetchSpecialDates(unit: DType): Map[String, () => Scalar] = unit match { @@ -61,3 +61,13 @@ > } > > def currentDate(): Int = localDateToDays(LocalDate.now()) +193c218 +< meta: RapidsMeta[_, _], +--- +> meta: RapidsMeta[_, _, _], +209,211c234 +< // Spark 2.x doesn't support, assume false +< val ansiEnabled = false +< if (ansiEnabled) { +--- +> if (SQLConf.get.ansiEnabled) { diff --git a/scripts/spark2diffs/GpuCSVScan.diff b/scripts/spark2diffs/GpuCSVScan.diff index 4ae82fa5ac5..8ea3e1a9eb0 100644 --- a/scripts/spark2diffs/GpuCSVScan.diff +++ b/scripts/spark2diffs/GpuCSVScan.diff @@ -38,17 +38,13 @@ < /* 113d103 < */ -143c133 -< dateFormatInRead(parsedOptions).foreach { dateFormat => ---- -> ShimLoader.getSparkShims.dateFormatInRead(parsedOptions).foreach { dateFormat => -190,192c180 +154,156c144 < < // Spark 2.x doesn't have zoneId, so use timeZone and then to id < if (!TypeChecks.areTimestampsSupported(parsedOptions.timeZone.toZoneId)) { --- > if (!TypeChecks.areTimestampsSupported(parsedOptions.zoneId)) { -195c183 +159c147 < timestampFormatInRead(parsedOptions).foreach { tsFormat => --- -> ShimLoader.getSparkShims.timestampFormatInRead(parsedOptions).foreach { tsFormat => +> FileOptionsShims.timestampFormatInRead(parsedOptions).foreach { tsFormat => diff --git a/scripts/spark2diffs/GpuCsvUtils.diff b/scripts/spark2diffs/GpuCsvUtils.diff new file mode 100644 index 00000000000..8c4a9a7c3de --- /dev/null +++ b/scripts/spark2diffs/GpuCsvUtils.diff @@ -0,0 +1,5 @@ +2,3c2 +< // spark 2.x uses FastDateFormat, use getPattern +< def dateFormatInRead(options: CSVOptions): String = options.dateFormat.getPattern +--- +> def dateFormatInRead(options: CSVOptions): String = options.dateFormat diff --git a/scripts/spark2diffs/GpuFileSourceScanExec.diff b/scripts/spark2diffs/GpuFileSourceScanExec.diff index fa522395fe7..bd0354e002f 100644 --- a/scripts/spark2diffs/GpuFileSourceScanExec.diff +++ b/scripts/spark2diffs/GpuFileSourceScanExec.diff @@ -1,4 +1,11 @@ -11a12,22 +8,10c8,18 +< // SPARK 2.x - We leave off Avro here since its a datasource v2 thing and off by default +< case f => +< meta.willNotWorkOnGpu(s"unsupported file format: ${f.getClass.getCanonicalName}") +--- +> case _ => ExternalSource.tagSupportForGpuFileSourceScanExec(meta) +> } +> } > > def convertFileFormat(format: FileFormat): FileFormat = { > format match { @@ -6,7 +13,4 @@ > case f if GpuOrcFileFormat.isSparkOrcFormat(f) => new GpuReadOrcFileFormat > case _: ParquetFileFormat => new GpuReadParquetFileFormat > case _: JsonFileFormat => new GpuReadJsonFileFormat -> case f => -> throw new IllegalArgumentException(s"${f.getClass.getCanonicalName} is not supported") -> } -> } +> case _ => ExternalSource.convertFileFormatForGpuFileSourceScanExec(format) diff --git a/scripts/spark2diffs/GpuGetArrayItemMeta.diff b/scripts/spark2diffs/GpuGetArrayItemMeta.diff index 4fbbd1d5d44..5779c3dda25 100644 --- a/scripts/spark2diffs/GpuGetArrayItemMeta.diff +++ b/scripts/spark2diffs/GpuGetArrayItemMeta.diff @@ -2,3 +2,5 @@ < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], +6a7 +> diff --git a/scripts/spark2diffs/GpuHashJoin.diff b/scripts/spark2diffs/GpuHashJoin.diff index 5771c5f5834..6cc06c26fd3 100644 --- a/scripts/spark2diffs/GpuHashJoin.diff +++ b/scripts/spark2diffs/GpuHashJoin.diff @@ -2,13 +2,13 @@ < def tagForGpu(joinType: JoinType, meta: RapidsMeta[_, _]): Unit = { --- > def tagForGpu(joinType: JoinType, meta: RapidsMeta[_, _, _]): Unit = { -69c69 +72c72 < object GpuHashJoin { --- > object GpuHashJoin extends Arm { -101a102 +98a99 > -122c123 +119c120 < } --- > diff --git a/scripts/spark2diffs/GpuJoinUtils.diff b/scripts/spark2diffs/GpuJoinUtils.diff index 3b7ea872635..1b465d1dacb 100644 --- a/scripts/spark2diffs/GpuJoinUtils.diff +++ b/scripts/spark2diffs/GpuJoinUtils.diff @@ -1,7 +1,7 @@ 16,18d15 -< package com.nvidia.spark.rapids.shims.v2 +< package com.nvidia.spark.rapids.shims < -< import com.nvidia.spark.rapids.shims.v2._ +< import com.nvidia.spark.rapids.shims._ 20,26c17 < import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide} < @@ -11,7 +11,7 @@ < */ < sealed abstract class GpuBuildSide --- -> package com.nvidia.spark.rapids.shims.v2 +> package com.nvidia.spark.rapids.shims 28c19 < case object GpuBuildRight extends GpuBuildSide --- diff --git a/scripts/spark2diffs/GpuJsonScan.diff b/scripts/spark2diffs/GpuJsonScan.diff index f5e1e2496dc..38fae493022 100644 --- a/scripts/spark2diffs/GpuJsonScan.diff +++ b/scripts/spark2diffs/GpuJsonScan.diff @@ -1,10 +1,6 @@ -3,16d2 -< def dateFormatInRead(fileOptions: Serializable): Option[String] = { -< fileOptions match { -< case jsonOpts: JSONOptions => Option(jsonOpts.dateFormat.getPattern) -< case _ => throw new RuntimeException("Wrong file options.") -< } -< } +3,12d2 +< // spark 2.x uses FastDateFormat, use getPattern +< def dateFormatInRead(options: JSONOptions): String = options.dateFormat.getPattern < < def timestampFormatInRead(fileOptions: Serializable): Option[String] = { < fileOptions match { @@ -13,7 +9,7 @@ < } < } < -41a28,37 +37a28,37 > def tagSupport(scanMeta: ScanMeta[JsonScan]) : Unit = { > val scan = scanMeta.wrapped > tagSupport( @@ -24,20 +20,20 @@ > scanMeta) > } > -47c43 +43c43 < meta: RapidsMeta[_, _]): Unit = { --- > meta: RapidsMeta[_, _, _]): Unit = { -109c105 -< dateFormatInRead(parsedOptions).foreach { dateFormat => +106c106 +< dateFormatInRead(parsedOptions), parseString = true) --- -> ShimLoader.getSparkShims.dateFormatInRead(parsedOptions).foreach { dateFormat => -117,118c113 +> GpuJsonUtils.dateFormatInRead(parsedOptions), parseString = true) +110,111c110 < // Spark 2.x doesn't have zoneId, so use timeZone and then to id < if (!TypeChecks.areTimestampsSupported(parsedOptions.timeZone.toZoneId)) { --- > if (!TypeChecks.areTimestampsSupported(parsedOptions.zoneId)) { -121c116 +114c113 < timestampFormatInRead(parsedOptions).foreach { tsFormat => --- -> ShimLoader.getSparkShims.timestampFormatInRead(parsedOptions).foreach { tsFormat => +> FileOptionsShims.timestampFormatInRead(parsedOptions).foreach { tsFormat => diff --git a/scripts/spark2diffs/GpuOverrides.diff b/scripts/spark2diffs/GpuOverrides.diff index da16231a4b2..13dac50ad0f 100644 --- a/scripts/spark2diffs/GpuOverrides.diff +++ b/scripts/spark2diffs/GpuOverrides.diff @@ -5,79 +5,78 @@ 24a25 > import ai.rapids.cudf.DType 26c27 -< import com.nvidia.spark.rapids.shims.v2._ +< import com.nvidia.spark.rapids.shims.{GpuBroadcastHashJoinMeta, GpuShuffledHashJoinMeta, GpuSortMergeJoinMeta, GpuSpecifiedWindowFrameMeta, GpuWindowExpressionMeta, GpuWindowSpecDefinitionMeta, OffsetWindowFunctionMeta} --- -> import com.nvidia.spark.rapids.shims.v2.{AQEUtils, GpuHashPartitioning, GpuSpecifiedWindowFrameMeta, GpuWindowExpressionMeta, OffsetWindowFunctionMeta} -31a33,35 -> import org.apache.spark.sql.catalyst.expressions.rapids.TimeStamp +> import com.nvidia.spark.rapids.shims.{AQEUtils, GpuHashPartitioning, GpuRangePartitioning, GpuSpecifiedWindowFrameMeta, GpuWindowExpressionMeta, OffsetWindowFunctionMeta, SparkShimImpl} +32a34,35 > import org.apache.spark.sql.catalyst.json.rapids.GpuJsonScan > import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero -33a38 +34a38 > import org.apache.spark.sql.catalyst.trees.TreeNodeTag -34a40 +35a40 > import org.apache.spark.sql.connector.read.Scan -36,37c42,43 +37,38c42,43 < import org.apache.spark.sql.execution.ScalarSubquery < import org.apache.spark.sql.execution.aggregate._ --- > import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, BroadcastQueryStageExec, ShuffleQueryStageExec} > import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} -43a50,52 +44a50,52 > import org.apache.spark.sql.execution.datasources.v2._ > import org.apache.spark.sql.execution.datasources.v2.csv.CSVScan > import org.apache.spark.sql.execution.datasources.v2.json.JsonScan -45a55 +46a55 > import org.apache.spark.sql.execution.python._ -49a60 +50a60 > import org.apache.spark.sql.rapids.catalyst.expressions.GpuRand -50a62,63 +51a62,63 > import org.apache.spark.sql.rapids.execution.python._ -> import org.apache.spark.sql.rapids.shims.v2.GpuTimeAdd -63c76 +> import org.apache.spark.sql.rapids.shims.GpuTimeAdd +64c76 < abstract class ReplacementRule[INPUT <: BASE, BASE, WRAP_TYPE <: RapidsMeta[INPUT, BASE]]( --- > abstract class ReplacementRule[INPUT <: BASE, BASE, WRAP_TYPE <: RapidsMeta[INPUT, BASE, _]]( -67c80 +68c80 < Option[RapidsMeta[_, _]], --- > Option[RapidsMeta[_, _, _]], -118c131 +119c131 < Option[RapidsMeta[_, _]], --- > Option[RapidsMeta[_, _, _]], -187c200 +188c200 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -202c215 +203c215 < Option[RapidsMeta[_, _]], --- > Option[RapidsMeta[_, _, _]], -217d229 +218d229 < /* -232c244 +233c244 < */ --- > -240c252 +241c252 < Option[RapidsMeta[_, _]], --- > Option[RapidsMeta[_, _, _]], -259c271 +260c271 < Option[RapidsMeta[_, _]], --- > Option[RapidsMeta[_, _, _]], -281c293 +282c293 < Option[RapidsMeta[_, _]], --- > Option[RapidsMeta[_, _, _]], -292d303 +293d303 < -296c307 +297c307 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -300,304c311 +301,305c311 < // spark 2.3 doesn't have this so just code it here < def sparkSessionActive: SparkSession = { < SparkSession.getActiveSession.getOrElse(SparkSession.getDefaultSession.getOrElse( @@ -85,25 +84,25 @@ < } --- > private var fileFormat: Option[ColumnarFileFormat] = None -311c318 +312c318 < val spark = sparkSessionActive --- > val spark = SparkSession.active -313c320 +314c320 < cmd.fileFormat match { --- > fileFormat = cmd.fileFormat match { -315a323 +316a323 > None -317a326 +318a326 > None -323a333 +324a333 > None -325a336 +326a336 > None -328d338 +329d338 < } -329a340,358 +330a340,358 > override def convertToGpu(): GpuDataWritingCommand = { > val format = fileFormat.getOrElse( > throw new IllegalStateException("fileFormat missing, tagSelfForGpu not called?")) @@ -123,11 +122,11 @@ > cmd.outputColumnNames) > } > } -334c363 +335c363 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -339,344c368 +340,345c368 < < // spark 2.3 doesn't have this so just code it here < def sparkSessionActive: SparkSession = { @@ -136,23 +135,23 @@ < } --- > private var gpuProvider: Option[ColumnarFileFormat] = None -354c378 +355c378 < val spark = sparkSessionActive --- > val spark = SparkSession.active -356c380 +357c380 < GpuDataSource.lookupDataSource(cmd.table.provider.get, spark.sessionState.conf) --- > GpuDataSource.lookupDataSourceWithFallback(cmd.table.provider.get, spark.sessionState.conf) -359c383 +360c383 < origProvider.getConstructor().newInstance() match { --- > gpuProvider = origProvider.getConstructor().newInstance() match { -362d385 +363d385 < None -366d388 +367d388 < None -372a395,406 +373a395,406 > override def convertToGpu(): GpuDataWritingCommand = { > val newProvider = gpuProvider.getOrElse( > throw new IllegalStateException("fileFormat unexpected, tagSelfForGpu not called?")) @@ -165,23 +164,25 @@ > origProvider, > newProvider) > } -374a409,411 +375a409,411 > /** > * Listener trait so that tests can confirm that the expected optimizations are being applied > */ -392d428 +393d428 < -405,407d440 +397d431 +< +410,412d443 < // copy here for 2.x < sealed abstract class Optimization < -409,413d441 +414,418d444 < // Spark 2.x - don't pull in cudf so hardcode here < val DECIMAL32_MAX_PRECISION = 9 < val DECIMAL64_MAX_PRECISION = 18 < val DECIMAL128_MAX_PRECISION = 38 < -471a500,562 +471a498,560 > private def convertPartToGpuIfPossible(part: Partitioning, conf: RapidsConf): Partitioning = { > part match { > case _: GpuPartitioning => part @@ -245,19 +246,19 @@ > } > } > -562c653 +562c651 < case dt: DecimalType if allowDecimal => dt.precision <= GpuOverrides.DECIMAL64_MAX_PRECISION --- > case dt: DecimalType if allowDecimal => dt.precision <= DType.DECIMAL64_MAX_PRECISION -585c676 +585c674 < def checkAndTagFloatAgg(dataType: DataType, conf: RapidsConf, meta: RapidsMeta[_,_]): Unit = { --- > def checkAndTagFloatAgg(dataType: DataType, conf: RapidsConf, meta: RapidsMeta[_,_,_]): Unit = { -599c690 +599c688 < meta: RapidsMeta[_,_]): Unit = { --- > meta: RapidsMeta[_,_,_]): Unit = { -609a701,732 +609a699,730 > /** > * Helper function specific to ANSI mode for the aggregate functions that should > * fallback, since we don't have the same overflow checks that Spark provides in @@ -290,11 +291,11 @@ > } > } > -613c736 +613c734 < doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _]], DataFromReplacementRule) --- > doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _, _]], DataFromReplacementRule) -620a744,753 +620a742,751 > def scan[INPUT <: Scan]( > desc: String, > doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _, _]], DataFromReplacementRule) @@ -305,58 +306,63 @@ > new ScanRule[INPUT](doWrap, desc, tag) > } > -624c757 +624c755 < doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _]], DataFromReplacementRule) --- > doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _, _]], DataFromReplacementRule) -642c775 +642c773 < p: Option[RapidsMeta[_, _]], --- > p: Option[RapidsMeta[_, _, _]], -651c784 +651c782 < doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _]], DataFromReplacementRule) --- > doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _, _]], DataFromReplacementRule) -661c794 +661c792 < doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _]], DataFromReplacementRule) --- > doWrap: (INPUT, RapidsConf, Option[RapidsMeta[_, _, _]], DataFromReplacementRule) -672c805 +672c803 < parent: Option[RapidsMeta[_, _]]): BaseExprMeta[INPUT] = --- > parent: Option[RapidsMeta[_, _, _]]): BaseExprMeta[INPUT] = -717a851 +709,710c840 +< // the only shim changin is Spark 3.3.X so leave off for now +< lazy val fileFormats = basicFormats // ++ SparkShimImpl.getFileFormats +--- +> lazy val fileFormats = basicFormats ++ SparkShimImpl.getFileFormats +726a857 > override def convertToGpu(child: Expression): GpuExpression = GpuSignum(child) -726a861,862 +735a867,868 > override def convertToGpu(child: Expression): GpuExpression = > GpuAlias(child, a.name)(a.exprId, a.qualifier, a.explicitMetadata) -736a873 +745a879 > override def convertToGpu(): Expression = att -750a888 +759a894 > override def convertToGpu(child: Expression): GpuExpression = GpuPromotePrecision(child) -768,769c906 +777,778c912 < // allowNegativeScaleOfDecimalEnabled is not in 2.x assume its default false < val t = if (s < 0 && !false) { --- > val t = if (s < 0 && !SQLConf.get.allowNegativeScaleOfDecimalEnabled) { -781,782c918 +790,791c924 < // Spark 2.X only has Cast, no AnsiCast so no CastBase, hardcode here to Cast < case p: PromotePrecision if p.child.isInstanceOf[Cast] && --- > case p: PromotePrecision if p.child.isInstanceOf[CastBase] && -784c920 +793c926 < val c = p.child.asInstanceOf[Cast] --- > val c = p.child.asInstanceOf[CastBase] -840c976 +849c982 < if (intermediatePrecision > GpuOverrides.DECIMAL128_MAX_PRECISION) { --- > if (intermediatePrecision > DType.DECIMAL128_MAX_PRECISION) { -855c991 +864c997 < if (intermediatePrecision > GpuOverrides.DECIMAL128_MAX_PRECISION) { --- > if (intermediatePrecision > DType.DECIMAL128_MAX_PRECISION) { -868a1005,1022 +877a1011,1028 > > override def convertToGpu(): GpuExpression = { > a.child match { @@ -375,59 +381,62 @@ > wrapped.dataType, wrapped.nullOnOverflow) > } > } -873a1028 +882a1034 > override def convertToGpu(child: Expression): GpuToDegrees = GpuToDegrees(child) -878a1034 +887a1040 > override def convertToGpu(child: Expression): GpuToRadians = GpuToRadians(child) -917a1074 +926a1080 > override def convertToGpu(): GpuExpression = GpuSpecialFrameBoundary(currentRow) -923a1081 +932a1087 > override def convertToGpu(): GpuExpression = GpuSpecialFrameBoundary(unboundedPreceding) -929a1088 +938a1094 > override def convertToGpu(): GpuExpression = GpuSpecialFrameBoundary(unboundedFollowing) -934a1094 +943a1100 > override def convertToGpu(): GpuExpression = GpuRowNumber -943a1104 +952a1110 > override def convertToGpu(): GpuExpression = GpuRank(childExprs.map(_.convertToGpu())) -952a1114 +961a1120 > override def convertToGpu(): GpuExpression = GpuDenseRank(childExprs.map(_.convertToGpu())) -972a1135,1136 +970a1130,1131 +> override def convertToGpu(): GpuExpression = +> GpuPercentRank(childExprs.map(_.convertToGpu())) +990a1152,1153 > override def convertToGpu(): GpuExpression = > GpuLead(input.convertToGpu(), offset.convertToGpu(), default.convertToGpu()) -992a1157,1158 +1010a1174,1175 > override def convertToGpu(): GpuExpression = > GpuLag(input.convertToGpu(), offset.convertToGpu(), default.convertToGpu()) -1002a1169,1170 +1020a1186,1187 > override def convertToGpu(child: Expression): GpuExpression = > GpuPreciseTimestampConversion(child, a.fromType, a.toType) -1011,1012c1179 +1029,1030c1196 < // val ansiEnabled = SQLConf.get.ansiEnabled < val ansiEnabled = false --- > val ansiEnabled = SQLConf.get.ansiEnabled -1015,1016d1181 +1033,1034d1198 < // Spark 2.x - ansi in not in 2.x < /* -1020,1021d1184 +1038,1039d1201 < < */ -1022a1186,1188 +1040a1203,1205 > > override def convertToGpu(child: Expression): GpuExpression = > GpuUnaryMinus(child, ansiEnabled) -1030a1197 +1048a1214 > override def convertToGpu(child: Expression): GpuExpression = GpuUnaryPositive(child) -1035a1203 +1053a1220 > override def convertToGpu(child: Expression): GpuExpression = GpuYear(child) -1040a1209 +1058a1226 > override def convertToGpu(child: Expression): GpuExpression = GpuMonth(child) -1045a1215 +1063a1232 > override def convertToGpu(child: Expression): GpuExpression = GpuQuarter(child) -1050a1221 +1068a1238 > override def convertToGpu(child: Expression): GpuExpression = GpuDayOfMonth(child) -1055a1227 +1073a1244 > override def convertToGpu(child: Expression): GpuExpression = GpuDayOfYear(child) -1060a1233,1244 +1078a1250,1261 > override def convertToGpu(child: Expression): GpuExpression = GpuAcos(child) > }), > expr[Acosh]( @@ -440,7 +449,7 @@ > } else { > GpuAcoshCompat(child) > } -1065a1250,1270 +1083a1267,1287 > override def convertToGpu(child: Expression): GpuExpression = GpuAsin(child) > }), > expr[Asinh]( @@ -462,71 +471,74 @@ > s"${RapidsConf.IMPROVED_FLOAT_OPS.key} is enabled") > } > } -1070a1276 +1088a1293 > override def convertToGpu(child: Expression): GpuExpression = GpuSqrt(child) -1075a1282 +1093a1299 > override def convertToGpu(child: Expression): GpuExpression = GpuCbrt(child) -1087c1294 +1102a1309,1310 +> override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = +> GpuHypot(lhs, rhs) +1114c1322 < if (precision > GpuOverrides.DECIMAL128_MAX_PRECISION) { --- > if (precision > DType.DECIMAL128_MAX_PRECISION) { -1094a1302 +1121a1330 > override def convertToGpu(child: Expression): GpuExpression = GpuFloor(child) -1106c1314 +1133c1342 < if (precision > GpuOverrides.DECIMAL128_MAX_PRECISION) { --- > if (precision > DType.DECIMAL128_MAX_PRECISION) { -1113a1322 +1140a1350 > override def convertToGpu(child: Expression): GpuExpression = GpuCeil(child) -1119a1329 +1146a1357 > override def convertToGpu(child: Expression): GpuExpression = GpuNot(child) -1127a1338 +1154a1366 > override def convertToGpu(child: Expression): GpuExpression = GpuIsNull(child) -1135a1347 +1162a1375 > override def convertToGpu(child: Expression): GpuExpression = GpuIsNotNull(child) -1141a1354 +1168a1382 > override def convertToGpu(child: Expression): GpuExpression = GpuIsNan(child) -1146a1360 +1173a1388 > override def convertToGpu(child: Expression): GpuExpression = GpuRint(child) -1152a1367 +1179a1395 > override def convertToGpu(child: Expression): GpuExpression = GpuBitwiseNot(child) -1161a1377,1378 +1188a1405,1406 > def convertToGpu(): GpuExpression = > GpuAtLeastNNonNulls(a.n, childExprs.map(_.convertToGpu())) -1170a1388,1389 +1197a1416,1417 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuDateAdd(lhs, rhs) -1179a1399,1400 +1206a1427,1428 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuDateSub(lhs, rhs) -1186a1408,1409 +1213a1436,1437 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuNaNvl(lhs, rhs) -1193a1417,1418 +1220a1445,1446 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuShiftLeft(lhs, rhs) -1200a1426,1427 +1227a1454,1455 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuShiftRight(lhs, rhs) -1207a1435,1436 +1234a1463,1464 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuShiftRightUnsigned(lhs, rhs) -1215a1445,1446 +1242a1473,1474 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuBitwiseAnd(lhs, rhs) -1223a1455,1456 +1250a1483,1484 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuBitwiseOr(lhs, rhs) -1231a1465,1466 +1258a1493,1494 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuBitwiseXor(lhs, rhs) -1241a1477 +1268a1505 > override def convertToGpu(): GpuExpression = GpuCoalesce(childExprs.map(_.convertToGpu())) -1250a1487 +1277a1515 > override def convertToGpu(): GpuExpression = GpuLeast(childExprs.map(_.convertToGpu())) -1259a1497 +1286a1525 > override def convertToGpu(): GpuExpression = GpuGreatest(childExprs.map(_.convertToGpu())) -1264a1503,1509 +1291a1531,1537 > override def convertToGpu(child: Expression): GpuExpression = GpuAtan(child) > }), > expr[Atanh]( @@ -534,43 +546,43 @@ > ExprChecks.mathUnaryWithAst, > (a, conf, p, r) => new UnaryAstExprMeta[Atanh](a, conf, p, r) { > override def convertToGpu(child: Expression): GpuExpression = GpuAtanh(child) -1269a1515 +1296a1543 > override def convertToGpu(child: Expression): GpuExpression = GpuCos(child) -1274a1521 +1301a1549 > override def convertToGpu(child: Expression): GpuExpression = GpuExp(child) -1279a1527 +1306a1555 > override def convertToGpu(child: Expression): GpuExpression = GpuExpm1(child) -1285a1534 +1312a1562 > override def convertToGpu(child: Expression): GpuExpression = GpuInitCap(child) -1290a1540 +1317a1568 > override def convertToGpu(child: Expression): GpuExpression = GpuLog(child) -1295a1546,1550 +1322a1574,1578 > override def convertToGpu(child: Expression): GpuExpression = { > // No need for overflow checking on the GpuAdd in Double as Double handles overflow > // the same in all modes. > GpuLog(GpuAdd(child, GpuLiteral(1d, DataTypes.DoubleType), false)) > } -1300a1556,1557 +1327a1584,1585 > override def convertToGpu(child: Expression): GpuExpression = > GpuLogarithm(child, GpuLiteral(2d, DataTypes.DoubleType)) -1305a1563,1564 +1332a1591,1592 > override def convertToGpu(child: Expression): GpuExpression = > GpuLogarithm(child, GpuLiteral(10d, DataTypes.DoubleType)) -1312a1572,1574 +1339a1600,1602 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > // the order of the parameters is transposed intentionally > GpuLogarithm(rhs, lhs) -1317a1580 +1344a1608 > override def convertToGpu(child: Expression): GpuExpression = GpuSin(child) -1322a1586 +1349a1614 > override def convertToGpu(child: Expression): GpuExpression = GpuSinh(child) -1327a1592 +1354a1620 > override def convertToGpu(child: Expression): GpuExpression = GpuCosh(child) -1332a1598 +1359a1626 > override def convertToGpu(child: Expression): GpuExpression = GpuCot(child) -1337a1604 +1364a1632 > override def convertToGpu(child: Expression): GpuExpression = GpuTanh(child) -1342a1610,1628 +1369a1638,1656 > override def convertToGpu(child: Expression): GpuExpression = GpuTan(child) > }), > expr[NormalizeNaNAndZero]( @@ -590,17 +602,17 @@ > (a, conf, p, r) => new UnaryExprMeta[KnownFloatingPointNormalized](a, conf, p, r) { > override def convertToGpu(child: Expression): GpuExpression = > GpuKnownFloatingPointNormalized(child) -1349a1636,1637 +1376a1664,1665 > override def convertToGpu(child: Expression): GpuExpression = > GpuKnownNotNull(child) -1356a1645,1647 +1383a1673,1675 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { > GpuDateDiff(lhs, rhs) > } -1375a1667,1668 +1402a1695,1696 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuTimeAdd(lhs, rhs) -1376a1670,1691 +1403a1698,1719 > expr[DateAddInterval]( > "Adds interval to date", > ExprChecks.binaryProject(TypeSig.DATE, TypeSig.DATE, @@ -623,15 +635,15 @@ > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuDateAddInterval(lhs, rhs) > }), -1386a1702,1703 +1413a1730,1731 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuDateFormatClass(lhs, rhs, strfFormat) -1399,1400c1716,1726 +1426,1427c1744,1754 < override def shouldFallbackOnAnsiTimestamp: Boolean = false -< // ShimLoader.getSparkShims.shouldFallbackOnAnsiTimestamp +< // SparkShimImpl.shouldFallbackOnAnsiTimestamp --- > override def shouldFallbackOnAnsiTimestamp: Boolean = -> ShimLoader.getSparkShims.shouldFallbackOnAnsiTimestamp +> SparkShimImpl.shouldFallbackOnAnsiTimestamp > > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { > if (conf.isImprovedTimestampOpsEnabled) { @@ -641,13 +653,13 @@ > GpuToUnixTimestamp(lhs, rhs, sparkFormat, strfFormat) > } > } -1412,1413c1738,1739 +1439,1440c1766,1767 < override def shouldFallbackOnAnsiTimestamp: Boolean = false -< // ShimLoader.getSparkShims.shouldFallbackOnAnsiTimestamp +< // SparkShimImpl.shouldFallbackOnAnsiTimestamp --- > override def shouldFallbackOnAnsiTimestamp: Boolean = -> ShimLoader.getSparkShims.shouldFallbackOnAnsiTimestamp -1414a1741,1748 +> SparkShimImpl.shouldFallbackOnAnsiTimestamp +1441a1769,1776 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { > if (conf.isImprovedTimestampOpsEnabled) { > // passing the already converted strf string for a little optimization @@ -656,58 +668,58 @@ > GpuUnixTimestamp(lhs, rhs, sparkFormat, strfFormat) > } > } -1424a1759 +1451a1787 > override def convertToGpu(expr: Expression): GpuExpression = GpuHour(expr) -1434a1770,1771 +1461a1798,1799 > override def convertToGpu(expr: Expression): GpuExpression = > GpuMinute(expr) -1443a1781,1783 +1470a1809,1811 > > override def convertToGpu(expr: Expression): GpuExpression = > GpuSecond(expr) -1449a1790,1791 +1476a1818,1819 > override def convertToGpu(expr: Expression): GpuExpression = > GpuWeekDay(expr) -1455a1798,1799 +1482a1826,1827 > override def convertToGpu(expr: Expression): GpuExpression = > GpuDayOfWeek(expr) -1460a1805,1806 +1487a1833,1834 > override def convertToGpu(expr: Expression): GpuExpression = > GpuLastDay(expr) -1471a1818,1820 +1498a1846,1848 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > // passing the already converted strf string for a little optimization > GpuFromUnixTime(lhs, rhs, strfFormat) -1478a1828,1829 +1505a1856,1857 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuPmod(lhs, rhs) -1490c1841 +1517c1869 < private val ansiEnabled = false --- > private val ansiEnabled = SQLConf.get.ansiEnabled -1492a1844,1846 +1519a1872,1874 > if (ansiEnabled && GpuAnsi.needBasicOpOverflowCheck(a.dataType)) { > willNotWorkInAst("AST Addition does not support ANSI mode.") > } -1494a1849,1850 +1521a1877,1878 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuAdd(lhs, rhs, failOnError = ansiEnabled) -1506c1862 +1533c1890 < private val ansiEnabled = false --- > private val ansiEnabled = SQLConf.get.ansiEnabled -1508a1865,1867 +1535a1893,1895 > if (ansiEnabled && GpuAnsi.needBasicOpOverflowCheck(a.dataType)) { > willNotWorkInAst("AST Subtraction does not support ANSI mode.") > } -1510a1870,1871 +1537a1898,1899 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuSubtract(lhs, rhs, ansiEnabled) -1523a1885,1887 +1550a1913,1915 > if (SQLConf.get.ansiEnabled && GpuAnsi.needBasicOpOverflowCheck(a.dataType)) { > willNotWorkOnGpu("GPU Multiplication does not support ANSI mode") > } -1525a1890,1897 +1552a1918,1925 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { > a.dataType match { > case _: DecimalType => throw new IllegalStateException( @@ -716,37 +728,37 @@ > GpuMultiply(lhs, rhs) > } > } -1532a1905,1906 +1559a1933,1934 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuAnd(lhs, rhs) -1539a1914,1915 +1566a1942,1943 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuOr(lhs, rhs) -1549a1926,1927 +1576a1954,1955 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuEqualNullSafe(lhs, rhs) -1560a1939,1940 +1587a1967,1968 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuEqualTo(lhs, rhs) -1571a1952,1953 +1598a1980,1981 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuGreaterThan(lhs, rhs) -1582a1965,1966 +1609a1993,1994 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuGreaterThanOrEqual(lhs, rhs) -1602a1987,1988 +1629a2015,2016 > override def convertToGpu(): GpuExpression = > GpuInSet(childExprs.head.convertToGpu(), in.list.asInstanceOf[Seq[Literal]].map(_.value)) -1613a2000,2001 +1640a2028,2029 > override def convertToGpu(): GpuExpression = > GpuInSet(childExprs.head.convertToGpu(), in.hset.toSeq) -1624a2013,2014 +1651a2041,2042 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuLessThan(lhs, rhs) -1635a2026,2027 +1662a2054,2055 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuLessThanOrEqual(lhs, rhs) -1640a2033,2044 +1667a2061,2072 > override def convertToGpu(): GpuExpression = { > val branches = childExprs.grouped(2).flatMap { > case Seq(cond, value) => Some((cond.convertToGpu(), value.convertToGpu())) @@ -759,15 +771,15 @@ > } > GpuCaseWhen(branches, elseValue) > } -1657a2062,2065 +1684a2090,2093 > override def convertToGpu(): GpuExpression = { > val Seq(boolExpr, trueExpr, falseExpr) = childExprs.map(_.convertToGpu()) > GpuIf(boolExpr, trueExpr, falseExpr) > } -1665a2074,2075 +1692a2102,2103 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuPow(lhs, rhs) -1679a2090,2109 +1706a2118,2137 > // Division of Decimal types is a little odd. To work around some issues with > // what Spark does the tagging/checks are in CheckOverflow instead of here. > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = @@ -788,15 +800,15 @@ > (a, conf, p, r) => new BinaryExprMeta[IntegralDivide](a, conf, p, r) { > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuIntegralDivide(lhs, rhs) -1687a2118,2119 +1714a2146,2147 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuRemainder(lhs, rhs) -1702c2134,2135 +1729c2162,2163 < // No filter parameter in 2.x --- > private val filter: Option[BaseExprMeta[_]] = > a.filter.map(GpuOverrides.wrapExpr(_, conf, Some(this))) -1706c2139,2154 +1733c2167,2182 < childrenExprMeta --- > childrenExprMeta ++ filter.toSeq @@ -815,11 +827,11 @@ > GpuAggregateExpression(childExprs.head.convertToGpu().asInstanceOf[GpuAggregateFunction], > a.mode, a.isDistinct, filter.map(_.convertToGpu()), resultId) > } -1729a2178,2180 +1756a2206,2208 > // One of the few expressions that are not replaced with a GPU version > override def convertToGpu(): Expression = > sortOrder.withNewChildren(childExprs.map(_.convertToGpu())) -1754a2206,2212 +1781a2234,2240 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = { > val Seq(pivotColumn, valueColumn) = childExprs > GpuPivotFirst(pivotColumn, valueColumn, pivot.pivotColumnValues) @@ -827,93 +839,93 @@ > > // Pivot does not overflow, so it doesn't need the ANSI check > override val needsAnsiCheck: Boolean = false -1769a2228,2229 +1796a2256,2257 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuCount(childExprs) -1798a2259,2264 +1825a2287,2292 > > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuMax(childExprs.head) > > // Max does not overflow, so it doesn't need the ANSI check > override val needsAnsiCheck: Boolean = false -1827a2294,2299 +1854a2322,2327 > > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuMin(childExprs.head) > > // Min does not overflow, so it doesn't need the ANSI check > override val needsAnsiCheck: Boolean = false -1840a2313,2314 +1867a2341,2342 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuSum(childExprs.head, a.dataType) -1854a2329,2333 +1881a2357,2361 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuFirst(childExprs.head, a.ignoreNulls) > > // First does not overflow, so it doesn't need the ANSI check > override val needsAnsiCheck: Boolean = false -1868a2348,2352 +1895a2376,2380 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuLast(childExprs.head, a.ignoreNulls) > > // Last does not overflow, so it doesn't need the ANSI check > override val needsAnsiCheck: Boolean = false -1887a2372,2373 +1914a2400,2401 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuBRound(lhs, rhs) -1906a2393,2394 +1933a2421,2422 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuRound(lhs, rhs) -1928a2417,2421 +1955a2445,2449 > > override def convertToGpu(): GpuExpression = > GpuPythonUDF(a.name, a.func, a.dataType, > childExprs.map(_.convertToGpu()), > a.evalType, a.udfDeterministic, a.resultId) -1936a2430 +1963a2458 > override def convertToGpu(child: Expression): GpuExpression = GpuRand(child) -1941a2436 +1968a2464 > override def convertToGpu(): GpuExpression = GpuSparkPartitionID() -1946a2442 +1973a2470 > override def convertToGpu(): GpuExpression = GpuMonotonicallyIncreasingID() -1951a2448 +1978a2476 > override def convertToGpu(): GpuExpression = GpuInputFileName() -1956a2454 +1983a2482 > override def convertToGpu(): GpuExpression = GpuInputFileBlockStart() -1961a2460 +1988a2488 > override def convertToGpu(): GpuExpression = GpuInputFileBlockLength() -1967a2467 +1994a2495 > override def convertToGpu(child: Expression): GpuExpression = GpuMd5(child) -1972a2473 +1999a2501 > override def convertToGpu(child: Expression): GpuExpression = GpuUpper(child) -1978a2480 +2005a2508 > override def convertToGpu(child: Expression): GpuExpression = GpuLower(child) -1995a2498,2502 +2022a2526,2530 > override def convertToGpu( > str: Expression, > width: Expression, > pad: Expression): GpuExpression = > GpuStringLPad(str, width, pad) -2011a2519,2523 +2038a2547,2551 > override def convertToGpu( > str: Expression, > width: Expression, > pad: Expression): GpuExpression = > GpuStringRPad(str, width, pad) -2032a2545,2546 +2059a2573,2574 > override def convertToGpu(arr: Expression): GpuExpression = > GpuGetStructField(arr, expr.ordinal, expr.name) -2090a2605,2609 +2125a2641,2645 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { > // This will be called under 3.0.x version, so set failOnError to false to match CPU > // behavior > GpuElementAt(lhs, rhs, failOnError = false) > } -2101a2621,2622 +2136a2657,2658 > override def convertToGpu(child: Expression): GpuExpression = > GpuMapKeys(child) -2112a2634,2649 +2147a2670,2685 > override def convertToGpu(child: Expression): GpuExpression = > GpuMapValues(child) > }), @@ -930,40 +942,40 @@ > (in, conf, p, r) => new UnaryExprMeta[MapEntries](in, conf, p, r) { > override def convertToGpu(child: Expression): GpuExpression = > GpuMapEntries(child) -2120,2121c2657,2658 +2162,2163c2700,2701 < .withPsNote(TypeEnum.DOUBLE, GpuOverrides.nanAggPsNote) < .withPsNote(TypeEnum.FLOAT, GpuOverrides.nanAggPsNote), --- > .withPsNote(TypeEnum.DOUBLE, nanAggPsNote) > .withPsNote(TypeEnum.FLOAT, nanAggPsNote), -2125c2662 +2167c2705 < GpuOverrides.checkAndTagFloatNanAgg("Min", in.dataType, conf, this) --- > checkAndTagFloatNanAgg("Min", in.dataType, conf, this) -2126a2664,2666 +2168a2707,2709 > > override def convertToGpu(child: Expression): GpuExpression = > GpuArrayMin(child) -2134,2135c2674,2675 +2176,2177c2717,2718 < .withPsNote(TypeEnum.DOUBLE, GpuOverrides.nanAggPsNote) < .withPsNote(TypeEnum.FLOAT, GpuOverrides.nanAggPsNote), --- > .withPsNote(TypeEnum.DOUBLE, nanAggPsNote) > .withPsNote(TypeEnum.FLOAT, nanAggPsNote), -2139c2679 +2181c2722 < GpuOverrides.checkAndTagFloatNanAgg("Max", in.dataType, conf, this) --- > checkAndTagFloatNanAgg("Max", in.dataType, conf, this) -2141a2682,2683 +2183a2725,2726 > override def convertToGpu(child: Expression): GpuExpression = > GpuArrayMax(child) -2146a2689,2690 +2188a2732,2733 > override def convertToGpu(): GpuExpression = > GpuCreateNamedStruct(childExprs.map(_.convertToGpu())) -2180a2725,2726 +2222a2768,2769 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuArrayContains(lhs, rhs) -2192c2738,2742 +2235c2782,2786 < }), --- > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { @@ -971,10 +983,10 @@ > } > } > ), -2217a2768,2769 +2260a2812,2813 > override def convertToGpu(): GpuExpression = > GpuCreateArray(childExprs.map(_.convertToGpu()), wrapped.useStringTypeWhenEmpty) -2233a2786,2792 +2276a2830,2836 > override def convertToGpu(): GpuExpression = { > val func = childExprs.head > val args = childExprs.tail @@ -982,15 +994,24 @@ > args.map(_.convertToGpu().asInstanceOf[NamedExpression]), > in.hidden) > } -2241a2801,2803 +2284a2845,2847 > override def convertToGpu(): GpuExpression = { > GpuNamedLambdaVariable(in.name, in.dataType, in.nullable, in.exprId) > } -2258a2821,2869 +2301a2865,2867 > override def convertToGpu(): GpuExpression = { > GpuArrayTransform(childExprs.head.convertToGpu(), childExprs(1).convertToGpu()) > } +2312a2879,2932 +> override def convertToGpu(): GpuExpression = { +> GpuArrayExists( +> childExprs.head.convertToGpu(), +> childExprs(1).convertToGpu(), +> SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC) +> ) +> } > }), +> > expr[TransformKeys]( > "Transform keys in a map using a transform function", > ExprChecks.projectOnly(TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + @@ -1036,79 +1057,80 @@ > override def convertToGpu(): GpuExpression = { > GpuTransformValues(childExprs.head.convertToGpu(), childExprs(1).convertToGpu()) > } -2266a2878,2882 +2320a2941,2945 > override def convertToGpu( > val0: Expression, > val1: Expression, > val2: Expression): GpuExpression = > GpuStringLocate(val0, val1, val2) -2274a2891,2895 +2328a2954,2958 > override def convertToGpu( > column: Expression, > position: Expression, > length: Expression): GpuExpression = > GpuSubstring(column, position, length) -2290a2912,2914 +2344a2975,2977 > override def convertToGpu( > input: Expression, > repeatTimes: Expression): GpuExpression = GpuStringRepeat(input, repeatTimes) -2298a2923,2927 +2352a2986,2990 > override def convertToGpu( > column: Expression, > target: Expression, > replace: Expression): GpuExpression = > GpuStringReplace(column, target, replace) -2306a2936,2939 +2360a2999,3002 > override def convertToGpu( > column: Expression, > target: Option[Expression] = None): GpuExpression = > GpuStringTrim(column, target) -2315a2949,2952 +2369a3012,3015 > override def convertToGpu( > column: Expression, > target: Option[Expression] = None): GpuExpression = > GpuStringTrimLeft(column, target) -2324a2962,2965 +2378a3025,3028 > override def convertToGpu( > column: Expression, > target: Option[Expression] = None): GpuExpression = > GpuStringTrimRight(column, target) -2331a2973,2974 +2385a3036,3037 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuStartsWith(lhs, rhs) -2338a2982,2983 +2392a3045,3046 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuEndsWith(lhs, rhs) -2349a2995 +2403a3058 > override def convertToGpu(child: Seq[Expression]): GpuExpression = GpuConcat(child) -2366a3013,3014 +2420a3076,3077 > override final def convertToGpu(): GpuExpression = > GpuConcatWs(childExprs.map(_.convertToGpu())) -2376a3025,3026 +2430a3088,3089 > def convertToGpu(): GpuExpression = > GpuMurmur3Hash(childExprs.map(_.convertToGpu()), a.seed) -2383a3034,3035 +2437a3097,3098 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuContains(lhs, rhs) -2390a3043,3044 +2444a3106,3107 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuLike(lhs, rhs, a.escapeChar) -2415a3070 +2464a3128 > override def convertToGpu(child: Expression): GpuExpression = GpuLength(child) -2423a3079,3080 +2472a3137,3138 > override def convertToGpu(child: Expression): GpuExpression = > GpuSize(child, a.legacySizeOfNull) -2430a3088 +2479a3146 > override def convertToGpu(child: Expression): GpuExpression = GpuUnscaledValue(child) -2436a3095,3096 +2485a3153,3154 > override def convertToGpu(child: Expression): GpuExpression = > GpuMakeDecimal(child, a.precision, a.scale, a.nullOnOverflow) -2450a3111 +2499a3169 > override def convertToGpu(): GpuExpression = GpuExplode(childExprs.head.convertToGpu()) -2464a3126 +2513a3184 > override def convertToGpu(): GpuExpression = GpuPosExplode(childExprs.head.convertToGpu()) -2477c3139,3205 +2526,2527c3197,3263 < }), +< // spark 2.x CollectList and CollectSet use TypedImperative which isn't in 2.x --- > override def convertToGpu(childExpr: Seq[Expression]): GpuExpression = > GpuReplicateRows(childExpr) @@ -1147,8 +1169,8 @@ > expr[CollectSet]( > "Collect a set of unique elements, not supported in reduction", > // GpuCollectSet is not yet supported in Reduction context. -> // Compared to CollectList, StructType is NOT in GpuCollectSet because underlying -> // method drop_list_duplicates doesn't support nested types. +> // Compared to CollectList, ArrayType and MapType are NOT supported in GpuCollectSet +> // because underlying cuDF operator drop_list_duplicates doesn't support LIST type. > ExprChecks.aggNotReduction( > TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + > TypeSig.NULL + TypeSig.STRUCT), @@ -1177,60 +1199,60 @@ > // Last does not overflow, so it doesn't need the ANSI check > override val needsAnsiCheck: Boolean = false > }), -2483a3212,3215 +2533a3270,3273 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = { -> val legacyStatisticalAggregate = ShimLoader.getSparkShims.getLegacyStatisticalAggregate +> val legacyStatisticalAggregate = SparkShimImpl.getLegacyStatisticalAggregate > GpuStddevPop(childExprs.head, !legacyStatisticalAggregate) > } -2491a3224,3227 +2541a3282,3285 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = { -> val legacyStatisticalAggregate = ShimLoader.getSparkShims.getLegacyStatisticalAggregate +> val legacyStatisticalAggregate = SparkShimImpl.getLegacyStatisticalAggregate > GpuStddevSamp(childExprs.head, !legacyStatisticalAggregate) > } -2498a3235,3238 +2548a3293,3296 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = { -> val legacyStatisticalAggregate = ShimLoader.getSparkShims.getLegacyStatisticalAggregate +> val legacyStatisticalAggregate = SparkShimImpl.getLegacyStatisticalAggregate > GpuVariancePop(childExprs.head, !legacyStatisticalAggregate) > } -2505a3246,3249 +2555a3304,3307 > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = { -> val legacyStatisticalAggregate = ShimLoader.getSparkShims.getLegacyStatisticalAggregate +> val legacyStatisticalAggregate = SparkShimImpl.getLegacyStatisticalAggregate > GpuVarianceSamp(childExprs.head, !legacyStatisticalAggregate) > } -2545a3290,3295 +2595a3348,3353 > > override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = > GpuApproximatePercentile(childExprs.head, > childExprs(1).asInstanceOf[GpuLiteral], > childExprs(2).asInstanceOf[GpuLiteral]) > -2560a3311,3312 +2610a3369,3370 > override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = > GpuGetJsonObject(lhs, rhs) -2563c3315 -< expr[ScalarSubquery]( ---- -> expr[org.apache.spark.sql.execution.ScalarSubquery]( -2569,2570c3321,3325 -< (a, conf, p, r) => new ExprMeta[ScalarSubquery](a, conf, p, r) { +2622c3382,3384 < }), --- -> (a, conf, p, r) => -> new ExprMeta[org.apache.spark.sql.execution.ScalarSubquery](a, conf, p, r) { > override def convertToGpu(): GpuExpression = GpuScalarSubquery(a.plan, a.exprId) > } > ), -2575c3330,3332 +2627c3389,3391 < }), --- > override def convertToGpu(): GpuExpression = GpuCreateMap(childExprs.map(_.convertToGpu())) > } > ), -2592c3349,3397 -< commonExpressions ++ GpuHiveOverrides.exprs ++ ShimGpuOverrides.shimExpressions ---- -> commonExpressions ++ TimeStamp.getExprs ++ GpuHiveOverrides.exprs ++ -> ShimLoader.getSparkShims.getExprs +2645a3410 +> override def convertToGpu(child: Expression): GpuExpression = GpuBitLength(child) +2652a3418 +> override def convertToGpu(child: Expression): GpuExpression = GpuOctetLength(child) +2666c3432 +< ) +--- +> ) +2672c3438,3485 +< ShimGpuOverrides.shimExpressions +--- +> SparkShimImpl.getExprs > > def wrapScan[INPUT <: Scan]( > scan: INPUT, @@ -1277,15 +1299,15 @@ > })).map(r => (r.getClassFor.asSubclass(classOf[Scan]), r)).toMap > > val scans: Map[Class[_ <: Scan], ScanRule[_ <: Scan]] = -> commonScans ++ ShimLoader.getSparkShims.getScans -2597c3402 +> commonScans ++ SparkShimImpl.getScans ++ ExternalSource.getScans +2677c3490 < parent: Option[RapidsMeta[_, _]]): PartMeta[INPUT] = --- > parent: Option[RapidsMeta[_, _, _]]): PartMeta[INPUT] = -2612a3418,3419 +2692a3506,3507 > override def convertToGpu(): GpuPartitioning = > GpuHashPartitioning(childExprs.map(_.convertToGpu()), hp.numPartitions) -2622a3430,3437 +2702a3518,3525 > override def convertToGpu(): GpuPartitioning = { > if (rp.numPartitions > 1) { > val gpuOrdering = childExprs.map(_.convertToGpu()).asInstanceOf[Seq[SortOrder]] @@ -1294,21 +1316,21 @@ > GpuSinglePartitioning > } > } -2627a3443,3445 +2707a3531,3533 > override def convertToGpu(): GpuPartitioning = { > GpuRoundRobinPartitioning(rrp.numPartitions) > } -2632a3451 +2712a3539 > override def convertToGpu(): GpuPartitioning = GpuSinglePartitioning -2639c3458 +2719c3546 < parent: Option[RapidsMeta[_, _]]): DataWritingCommandMeta[INPUT] = --- > parent: Option[RapidsMeta[_, _, _]]): DataWritingCommandMeta[INPUT] = -2657c3476 +2737c3564 < parent: Option[RapidsMeta[_, _]]): SparkPlanMeta[INPUT] = --- > parent: Option[RapidsMeta[_, _, _]]): SparkPlanMeta[INPUT] = -2680c3499,3517 +2760c3587,3605 < (range, conf, p, r) => new SparkPlanMeta[RangeExec](range, conf, p, r) { --- > (range, conf, p, r) => { @@ -1330,14 +1352,14 @@ > > override def convertToGpu(): GpuExec = > GpuBatchScanExec(p.output, childScans.head.convertToGpu()) -2687a3525,3526 +2767a3613,3614 > override def convertToGpu(): GpuExec = > GpuCoalesceExec(coalesce.numPartitions, childPlans.head.convertIfNeeded()) -2700a3540,3542 +2780a3628,3630 > override def convertToGpu(): GpuExec = > GpuDataWritingCommandExec(childDataWriteCmds.head.convertToGpu(), > childPlans.head.convertIfNeeded()) -2715a3558,3580 +2795a3646,3668 > override def convertToGpu(): GpuExec = { > // To avoid metrics confusion we split a single stage up into multiple parts but only > // if there are multiple partitions to make it worth doing. @@ -1351,7 +1373,7 @@ > takeExec.limit, > so, > projectList.map(_.convertToGpu().asInstanceOf[NamedExpression]), -> ShimLoader.getSparkShims.getGpuShuffleExchangeExec( +> SparkShimImpl.getGpuShuffleExchangeExec( > GpuSinglePartitioning, > GpuTopN( > takeExec.limit, @@ -1361,26 +1383,26 @@ > SinglePartition))(takeExec.sortOrder) > } > } -2723a3589,3590 +2803a3677,3678 > override def convertToGpu(): GpuExec = > GpuLocalLimitExec(localLimitExec.limit, childPlans.head.convertIfNeeded()) -2731a3599,3600 +2811a3687,3688 > override def convertToGpu(): GpuExec = > GpuGlobalLimitExec(globalLimitExec.limit, childPlans.head.convertIfNeeded()) -2738,2741c3607 +2818,2821c3695 < (collectLimitExec, conf, p, r) => < new SparkPlanMeta[CollectLimitExec](collectLimitExec, conf, p, r) { < override val childParts: scala.Seq[PartMeta[_]] = < Seq(GpuOverrides.wrapPart(collectLimitExec.outputPartitioning, conf, Some(this)))}) --- > (collectLimitExec, conf, p, r) => new GpuCollectLimitMeta(collectLimitExec, conf, p, r)) -2749a3616,3617 +2829a3704,3705 > override def convertToGpu(): GpuExec = > GpuFilterExec(childExprs.head.convertToGpu(), childPlans.head.convertIfNeeded()) -2770a3639,3640 +2850a3727,3728 > override def convertToGpu(): GpuExec = > GpuUnionExec(childPlans.map(_.convertIfNeeded())) -2801a3672,3682 +2881a3760,3770 > override def convertToGpu(): GpuExec = { > val Seq(left, right) = childPlans.map(_.convertIfNeeded()) > val joinExec = GpuCartesianProductExec( @@ -1392,7 +1414,7 @@ > // as a filter after the join when possible. > condition.map(c => GpuFilterExec(c.convertToGpu(), joinExec)).getOrElse(joinExec) > } -2814a3696,3711 +2894a3784,3799 > exec[ObjectHashAggregateExec]( > "The backend for hash based aggregations supporting TypedImperativeAggregate functions", > ExecChecks( @@ -1409,38 +1431,38 @@ > "not allowed for grouping expressions if containing Array or Map as child"), > TypeSig.all), > (agg, conf, p, r) => new GpuObjectHashAggregateExecMeta(agg, conf, p, r)), -2821,2822d3717 +2901,2902d3805 < // SPARK 2.x we can't check for the TypedImperativeAggregate properly so < // map/arrya/struct left off -2825c3720 +2905c3808 < TypeSig.MAP + TypeSig.BINARY) --- > TypeSig.MAP + TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.BINARY) -2829c3724,3727 +2909c3812,3815 < .withPsNote(TypeEnum.MAP, "not allowed for grouping expressions"), --- > .withPsNote(TypeEnum.ARRAY, "not allowed for grouping expressions") > .withPsNote(TypeEnum.MAP, "not allowed for grouping expressions") > .withPsNote(TypeEnum.STRUCT, > "not allowed for grouping expressions if containing Array or Map as child"), -2832,2833d3729 +2912,2913d3817 < // SPARK 2.x we can't check for the TypedImperativeAggregate properly so don't say we do the < // ObjectHashAggregate -2866c3762 +2949c3853,3858 < (sample, conf, p, r) => new GpuSampleExecMeta(sample, conf, p, r) {} --- > (sample, conf, p, r) => new GpuSampleExecMeta(sample, conf, p, r) -2868,2870c3764,3797 -< // ShimLoader.getSparkShims.aqeShuffleReaderExec, -< // ShimLoader.getSparkShims.neverReplaceShowCurrentNamespaceCommand, -< neverReplaceExec[ExecutedCommandExec]("Table metadata operation") ---- +> ), > exec[SubqueryBroadcastExec]( > "Plan to collect and transform the broadcast key values", > ExecChecks(TypeSig.all, TypeSig.all), > (s, conf, p, r) => new GpuSubqueryBroadcastMeta(s, conf, p, r) -> ), -> ShimLoader.getSparkShims.aqeShuffleReaderExec, +2951,2953c3860,3888 +< // SparkShimImpl.aqeShuffleReaderExec, +< // SparkShimImpl.neverReplaceShowCurrentNamespaceCommand, +< neverReplaceExec[ExecutedCommandExec]("Table metadata operation") +--- +> SparkShimImpl.aqeShuffleReaderExec, > exec[FlatMapCoGroupsInPandasExec]( > "The backend for CoGrouped Aggregation Pandas UDF, it runs on CPU itself now but supports" + > " scheduling GPU resources for the Python process when enabled", @@ -1452,7 +1474,7 @@ > neverReplaceExec[DescribeNamespaceExec]("Namespace metadata operation"), > neverReplaceExec[DropNamespaceExec]("Namespace metadata operation"), > neverReplaceExec[SetCatalogAndNamespaceExec]("Namespace metadata operation"), -> ShimLoader.getSparkShims.neverReplaceShowCurrentNamespaceCommand, +> SparkShimImpl.neverReplaceShowCurrentNamespaceCommand, > neverReplaceExec[ShowNamespacesExec]("Namespace metadata operation"), > neverReplaceExec[ExecutedCommandExec]("Table metadata operation"), > neverReplaceExec[AlterTableExec]("Table metadata operation"), @@ -1469,23 +1491,23 @@ > neverReplaceExec[AdaptiveSparkPlanExec]("Wrapper for adaptive query plan"), > neverReplaceExec[BroadcastQueryStageExec]("Broadcast query stage"), > neverReplaceExec[ShuffleQueryStageExec]("Shuffle query stage") -2874c3801 +2957c3892 < commonExecs ++ ShimGpuOverrides.shimExecs --- -> commonExecs ++ ShimLoader.getSparkShims.getExecs -2877,2879c3804 +> commonExecs ++ SparkShimImpl.getExecs +2960,2962c3895 < // val key = SQLConf.LEGACY_TIME_PARSER_POLICY.key < val key = "2xgone" < val policy = SQLConf.get.getConfString(key, "EXCEPTION") --- > val policy = SQLConf.get.getConfString(SQLConf.LEGACY_TIME_PARSER_POLICY.key, "EXCEPTION") -2886a3812,3816 +2969a3903,3907 > val preRowToColProjection = TreeNodeTag[Seq[NamedExpression]]("rapids.gpu.preRowToColProcessing") > > val postColToRowProjection = TreeNodeTag[Seq[NamedExpression]]( > "rapids.gpu.postColToRowProcessing") > -2892a3823,3830 +2975a3914,3921 > private def doConvertPlan(wrap: SparkPlanMeta[SparkPlan], conf: RapidsConf, > optimizations: Seq[Optimization]): SparkPlan = { > val convertedPlan = wrap.convertIfNeeded() @@ -1494,7 +1516,7 @@ > sparkPlan > } > -2895c3833,3881 +2978c3924,3972 < Seq.empty --- > if (conf.optimizerEnabled) { @@ -1546,7 +1568,7 @@ > } > } > operator.withNewChildren(children) -2905,2906c3891,3897 +2988,2989c3982,3988 < // Only run the explain and don't actually convert or run on GPU. < def explainPotentialGpuPlan(df: DataFrame, explain: String = "ALL"): String = { --- @@ -1557,7 +1579,7 @@ > * to make it close to when the columnar rules would normally run on the plan. > */ > def explainPotentialGpuPlan(df: DataFrame, explain: String): String = { -2932a3924,3940 +3015a4015,4031 > /** > * Use explain mode on an active SQL plan as its processed through catalyst. > * This path is the same as being run through the plugin running on hosts with @@ -1575,24 +1597,28 @@ > } > } > -2955c3963 +3038c4054 < // case c2r: ColumnarToRowExec => prepareExplainOnly(c2r.child) --- > case c2r: ColumnarToRowExec => prepareExplainOnly(c2r.child) -2957,2958c3965,3966 +3040,3041c4056,4057 < // case aqe: AdaptiveSparkPlanExec => -< // prepareExplainOnly(ShimLoader.getSparkShims.getAdaptiveInputPlan(aqe)) +< // prepareExplainOnly(SparkShimImpl.getAdaptiveInputPlan(aqe)) --- > case aqe: AdaptiveSparkPlanExec => -> prepareExplainOnly(ShimLoader.getSparkShims.getAdaptiveInputPlan(aqe)) -2965,2969c3973,4028 +> prepareExplainOnly(SparkShimImpl.getAdaptiveInputPlan(aqe)) +3048,3052c4064,4090 < // Spark 2.x < object GpuUserDefinedFunction { < // UDFs can support all types except UDT which does not have a clear columnar representation. < val udfTypeSig: TypeSig = (TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL + < TypeSig.BINARY + TypeSig.CALENDAR + TypeSig.ARRAY + TypeSig.MAP + TypeSig.STRUCT).nested() --- -> class ExplainPlanImpl extends ExplainPlanBase { +> /** +> * Note, this class should not be referenced directly in source code. +> * It should be loaded by reflection using ShimLoader.newInstanceOf, see ./docs/dev/shims.md +> */ +> protected class ExplainPlanImpl extends ExplainPlanBase { > override def explainPotentialGpuPlan(df: DataFrame, explain: String): String = { > GpuOverrides.explainPotentialGpuPlan(df, explain) > } @@ -1615,7 +1641,7 @@ > } > } > } -> +3053a4092,4163 > /** Tag the initial plan when AQE is enabled */ > case class GpuQueryStagePrepOverrides() extends Rule[SparkPlan] with Logging { > override def apply(sparkPlan: SparkPlan): SparkPlan = GpuOverrideUtil.tryOverride { plan => @@ -1648,7 +1674,7 @@ > plan > } > }(sparkPlan) -2970a4030,4068 +> > private def updateForAdaptivePlan(plan: SparkPlan, conf: RapidsConf): SparkPlan = { > if (plan.conf.adaptiveExecutionEnabled) { > // AQE can cause Spark to inject undesired CPU shuffles into the plan because GPU and CPU diff --git a/scripts/spark2diffs/GpuParquetFileFormat.diff b/scripts/spark2diffs/GpuParquetFileFormat.diff index 980d6308a00..34b1aee76d7 100644 --- a/scripts/spark2diffs/GpuParquetFileFormat.diff +++ b/scripts/spark2diffs/GpuParquetFileFormat.diff @@ -6,7 +6,11 @@ < schema: StructType): Unit = { --- > schema: StructType): Option[GpuParquetFileFormat] = { -52,56c52 +8a9,11 +> +> ParquetFieldIdShims.tagGpuSupportWriteForFieldId(meta, schema, sqlConf) +> +52,56c55 < // Spark 2.x doesn't have the rebase mode because the changes of calendar type weren't made < // so just skip the checks, since this is just explain only it would depend on how < // they set when they get to 3.x. The default in 3.x is EXCEPTION which would be good @@ -14,7 +18,7 @@ < /* --- > -78c74,79 +78c77,82 < */ --- > @@ -23,12 +27,12 @@ > } else { > None > } -81,82c82 +81,82c85 < // SPARK 2.X - just return String rather then CompressionType < def parseCompressionType(compressionType: String): Option[String] = { --- > def parseCompressionType(compressionType: String): Option[CompressionType] = { -84,85c84,85 +84,85c87,88 < case "NONE" | "UNCOMPRESSED" => Some("NONE") < case "SNAPPY" => Some("SNAPPY") --- diff --git a/scripts/spark2diffs/GpuParquetScanBase.diff b/scripts/spark2diffs/GpuParquetScanBase.diff index 64a998e8382..223533ce148 100644 --- a/scripts/spark2diffs/GpuParquetScanBase.diff +++ b/scripts/spark2diffs/GpuParquetScanBase.diff @@ -29,7 +29,10 @@ < meta: RapidsMeta[_, _]): Unit = { --- > meta: RapidsMeta[_, _, _]): Unit = { -59,65d84 +8a35,36 +> ParquetFieldIdShims.tagGpuSupportReadForFieldId(meta, sparkSession.sessionState.conf) +> +59,65d86 < // Spark 2.x doesn't have the rebase mode because the changes of calendar type weren't made < // so just skip the checks, since this is just explain only it would depend on how < // they set when they get to 3.x. The default in 3.x is EXCEPTION which would be good @@ -37,5 +40,5 @@ < < // Spark 2.x doesn't support the rebase mode < /* -95d113 +95d115 < */ diff --git a/scripts/spark2diffs/GpuRLikeMeta.diff b/scripts/spark2diffs/GpuRLikeMeta.diff index 4f459f3ea36..523abb697b6 100644 --- a/scripts/spark2diffs/GpuRLikeMeta.diff +++ b/scripts/spark2diffs/GpuRLikeMeta.diff @@ -2,5 +2,5 @@ < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -22a23 +23a24 > diff --git a/scripts/spark2diffs/GpuReadCSVFileFormat.diff b/scripts/spark2diffs/GpuReadCSVFileFormat.diff index b381da92f26..ff503c14a73 100644 --- a/scripts/spark2diffs/GpuReadCSVFileFormat.diff +++ b/scripts/spark2diffs/GpuReadCSVFileFormat.diff @@ -1,4 +1,4 @@ 5c5 < fsse.sqlContext.sparkSession, --- -> ShimLoader.getSparkShims.sessionFromPlan(fsse), +> SparkShimImpl.sessionFromPlan(fsse), diff --git a/scripts/spark2diffs/GpuReadJsonFileFormat.diff b/scripts/spark2diffs/GpuReadJsonFileFormat.diff index b381da92f26..ff503c14a73 100644 --- a/scripts/spark2diffs/GpuReadJsonFileFormat.diff +++ b/scripts/spark2diffs/GpuReadJsonFileFormat.diff @@ -1,4 +1,4 @@ 5c5 < fsse.sqlContext.sparkSession, --- -> ShimLoader.getSparkShims.sessionFromPlan(fsse), +> SparkShimImpl.sessionFromPlan(fsse), diff --git a/scripts/spark2diffs/GpuReadOrcFileFormat.diff b/scripts/spark2diffs/GpuReadOrcFileFormat.diff index 3e510390759..a2a89b0932f 100644 --- a/scripts/spark2diffs/GpuReadOrcFileFormat.diff +++ b/scripts/spark2diffs/GpuReadOrcFileFormat.diff @@ -1,4 +1,4 @@ 8c8 < fsse.sqlContext.sparkSession, --- -> ShimLoader.getSparkShims.sessionFromPlan(fsse), +> SparkShimImpl.sessionFromPlan(fsse), diff --git a/scripts/spark2diffs/GpuReadParquetFileFormat.diff b/scripts/spark2diffs/GpuReadParquetFileFormat.diff index b381da92f26..ff503c14a73 100644 --- a/scripts/spark2diffs/GpuReadParquetFileFormat.diff +++ b/scripts/spark2diffs/GpuReadParquetFileFormat.diff @@ -1,4 +1,4 @@ 5c5 < fsse.sqlContext.sparkSession, --- -> ShimLoader.getSparkShims.sessionFromPlan(fsse), +> SparkShimImpl.sessionFromPlan(fsse), diff --git a/scripts/spark2diffs/GpuRegExpExtractMeta.diff b/scripts/spark2diffs/GpuRegExpExtractMeta.diff index 2d63b1c8ccf..21fd18519b0 100644 --- a/scripts/spark2diffs/GpuRegExpExtractMeta.diff +++ b/scripts/spark2diffs/GpuRegExpExtractMeta.diff @@ -2,5 +2,5 @@ < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -50a51 +51a52 > diff --git a/scripts/spark2diffs/GpuRegExpReplaceMeta.diff b/scripts/spark2diffs/GpuRegExpReplaceMeta.diff index c7e3b7af610..b7138fe131a 100644 --- a/scripts/spark2diffs/GpuRegExpReplaceMeta.diff +++ b/scripts/spark2diffs/GpuRegExpReplaceMeta.diff @@ -2,5 +2,18 @@ < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -37a38 +6c6 +< extends TernaryExprMeta[RegExpReplace](expr, conf, parent, rule) { +--- +> extends QuaternaryExprMeta[RegExpReplace](expr, conf, parent, rule) { +11a12 +> GpuRegExpUtils.tagForRegExpEnabled(this) +36a38,43 +> +> GpuOverrides.extractLit(expr.pos).foreach { lit => +> if (lit.value.asInstanceOf[Int] != 1) { +> willNotWorkOnGpu("only a search starting position of 1 is supported") +> } +> } +37a45 > diff --git a/scripts/spark2diffs/GpuShuffleMeta.diff b/scripts/spark2diffs/GpuShuffleMeta.diff index 8498313b618..45ca70b5a97 100644 --- a/scripts/spark2diffs/GpuShuffleMeta.diff +++ b/scripts/spark2diffs/GpuShuffleMeta.diff @@ -5,7 +5,7 @@ 44c44 < if shuffle.sqlContext.sparkSession.sessionState.conf --- -> if ShimLoader.getSparkShims.sessionFromPlan(shuffle).sessionState.conf +> if SparkShimImpl.sessionFromPlan(shuffle).sessionState.conf 55a56,65 > // When AQE is enabled, we need to preserve meta data as outputAttributes and > // availableRuntimeDataTransition to the spark plan for the subsequent query stages. diff --git a/scripts/spark2diffs/GpuSortMergeJoinMeta.diff b/scripts/spark2diffs/GpuSortMergeJoinMeta.diff index 34c8687edd3..d34d97084e4 100644 --- a/scripts/spark2diffs/GpuSortMergeJoinMeta.diff +++ b/scripts/spark2diffs/GpuSortMergeJoinMeta.diff @@ -3,10 +3,10 @@ --- > * Copyright (c) 2019-2022, NVIDIA CORPORATION. 17,20c17 -< package com.nvidia.spark.rapids.shims.v2 +< package com.nvidia.spark.rapids.shims < < import com.nvidia.spark.rapids._ -< import com.nvidia.spark.rapids.shims.v2._ +< import com.nvidia.spark.rapids.shims._ --- > package com.nvidia.spark.rapids 29c26 diff --git a/scripts/spark2diffs/GpuStringSplitMeta.diff b/scripts/spark2diffs/GpuStringSplitMeta.diff index a349654ab10..b98168b9e2d 100644 --- a/scripts/spark2diffs/GpuStringSplitMeta.diff +++ b/scripts/spark2diffs/GpuStringSplitMeta.diff @@ -3,16 +3,18 @@ --- > parent: Option[RapidsMeta[_, _, _]], 6c6 -< extends BinaryExprMeta[StringSplit](expr, conf, parent, rule) { +< extends StringSplitRegBinaryExpMeta[StringSplit](expr, conf, parent, rule) { --- -> extends TernaryExprMeta[StringSplit](expr, conf, parent, rule) { -10,11c10 +> extends StringSplitRegExpMeta[StringSplit](expr, conf, parent, rule) { +12,13c12 < // 2.x uses expr.pattern not expr.regex -< val regexp = extractLit(expr.pattern) +< delimInfo = checkRegExp(expr.pattern) --- -> val regexp = extractLit(expr.regex) -27,28d25 +> delimInfo = checkRegExp(expr.regex) +15,16d13 < // 2.x has no limit parameter < /* -32d28 +26d22 < */ +27a24 +> diff --git a/scripts/spark2diffs/GpuStringToMapMeta.diff b/scripts/spark2diffs/GpuStringToMapMeta.diff new file mode 100644 index 00000000000..ff5264aeb27 --- /dev/null +++ b/scripts/spark2diffs/GpuStringToMapMeta.diff @@ -0,0 +1,6 @@ +3c3 +< parent: Option[RapidsMeta[_, _]], +--- +> parent: Option[RapidsMeta[_, _, _]], +20a21 +> diff --git a/scripts/spark2diffs/GpuToTimestamp.diff b/scripts/spark2diffs/GpuToTimestamp.diff index 182eec2a41e..ee4958c4c9e 100644 --- a/scripts/spark2diffs/GpuToTimestamp.diff +++ b/scripts/spark2diffs/GpuToTimestamp.diff @@ -2,9 +2,7 @@ < object GpuToTimestamp { --- > object GpuToTimestamp extends Arm { -26a27 -> -44d44 +53d52 < } -45a46 +54a54 > /** remove whitespace before month and day */ diff --git a/scripts/spark2diffs/OffsetWindowFunctionMeta.diff b/scripts/spark2diffs/OffsetWindowFunctionMeta.diff index 93b5d531340..51176c591ea 100644 --- a/scripts/spark2diffs/OffsetWindowFunctionMeta.diff +++ b/scripts/spark2diffs/OffsetWindowFunctionMeta.diff @@ -1,8 +1,8 @@ 2c2 < * Copyright (c) 2022, NVIDIA CORPORATION. --- -> * Copyright (c) 2021, NVIDIA CORPORATION. -27c27 +> * Copyright (c) 2021-2022, NVIDIA CORPORATION. +37c37 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], diff --git a/scripts/spark2diffs/RapidsConf.diff b/scripts/spark2diffs/RapidsConf.diff index f198f3f0522..09e6263986e 100644 --- a/scripts/spark2diffs/RapidsConf.diff +++ b/scripts/spark2diffs/RapidsConf.diff @@ -2,28 +2,49 @@ < * Copyright (c) 2022, NVIDIA CORPORATION. --- > * Copyright (c) 2019-2022, NVIDIA CORPORATION. -311c311 +23a24,25 +> import ai.rapids.cudf.Cuda +> +311c313 < .createWithDefault(ByteUnit.GiB.toBytes(1).toLong) --- > .createWithDefault(ByteUnit.GiB.toBytes(1)) -361c361 +368c370 < .createWithDefault(ByteUnit.MiB.toBytes(640).toLong) --- > .createWithDefault(ByteUnit.MiB.toBytes(640)) -393c393 +400c402 < .createWithDefault(ByteUnit.MiB.toBytes(8).toLong) --- > .createWithDefault(ByteUnit.MiB.toBytes(8)) -1378c1378 -< |$SPARK_HOME/bin/spark --jars 'rapids-4-spark_2.12-22.02.0-SNAPSHOT.jar,cudf-22.02.0-SNAPSHOT-cuda11.jar' \ +1372c1374 +< |$SPARK_HOME/bin/spark --jars 'rapids-4-spark_2.12-22.04.0-SNAPSHOT.jar,cudf-22.04.0-SNAPSHOT-cuda11.jar' \ --- -> |${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-22.02.0-SNAPSHOT.jar,cudf-22.02.0-SNAPSHOT-cuda11.jar' \ -1433a1434,1437 +> |${SPARK_HOME}/bin/spark --jars 'rapids-4-spark_2.12-22.04.0-SNAPSHOT.jar,cudf-22.04.0-SNAPSHOT-cuda11.jar' \ +1427a1430,1433 > printToggleHeader("Scans\n") > } > GpuOverrides.scans.values.toSeq.sortBy(_.tag.toString).foreach(_.confHelp(asTable)) > if (asTable) { -1440c1444 +1434c1440 < // com.nvidia.spark.rapids.python.PythonConfEntries.init() --- > com.nvidia.spark.rapids.python.PythonConfEntries.init() +1506d1511 +< // Spark 2.x doesn't have access to Cuda in CUDF so just allow +1508a1514,1528 +> if ("ASYNC".equalsIgnoreCase(pool)) { +> val driverVersion = Cuda.getDriverVersion +> val runtimeVersion = Cuda.getRuntimeVersion +> var fallbackMessage: Option[String] = None +> if (runtimeVersion < 11020 || driverVersion < 11020) { +> fallbackMessage = Some("CUDA runtime/driver does not support the ASYNC allocator") +> } else if (driverVersion < 11050) { +> fallbackMessage = Some("CUDA drivers before 11.5 have known incompatibilities with " + +> "the ASYNC allocator") +> } +> if (fallbackMessage.isDefined) { +> logWarning(s"${fallbackMessage.get}, falling back to ARENA") +> pool = "ARENA" +> } +> } diff --git a/scripts/spark2diffs/RapidsMeta.diff b/scripts/spark2diffs/RapidsMeta.diff index 93986450c2f..4b9ed979a94 100644 --- a/scripts/spark2diffs/RapidsMeta.diff +++ b/scripts/spark2diffs/RapidsMeta.diff @@ -1,89 +1,91 @@ 2c2 < * Copyright (c) 2022, NVIDIA CORPORATION. --- -> * Copyright (c) 2019-2021, NVIDIA CORPORATION. -23c23 +> * Copyright (c) 2019-2022, NVIDIA CORPORATION. +23c23,25 < import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, BinaryExpression, ComplexTypeMergingExpression, Expression, String2TrimExpression, TernaryExpression, UnaryExpression, WindowExpression, WindowFunction} --- +> import com.nvidia.spark.rapids.shims.SparkShimImpl +> > import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, BinaryExpression, ComplexTypeMergingExpression, Expression, QuaternaryExpression, String2TrimExpression, TernaryExpression, UnaryExpression, WindowExpression, WindowFunction} -25a26,27 +25a28,29 > import org.apache.spark.sql.catalyst.trees.TreeNodeTag > import org.apache.spark.sql.connector.read.Scan -27c29 +27c31 < import org.apache.spark.sql.execution.aggregate._ --- > import org.apache.spark.sql.execution.aggregate.BaseAggregateExec -30c32 +30c34 < import org.apache.spark.sql.execution.window.WindowExec --- > import org.apache.spark.sql.rapids.{CpuToGpuAggregateBufferConverter, GpuToCpuAggregateBufferConverter} -54a57 +54a59 > val gpuSupportedTag = TreeNodeTag[Set[String]]("rapids.gpu.supported") -67a71,72 +67a73,74 > * @tparam OUTPUT when converting to a GPU enabled version of the plan, the generic base > * type for all GPU enabled versions. -69c74 +69c76 < abstract class RapidsMeta[INPUT <: BASE, BASE]( --- > abstract class RapidsMeta[INPUT <: BASE, BASE, OUTPUT <: BASE]( -72c77 +72c79 < val parent: Option[RapidsMeta[_, _]], --- > val parent: Option[RapidsMeta[_, _, _]], -85a91,95 +85a93,97 > * The wrapped scans that should be examined > */ > val childScans: Seq[ScanMeta[_]] > > /** -95a106,110 +95a108,112 > * Convert what this wraps to a GPU enabled version. > */ > def convertToGpu(): OUTPUT > > /** -110a126 +110a128 > import RapidsMeta.gpuSupportedTag -127a144 +127a146 > childScans.foreach(_.recursiveCostPreventsRunningOnGpu()) -133a151 +133a153 > childScans.foreach(_.recursiveSparkPlanPreventsRunningOnGpu()) -140a159 +140a161 > childScans.foreach(_.recursiveSparkPlanRemoved()) -158a178,183 +158a180,185 > wrapped match { > case p: SparkPlan => > p.setTagValue(gpuSupportedTag, > p.getTagValue(gpuSupportedTag).getOrElse(Set.empty) + because) > case _ => > } -214a240,244 +214a242,246 > * Returns true iff all of the scans can be replaced. > */ > def canScansBeReplaced: Boolean = childScans.forall(_.canThisBeReplaced) > > /** -244a275 +244a277 > childScans.foreach(_.tagForGpu()) -380a412 +380a414 > childScans.foreach(_.print(append, depth + 1, all)) -403c435 +403c437 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -405,407c437 +405,407c439 < extends RapidsMeta[INPUT, Partitioning](part, conf, parent, rule) { < // 2.x - replaced GpuPartitioning with Partitioning, should be fine < // since BASE only used for convert --- > extends RapidsMeta[INPUT, Partitioning, GpuPartitioning](part, conf, parent, rule) { -410a441 +410a443 > override val childScans: Seq[ScanMeta[_]] = Seq.empty -431c462 +431c464 < parent: Option[RapidsMeta[_, _]]) --- > parent: Option[RapidsMeta[_, _, _]]) -437a469,505 +437a471,507 > override def convertToGpu(): GpuPartitioning = > throw new IllegalStateException("Cannot be converted to GPU") > } @@ -121,57 +123,57 @@ > > override def convertToGpu(): Scan = > throw new IllegalStateException("Cannot be converted to GPU") -446c514 +446c516 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -448c516 +448c518 < extends RapidsMeta[INPUT, DataWritingCommand](cmd, conf, parent, rule) { --- > extends RapidsMeta[INPUT, DataWritingCommand, GpuDataWritingCommand](cmd, conf, parent, rule) { -451a520 +451a522 > override val childScans: Seq[ScanMeta[_]] = Seq.empty -464c533 +464c535 < parent: Option[RapidsMeta[_, _]]) --- > parent: Option[RapidsMeta[_, _, _]]) -469a539,541 +469a541,543 > > override def convertToGpu(): GpuDataWritingCommand = > throw new IllegalStateException("Cannot be converted to GPU") -477c549 +477c551 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -479c551 +479c553 < extends RapidsMeta[INPUT, SparkPlan](plan, conf, parent, rule) { --- > extends RapidsMeta[INPUT, SparkPlan, GpuExec](plan, conf, parent, rule) { -484a557 +484a559 > childScans.foreach(_.recursiveSparkPlanPreventsRunningOnGpu()) -489a563 +489a565 > childScans.foreach(_.recursiveSparkPlanRemoved()) -514a589 +514a591 > override val childScans: Seq[ScanMeta[_]] = Seq.empty -550a626,630 +550a628,632 > > childPlans.head.wrapped > .getTagValue(GpuOverrides.preRowToColProjection).foreach { r2c => > wrapped.setTagValue(GpuOverrides.preRowToColProjection, r2c) > } -592c672 +592c674 < /*if (!canScansBeReplaced) { --- > if (!canScansBeReplaced) { -594c674 +594c676 < } */ --- > } -613a694,696 +613a696,698 > wrapped.getTagValue(RapidsMeta.gpuSupportedTag) > .foreach(_.diff(cannotBeReplacedReasons.get) > .foreach(willNotWorkOnGpu)) -637c720,724 +637c722,726 < convertToCpu --- > if (canThisBeReplaced) { @@ -179,27 +181,27 @@ > } else { > convertToCpu() > } -707c794 +707c796 < parent: Option[RapidsMeta[_, _]]) --- > parent: Option[RapidsMeta[_, _, _]]) -711a799,801 +711a801,803 > > override def convertToGpu(): GpuExec = > throw new IllegalStateException("Cannot be converted to GPU") -720c810 +720c812 < parent: Option[RapidsMeta[_, _]]) --- > parent: Option[RapidsMeta[_, _, _]]) -727a818,820 +727a820,822 > > override def convertToGpu(): GpuExec = > throw new IllegalStateException("Cannot be converted to GPU") -768c861 +768c863 < case agg: SparkPlan if agg.isInstanceOf[WindowExec] => --- -> case agg: SparkPlan if ShimLoader.getSparkShims.isWindowFunctionExec(agg) => -770,777c863 +> case agg: SparkPlan if SparkShimImpl.isWindowFunctionExec(agg) => +770,777c865 < case agg: HashAggregateExec => < // Spark 2.x doesn't have the BaseAggregateExec class < if (agg.groupingExpressions.isEmpty) { @@ -210,50 +212,50 @@ < case agg: SortAggregateExec => --- > case agg: BaseAggregateExec => -788c874 +788c876 < def getRegularOperatorContext(meta: RapidsMeta[_, _]): ExpressionContext = meta.wrapped match { --- > def getRegularOperatorContext(meta: RapidsMeta[_, _, _]): ExpressionContext = meta.wrapped match { -844c930 +844c932 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -846c932 +846c934 < extends RapidsMeta[INPUT, Expression](expr, conf, parent, rule) { --- > extends RapidsMeta[INPUT, Expression, Expression](expr, conf, parent, rule) { -852a939 +852a941 > override val childScans: Seq[ScanMeta[_]] = Seq.empty -991c1078 +991c1080 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -993a1081,1082 +993a1083,1084 > > override def convertToGpu(): GpuExpression -1002c1091 +1002c1093 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1005a1095,1099 +1005a1097,1101 > override final def convertToGpu(): GpuExpression = > convertToGpu(childExprs.head.convertToGpu()) > > def convertToGpu(child: Expression): GpuExpression > -1021c1115 +1021c1117 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1032c1126 +1032c1128 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1037a1132,1134 +1037a1134,1136 > if (needsAnsiCheck) { > GpuOverrides.checkAndTagAnsiAgg(ansiTypeToCheck, this) > } -1041a1139,1151 +1041a1141,1153 > > override final def convertToGpu(): GpuExpression = > convertToGpu(childExprs.map(_.convertToGpu())) @@ -267,18 +269,18 @@ > // The type to use to determine whether the aggregate could overflow. > // Set to None, if we should fallback for all types > val ansiTypeToCheck: Option[DataType] = Some(expr.dataType) -1050c1160 +1050c1162 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1052a1163,1164 +1052a1165,1166 > > def convertToGpu(childExprs: Seq[Expression]): GpuExpression -1061c1173 +1061c1175 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1072a1185,1200 +1072a1187,1202 > * Returns a buffer converter who can generate a Expression to transform the aggregation buffer > * of wrapped function from CPU format to GPU format. The conversion occurs on the CPU, so the > * generated expression should be a CPU Expression executed by row. @@ -295,11 +297,11 @@ > throw new NotImplementedError("The method should be implemented by specific functions") > > /** -1086c1214 +1086c1216 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1088a1217,1223 +1088a1219,1225 > > override final def convertToGpu(): GpuExpression = { > val Seq(lhs, rhs) = childExprs.map(_.convertToGpu()) @@ -307,11 +309,11 @@ > } > > def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression -1095c1230 +1095c1232 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1113c1248,1267 +1113c1250,1269 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], @@ -334,7 +336,7 @@ > expr: INPUT, > conf: RapidsConf, > parent: Option[RapidsMeta[_, _, _]], -1115a1270,1277 +1115a1272,1279 > > override final def convertToGpu(): GpuExpression = { > val Seq(child0, child1, child2, child3) = childExprs.map(_.convertToGpu()) @@ -343,11 +345,11 @@ > > def convertToGpu(val0: Expression, val1: Expression, > val2: Expression, val3: Expression): GpuExpression -1121c1283 +1121c1285 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1123a1286,1292 +1123a1288,1294 > > override final def convertToGpu(): GpuExpression = { > val gpuCol :: gpuTrimParam = childExprs.map(_.convertToGpu()) @@ -355,20 +357,20 @@ > } > > def convertToGpu(column: Expression, target: Option[Expression] = None): GpuExpression -1132c1301 +1132c1303 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -1134a1304,1307 +1134a1306,1309 > override final def convertToGpu(): GpuExpression = > convertToGpu(childExprs.map(_.convertToGpu())) > > def convertToGpu(childExprs: Seq[Expression]): GpuExpression -1143c1316 +1143c1318 < parent: Option[RapidsMeta[_, _]]) --- > parent: Option[RapidsMeta[_, _, _]]) -1147a1321,1323 +1147a1323,1325 > > override def convertToGpu(): GpuExpression = > throw new IllegalStateException("Cannot be converted to GPU") diff --git a/scripts/spark2diffs/RegexParser.diff b/scripts/spark2diffs/RegexParser.diff index 00a37973fc6..d0c11696b0b 100644 --- a/scripts/spark2diffs/RegexParser.diff +++ b/scripts/spark2diffs/RegexParser.diff @@ -2,7 +2,7 @@ < * Copyright (c) 2022, NVIDIA CORPORATION. --- > * Copyright (c) 2021-2022, NVIDIA CORPORATION. -794c794 +950c950 < } --- > } diff --git a/scripts/spark2diffs/ScalaUDF.diff b/scripts/spark2diffs/ScalaUDF.diff index 22fc16800b5..5bc36352585 100644 --- a/scripts/spark2diffs/ScalaUDF.diff +++ b/scripts/spark2diffs/ScalaUDF.diff @@ -2,13 +2,14 @@ < GpuOverrides.expr[ScalaUDF]( --- > def exprMeta: ExprRule[ScalaUDF] = GpuOverrides.expr[ScalaUDF]( -9a10,18 +9a10,19 > override protected def rowBasedScalaUDF: GpuRowBasedScalaUDFBase = > GpuRowBasedScalaUDF( > expr.function, > expr.dataType, > childExprs.map(_.convertToGpu()), > expr.inputEncoders, +> expr.outputEncoder, > expr.udfName, > expr.nullable, > expr.udfDeterministic) diff --git a/scripts/spark2diffs/StringSplitRegExpMeta.diff b/scripts/spark2diffs/StringSplitRegExpMeta.diff new file mode 100644 index 00000000000..88f19fb1f69 --- /dev/null +++ b/scripts/spark2diffs/StringSplitRegExpMeta.diff @@ -0,0 +1,4 @@ +3c3 +< parent: Option[RapidsMeta[_, _]], +--- +> parent: Option[RapidsMeta[_, _, _]], diff --git a/scripts/spark2diffs/Timestamp.diff b/scripts/spark2diffs/Timestamp.diff new file mode 100644 index 00000000000..5ccf0f19ea1 --- /dev/null +++ b/scripts/spark2diffs/Timestamp.diff @@ -0,0 +1,39 @@ +2c2 +< * Copyright (c) 2022, NVIDIA CORPORATION. +--- +> * Copyright (c) 2021-2022, NVIDIA CORPORATION. +20c20,21 +< import com.nvidia.spark.rapids.ExprRule +--- +> import com.nvidia.spark.rapids.{ExprChecks, ExprRule, GpuExpression, GpuOverrides, TypeEnum, TypeSig} +> import com.nvidia.spark.rapids.shims.SparkShimImpl +22c23,24 +< import org.apache.spark.sql.catalyst.expressions.Expression +--- +> import org.apache.spark.sql.catalyst.expressions.{Expression, GetTimestamp} +> import org.apache.spark.sql.rapids.{GpuGetTimestamp, UnixTimeExprMeta} +29,30c31,50 +< def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Map.empty +< } +--- +> def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq( +> GpuOverrides.expr[GetTimestamp]( +> "Gets timestamps from strings using given pattern.", +> ExprChecks.binaryProject(TypeSig.TIMESTAMP, TypeSig.TIMESTAMP, +> ("timeExp", +> TypeSig.STRING + TypeSig.DATE + TypeSig.TIMESTAMP, +> TypeSig.STRING + TypeSig.DATE + TypeSig.TIMESTAMP), +> ("format", TypeSig.lit(TypeEnum.STRING) +> .withPsNote(TypeEnum.STRING, "A limited number of formats are supported"), +> TypeSig.STRING)), +> (a, conf, p, r) => new UnixTimeExprMeta[GetTimestamp](a, conf, p, r) { +> override def shouldFallbackOnAnsiTimestamp: Boolean = +> SparkShimImpl.shouldFallbackOnAnsiTimestamp +> +> override def convertToGpu(lhs: Expression, rhs: Expression): GpuExpression = { +> GpuGetTimestamp(lhs, rhs, sparkFormat, strfFormat) +> } +> }) +> ).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap +> } +\ No newline at end of file diff --git a/scripts/spark2diffs/TreeNode.diff b/scripts/spark2diffs/TreeNode.diff index 9c7e90dc82f..5ed8ae6ef32 100644 --- a/scripts/spark2diffs/TreeNode.diff +++ b/scripts/spark2diffs/TreeNode.diff @@ -1,4 +1,4 @@ 2c2 < * Copyright (c) 2022, NVIDIA CORPORATION. --- -> * Copyright (c) 2021, NVIDIA CORPORATION. +> * Copyright (c) 2021-2022, NVIDIA CORPORATION. diff --git a/scripts/spark2diffs/TypeChecks.diff b/scripts/spark2diffs/TypeChecks.diff index 699564ac8fa..f8591008494 100644 --- a/scripts/spark2diffs/TypeChecks.diff +++ b/scripts/spark2diffs/TypeChecks.diff @@ -14,79 +14,79 @@ < meta: RapidsMeta[_, _], --- > meta: RapidsMeta[_, _, _], -562c562 +586c586 < val DECIMAL_64: TypeSig = decimal(GpuOverrides.DECIMAL64_MAX_PRECISION) --- > val DECIMAL_64: TypeSig = decimal(DType.DECIMAL64_MAX_PRECISION) -569c569 +593c593 < val DECIMAL_128: TypeSig = decimal(GpuOverrides.DECIMAL128_MAX_PRECISION) --- > val DECIMAL_128: TypeSig = decimal(DType.DECIMAL128_MAX_PRECISION) -703c703 +733c733 < def tag(meta: RapidsMeta[_, _]): Unit --- > def tag(meta: RapidsMeta[_, _, _]): Unit -716c716 +746c746 < meta: RapidsMeta[_, _], --- > meta: RapidsMeta[_, _, _], -773c773 +803c803 < override def tag(rapidsMeta: RapidsMeta[_, _]): Unit = { --- > override def tag(rapidsMeta: RapidsMeta[_, _, _]): Unit = { -777c777 +807c807 < private[this] def tagBase(rapidsMeta: RapidsMeta[_, _], willNotWork: String => Unit): Unit = { --- > private[this] def tagBase(rapidsMeta: RapidsMeta[_, _, _], willNotWork: String => Unit): Unit = { -835c835 +865c865 < def tag(meta: RapidsMeta[_, _], --- > def tag(meta: RapidsMeta[_, _, _], -846c846 +876c876 < override def tag(meta: RapidsMeta[_, _]): Unit = --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = -872c872 +902c902 < def tag(meta: RapidsMeta[_, _], --- > def tag(meta: RapidsMeta[_, _, _], -893c893 +923c923 < override def tag(rapidsMeta: RapidsMeta[_, _]): Unit = { --- > override def tag(rapidsMeta: RapidsMeta[_, _, _]): Unit = { -968c968 +998c998 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1020c1020 +1050c1050 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1063c1063 +1093c1093 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1113c1113 +1143c1143 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1159c1159 +1189c1189 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1195c1195 +1225c1225 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1345c1345 +1376c1376 < override def tag(meta: RapidsMeta[_, _]): Unit = { --- > override def tag(meta: RapidsMeta[_, _, _]): Unit = { -1355c1355 +1386c1386 < private[this] def tagBase(meta: RapidsMeta[_, _], willNotWork: String => Unit): Unit = { --- > private[this] def tagBase(meta: RapidsMeta[_, _, _], willNotWork: String => Unit): Unit = { -1690,1698d1689 +1721,1729d1720 < def getSparkVersion: String = { < // hack for databricks, try to find something more reliable? < if (SPARK_BUILD_USER.equals("Databricks")) { @@ -96,23 +96,19 @@ < } < } < -1713c1704 +1744c1735 < println(s"against version ${getSparkVersion} of Spark. Most of this should still") --- > println(s"against version ${ShimLoader.getSparkVersion} of Spark. Most of this should still") -1721c1712 -< println(s"supports a precision up to ${GpuOverrides.DECIMAL64_MAX_PRECISION} digits. Note that") ---- -> println(s"supports a precision up to ${DType.DECIMAL64_MAX_PRECISION} digits. Note that") -1823c1814 +1852c1843 < val allData = allSupportedTypes.toList.map { t => --- > val allData = allSupportedTypes.map { t => -1906c1897 +1935c1926 < val allData = allSupportedTypes.toList.map { t => --- > val allData = allSupportedTypes.map { t => -2010c2001 +2039c2030 < val allData = allSupportedTypes.toList.map { t => --- > val allData = allSupportedTypes.map { t => diff --git a/scripts/spark2diffs/TypeSigUtil.diff b/scripts/spark2diffs/TypeSigUtil.diff index 3911cb60122..be7de486bf9 100644 --- a/scripts/spark2diffs/TypeSigUtil.diff +++ b/scripts/spark2diffs/TypeSigUtil.diff @@ -1,7 +1,7 @@ 2c2 < * Copyright (c) 2022, NVIDIA CORPORATION. --- -> * Copyright (c) 2021, NVIDIA CORPORATION. +> * Copyright (c) 2021-2022, NVIDIA CORPORATION. 19c19 < import com.nvidia.spark.rapids.{TypeEnum, TypeSig} --- diff --git a/scripts/spark2diffs/UnixTimeExprMeta.diff b/scripts/spark2diffs/UnixTimeExprMeta.diff index 6925f619975..780bf3fb61b 100644 --- a/scripts/spark2diffs/UnixTimeExprMeta.diff +++ b/scripts/spark2diffs/UnixTimeExprMeta.diff @@ -2,13 +2,5 @@ < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -36,37d35 -< // Spark 2.x - ansi not available -< /* -40,41c38 -< } else */ -< if (!conf.incompatDateFormats) { ---- -> } else if (!conf.incompatDateFormats) { -76a74 +30a31 > diff --git a/scripts/spark2diffs/abs.diff b/scripts/spark2diffs/abs.diff index c5bd73367e2..603e15020b5 100644 --- a/scripts/spark2diffs/abs.diff +++ b/scripts/spark2diffs/abs.diff @@ -1,3 +1,3 @@ 6a7,8 -> // ANSI support for ABS was added in 3.2.0 SPARK-33275 -> override def convertToGpu(child: Expression): GpuExpression = GpuAbs(child, false) +> // ANSI support for ABS was added in 3.2.0 SPARK-33275 +> override def convertToGpu(child: Expression): GpuExpression = GpuAbs(child, false) diff --git a/scripts/spark2diffs/aggregate.diff b/scripts/spark2diffs/aggregate.diff index 75a9c45ce85..3738d2da935 100644 --- a/scripts/spark2diffs/aggregate.diff +++ b/scripts/spark2diffs/aggregate.diff @@ -12,7 +12,7 @@ > import ai.rapids.cudf.NvtxColor > import com.nvidia.spark.rapids.GpuMetric._ > import com.nvidia.spark.rapids.RapidsPluginImplicits._ -> import com.nvidia.spark.rapids.shims.v2.ShimUnaryExecNode +> import com.nvidia.spark.rapids.shims.{AggregationTagging, ShimUnaryExecNode, SparkShimImpl} > > import org.apache.spark.TaskContext > import org.apache.spark.internal.Logging @@ -39,7 +39,7 @@ < // BaseAggregateExec class in Spark 2.x --- > object AggregateUtils { -31,32c50,808 +31,32c50,827 < class GpuHashAggregateMeta( < val agg: HashAggregateExec, --- @@ -419,9 +419,8 @@ > "without grouping keys") > } > -> val shims = ShimLoader.getSparkShims > val groupingAttributes = groupingExpressions.map(_.toAttribute) -> val ordering = groupingAttributes.map(shims.sortOrder(_, Ascending, NullsFirst)) +> val ordering = groupingAttributes.map(SparkShimImpl.sortOrder(_, Ascending, NullsFirst)) > val aggBufferAttributes = groupingAttributes ++ > aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes) > val sorter = new GpuSorter(ordering, aggBufferAttributes) @@ -799,10 +798,30 @@ > } > } > +> object GpuBaseAggregateMeta { +> private val aggPairReplaceChecked = TreeNodeTag[Boolean]( +> "rapids.gpu.aggPairReplaceChecked") +> +> def getAggregateOfAllStages( +> currentMeta: SparkPlanMeta[_], logical: LogicalPlan): List[GpuBaseAggregateMeta[_]] = { +> currentMeta match { +> case aggMeta: GpuBaseAggregateMeta[_] if aggMeta.agg.logicalLink.contains(logical) => +> List[GpuBaseAggregateMeta[_]](aggMeta) ++ +> getAggregateOfAllStages(aggMeta.childPlans.head, logical) +> case shuffleMeta: GpuShuffleMeta => +> getAggregateOfAllStages(shuffleMeta.childPlans.head, logical) +> case sortMeta: GpuSortMeta => +> getAggregateOfAllStages(sortMeta.childPlans.head, logical) +> case _ => +> List[GpuBaseAggregateMeta[_]]() +> } +> } +> } +> > abstract class GpuBaseAggregateMeta[INPUT <: SparkPlan]( > plan: INPUT, > aggRequiredChildDistributionExpressions: Option[Seq[Expression]], -34,36c810,813 +34,36c829,832 < parent: Option[RapidsMeta[_, _]], < rule: DataFromReplacementRule) < extends SparkPlanMeta[HashAggregateExec](agg, conf, parent, rule) { @@ -811,7 +830,7 @@ > rule: DataFromReplacementRule) extends SparkPlanMeta[INPUT](plan, conf, parent, rule) { > > val agg: BaseAggregateExec -60a838,846 +60a857,869 > > if (agg.aggregateExpressions.exists(expr => expr.isDistinct) > && agg.aggregateExpressions.exists(expr => expr.filter.isDefined)) { @@ -821,7 +840,39 @@ > willNotWorkOnGpu( > "DISTINCT and FILTER cannot be used in aggregate functions at the same time") > } -124a911,921 +> +> if (AggregationTagging.mustReplaceBoth) { +> tagForMixedReplacement() +> } +124a934,972 +> +> /** Prevent mixing of CPU and GPU aggregations */ +> private def tagForMixedReplacement(): Unit = { +> // only run the check for final stages that have not already been checked +> val haveChecked = +> agg.getTagValue[Boolean](GpuBaseAggregateMeta.aggPairReplaceChecked).contains(true) +> val needCheck = !haveChecked && agg.aggregateExpressions.exists { +> case e: AggregateExpression if e.mode == Final => true +> case _ => false +> } +> +> if (needCheck) { +> agg.setTagValue(GpuBaseAggregateMeta.aggPairReplaceChecked, true) +> val stages = GpuBaseAggregateMeta.getAggregateOfAllStages(this, agg.logicalLink.get) +> // check if aggregations will mix CPU and GPU across stages +> val hasMixedAggs = stages.indices.exists { +> case i if i == stages.length - 1 => false +> case i => stages(i).canThisBeReplaced ^ stages(i + 1).canThisBeReplaced +> } +> if (hasMixedAggs) { +> stages.foreach { +> case aggMeta if aggMeta.canThisBeReplaced => +> aggMeta.willNotWorkOnGpu("mixing CPU and GPU aggregations is not supported") +> case _ => +> } +> } +> } +> } > > override def convertToGpu(): GpuExec = { > GpuHashAggregateExec( @@ -833,7 +884,7 @@ > childPlans.head.convertIfNeeded(), > conf.gpuTargetBatchSizeBytes) > } -127,128c924,930 +127,128c975,981 < class GpuSortAggregateExecMeta( < val agg: SortAggregateExec, --- @@ -844,7 +895,7 @@ > abstract class GpuTypedImperativeSupportedAggregateExecMeta[INPUT <: BaseAggregateExec]( > plan: INPUT, > aggRequiredChildDistributionExpressions: Option[Seq[Expression]], -130,132c932,940 +130,132c983,991 < parent: Option[RapidsMeta[_, _]], < rule: DataFromReplacementRule) < extends SparkPlanMeta[SortAggregateExec](agg, conf, parent, rule) { @@ -858,7 +909,7 @@ > expr.aggregateFunction.isInstanceOf[TypedImperativeAggregate[_]] && > (expr.mode == Partial || expr.mode == PartialMerge) > } -134,141c942,943 +134,141c993,994 < val groupingExpressions: Seq[BaseExprMeta[_]] = < agg.groupingExpressions.map(GpuOverrides.wrapExpr(_, conf, Some(this))) < val aggregateExpressions: Seq[BaseExprMeta[_]] = @@ -870,7 +921,7 @@ --- > // overriding data types of Aggregation Buffers if necessary > if (mayNeedAggBufferConversion) overrideAggBufTypes() -143,144c945,956 +143,144c996,1007 < override val childExprs: Seq[BaseExprMeta[_]] = < groupingExpressions ++ aggregateExpressions ++ aggregateAttributes ++ resultExpressions --- @@ -886,7 +937,7 @@ > case aggMeta: TypedImperativeAggExprMeta[_] => aggMeta.supportBufferConversion > case _ => true > } -147,153c959,1003 +147,153c1010,1054 < // We don't support Arrays and Maps as GroupBy keys yet, even they are nested in Structs. So, < // we need to run recursive type check on the structs. < val arrayOrMapGroupings = agg.groupingExpressions.exists(e => @@ -940,9 +991,9 @@ > conf.gpuTargetBatchSizeBytes) > } else { > super.convertToGpu() -154a1005 +154a1056 > } -156c1007,1268 +156c1058,1304 < tagForReplaceMode() --- > /** @@ -1033,7 +1084,7 @@ > meta.agg.setTagValue(bufferConverterInjected, true) > > // Fetch AggregateMetas of all stages which belong to current Aggregate -> val stages = getAggregateOfAllStages(meta, meta.agg.logicalLink.get) +> val stages = GpuBaseAggregateMeta.getAggregateOfAllStages(meta, meta.agg.logicalLink.get) > > // Find out stages in which the buffer converters are essential. > val needBufferConversion = stages.indices.map { @@ -1151,10 +1202,10 @@ > } > converters.dequeue() match { > case Left(converter) => -> ShimLoader.getSparkShims.alias(converter.createExpression(ref), +> SparkShimImpl.alias(converter.createExpression(ref), > ref.name + "_converted")(NamedExpression.newExprId) > case Right(converter) => -> ShimLoader.getSparkShims.alias(converter.createExpression(ref), +> SparkShimImpl.alias(converter.createExpression(ref), > ref.name + "_converted")(NamedExpression.newExprId) > } > case retExpr => @@ -1164,21 +1215,6 @@ > expressions > } > -> private def getAggregateOfAllStages( -> currentMeta: SparkPlanMeta[_], logical: LogicalPlan): List[GpuBaseAggregateMeta[_]] = { -> currentMeta match { -> case aggMeta: GpuBaseAggregateMeta[_] if aggMeta.agg.logicalLink.contains(logical) => -> List[GpuBaseAggregateMeta[_]](aggMeta) ++ -> getAggregateOfAllStages(aggMeta.childPlans.head, logical) -> case shuffleMeta: GpuShuffleMeta => -> getAggregateOfAllStages(shuffleMeta.childPlans.head, logical) -> case sortMeta: GpuSortMeta => -> getAggregateOfAllStages(sortMeta.childPlans.head, logical) -> case _ => -> List[GpuBaseAggregateMeta[_]]() -> } -> } -> > @tailrec > private def nextEdgeForConversion(meta: SparkPlanMeta[_]): Seq[SparkPlanMeta[_]] = { > val child = meta.childPlans.head @@ -1207,7 +1243,7 @@ > agg.requiredChildDistributionExpressions, conf, parent, rule) { > override def tagPlanForGpu(): Unit = { > super.tagPlanForGpu() -183,245d1294 +183,245d1330 < < /** < * Tagging checks tied to configs that control the aggregation modes that are replaced. @@ -1271,15 +1307,15 @@ < s"Set ${conf.partialMergeDistinctEnabled} to true if desired") < } < } -248,250d1296 +248,250d1332 < // SPARK 2.x we can't check for the TypedImperativeAggregate properly so don't say we do the < // ObjectHashAggregate < /* -254c1300 +254c1336 < parent: Option[RapidsMeta[_, _]], --- > parent: Option[RapidsMeta[_, _, _]], -259c1305,1506 +259c1341,1542 < */ --- > /** diff --git a/scripts/spark2diffs/average.diff b/scripts/spark2diffs/average.diff index 243dcaf2728..2ac0dc5485d 100644 --- a/scripts/spark2diffs/average.diff +++ b/scripts/spark2diffs/average.diff @@ -1,9 +1,6 @@ -32c32,37 -< ---- +32a33,37 +> override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = +> GpuAverage(childExprs.head) > -> override def convertToGpu(childExprs: Seq[Expression]): GpuExpression = -> GpuAverage(childExprs.head) -> -> // Average is not supported in ANSI mode right now, no matter the type -> override val ansiTypeToCheck: Option[DataType] = None +> // Average is not supported in ANSI mode right now, no matter the type +> override val ansiTypeToCheck: Option[DataType] = None diff --git a/scripts/spark2diffs/cast.diff b/scripts/spark2diffs/cast.diff index 0b4eac8a499..b40aec3b84f 100644 --- a/scripts/spark2diffs/cast.diff +++ b/scripts/spark2diffs/cast.diff @@ -1,3 +1,7 @@ +1c1 +< GpuOverrides.expr[Cast]( +--- +> GpuOverrides.expr[Cast]( 4c4,5 < (cast, conf, p, r) => new CastExprMeta[Cast](cast, false, conf, p, r, --- diff --git a/scripts/spark2diffs/createCudfDecimal.diff b/scripts/spark2diffs/createCudfDecimal.diff index 152836ed88c..c2ca1a7273d 100644 --- a/scripts/spark2diffs/createCudfDecimal.diff +++ b/scripts/spark2diffs/createCudfDecimal.diff @@ -1,20 +1,6 @@ -1,7c1,9 -< def createCudfDecimal(precision: Int, scale: Int): Option[String] = { -< if (precision <= GpuOverrides.DECIMAL32_MAX_PRECISION) { -< Some("DECIMAL32") -< } else if (precision <= GpuOverrides.DECIMAL64_MAX_PRECISION) { -< Some("DECIMAL64") -< } else if (precision <= GpuOverrides.DECIMAL128_MAX_PRECISION) { -< Some("DECIMAL128") +1,2c1,2 +< def createCudfDecimal(dt: DecimalType): Option[String] = +< createInternalCudfDecimal(dt.precision, dt.scale) --- -> def createCudfDecimal(dt: DecimalType): DType = { -> createCudfDecimal(dt.precision, dt.scale) -> def createCudfDecimal(precision: Int, scale: Int): DType = { -> if (precision <= DType.DECIMAL32_MAX_PRECISION) { -> DType.create(DType.DTypeEnum.DECIMAL32, -scale) -> } else if (precision <= DType.DECIMAL64_MAX_PRECISION) { -> DType.create(DType.DTypeEnum.DECIMAL64, -scale) -> } else if (precision <= DType.DECIMAL128_MAX_PRECISION) { -> DType.create(DType.DTypeEnum.DECIMAL128, -scale) -10d11 -< None +> def createCudfDecimal(dt: DecimalType): DType = +> DecimalUtils.createDecimalType(dt.precision, dt.scale) diff --git a/scripts/spark2diffs/getPrecisionForIntegralType.diff b/scripts/spark2diffs/getPrecisionForIntegralType.diff deleted file mode 100644 index ec4b160039e..00000000000 --- a/scripts/spark2diffs/getPrecisionForIntegralType.diff +++ /dev/null @@ -1,12 +0,0 @@ -1,5c1,5 -< def getPrecisionForIntegralType(input: String): Int = input match { -< case "INT8" => 3 // -128 to 127 -< case "INT16" => 5 // -32768 to 32767 -< case "INT32" => 10 // -2147483648 to 2147483647 -< case "INT64" => 19 // -9223372036854775808 to 9223372036854775807 ---- -> def getPrecisionForIntegralType(input: DType): Int = input match { -> case DType.INT8 => 3 // -128 to 127 -> case DType.INT16 => 5 // -32768 to 32767 -> case DType.INT32 => 10 // -2147483648 to 2147483647 -> case DType.INT64 => 19 // -9223372036854775808 to 9223372036854775807 diff --git a/scripts/spark2diffs/optionallyAsDecimalType.diff b/scripts/spark2diffs/optionallyAsDecimalType.diff index 682fc15f583..a683c41ce41 100644 --- a/scripts/spark2diffs/optionallyAsDecimalType.diff +++ b/scripts/spark2diffs/optionallyAsDecimalType.diff @@ -1,4 +1,6 @@ -4c4 +4,6c4 +< // Spark 2.x diff using string for type so don't need CUDF < val prec = DecimalUtil.getPrecisionForIntegralType(getNonNestedRapidsType(t)) +< Some(DecimalType(prec, 0)) --- -> val prec = DecimalUtil.getPrecisionForIntegralType(GpuColumnVector.getNonNestedRapidsType(t)) +> Some(DecimalType(GpuColumnVector.getNonNestedRapidsType(t).getPrecisionForInt, 0)) diff --git a/scripts/spark2diffs/regexreplace.diff b/scripts/spark2diffs/regexreplace.diff new file mode 100644 index 00000000000..849d753e2c3 --- /dev/null +++ b/scripts/spark2diffs/regexreplace.diff @@ -0,0 +1,2 @@ +10a11 +> // Spark 3.1.1-specific LEAD expression, using custom OffsetWindowFunctionMeta. diff --git a/scripts/spark2diffs/toRapidsStringOrNull.diff b/scripts/spark2diffs/toRapidsStringOrNull.diff index 485a19004ed..815fc376a62 100644 --- a/scripts/spark2diffs/toRapidsStringOrNull.diff +++ b/scripts/spark2diffs/toRapidsStringOrNull.diff @@ -1,4 +1,4 @@ -1,20c1,31 +1,22c1,34 < def toRapidsStringOrNull(dtype: DataType): Option[String] = { < dtype match { < case _: LongType => Some("INT64") @@ -17,10 +17,12 @@ < // Decimal supportable check has been conducted in the GPU plan overriding stage. < // So, we don't have to handle decimal-supportable problem at here. < val dt = dtype.asInstanceOf[DecimalType] -< createCudfDecimal(dt.precision, dt.scale) +< createCudfDecimal(dt) +< case _: GpuUnsignedIntegerType => Some("UINT32") +< case _: GpuUnsignedLongType => Some("UINT64") < case _ => None --- -> private static DType toRapidsOrNull(DataType type) { +> private static DType toRapidsOrNullCommon(DataType type) { > if (type instanceof LongType) { > return DType.INT64; > } else if (type instanceof DoubleType) { @@ -49,7 +51,10 @@ > } else if (type instanceof DecimalType) { > // Decimal supportable check has been conducted in the GPU plan overriding stage. > // So, we don't have to handle decimal-supportable problem at here. -> DecimalType dt = (DecimalType) type; -> return DecimalUtil.createCudfDecimal(dt.precision(), dt.scale()); -21a33 +> return DecimalUtil.createCudfDecimal((DecimalType) type); +> } else if (type instanceof GpuUnsignedIntegerType) { +> return DType.UINT32; +> } else if (type instanceof GpuUnsignedLongType) { +> return DType.UINT64; +23a36 > return null; diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DateUtils.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DateUtils.scala index e856e306596..d6d205f68a8 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DateUtils.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DateUtils.scala @@ -21,6 +21,8 @@ import java.time._ import scala.collection.mutable.ListBuffer import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.rapids.{GpuToTimestamp, LegacyTimeParserPolicy} /** * Class for helper functions for Date @@ -186,4 +188,57 @@ object DateUtils { } case class TimestampFormatConversionException(reason: String) extends Exception + + def tagAndGetCudfFormat( + meta: RapidsMeta[_, _], + sparkFormat: String, + parseString: Boolean): String = { + var strfFormat: String = null + if (GpuOverrides.getTimeParserPolicy == LegacyTimeParserPolicy) { + try { + // try and convert the format to cuDF format - this will throw an exception if + // the format contains unsupported characters or words + strfFormat = toStrf(sparkFormat, parseString) + // format parsed ok but we have no 100% compatible formats in LEGACY mode + if (GpuToTimestamp.LEGACY_COMPATIBLE_FORMATS.contains(sparkFormat)) { + // LEGACY support has a number of issues that mean we cannot guarantee + // compatibility with CPU + // - we can only support 4 digit years but Spark supports a wider range + // - we use a proleptic Gregorian calender but Spark uses a hybrid Julian+Gregorian + // calender in LEGACY mode + // Spark 2.x doesn't support, assume false + val ansiEnabled = false + if (ansiEnabled) { + meta.willNotWorkOnGpu("LEGACY format in ANSI mode is not supported on the GPU") + } else if (!meta.conf.incompatDateFormats) { + meta.willNotWorkOnGpu(s"LEGACY format '$sparkFormat' on the GPU is not guaranteed " + + s"to produce the same results as Spark on CPU. Set " + + s"${RapidsConf.INCOMPATIBLE_DATE_FORMATS.key}=true to force onto GPU.") + } + } else { + meta.willNotWorkOnGpu(s"LEGACY format '$sparkFormat' is not supported on the GPU.") + } + } catch { + case e: TimestampFormatConversionException => + meta.willNotWorkOnGpu(s"Failed to convert ${e.reason} ${e.getMessage}") + } + } else { + try { + // try and convert the format to cuDF format - this will throw an exception if + // the format contains unsupported characters or words + strfFormat = toStrf(sparkFormat, parseString) + // format parsed ok, so it is either compatible (tested/certified) or incompatible + if (!GpuToTimestamp.CORRECTED_COMPATIBLE_FORMATS.contains(sparkFormat) && + !meta.conf.incompatDateFormats) { + meta.willNotWorkOnGpu(s"CORRECTED format '$sparkFormat' on the GPU is not guaranteed " + + s"to produce the same results as Spark on CPU. Set " + + s"${RapidsConf.INCOMPATIBLE_DATE_FORMATS.key}=true to force onto GPU.") + } + } catch { + case e: TimestampFormatConversionException => + meta.willNotWorkOnGpu(s"Failed to convert ${e.reason} ${e.getMessage}") + } + } + strfFormat + } } diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala index 90c4cdc67d1..c52d8a9f867 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/DecimalUtil.scala @@ -26,7 +26,10 @@ object DecimalUtil { IllegalArgumentException(dtype + " is not supported for GPU processing yet.")) } - def createCudfDecimal(precision: Int, scale: Int): Option[String] = { + def createCudfDecimal(dt: DecimalType): Option[String] = + createInternalCudfDecimal(dt.precision, dt.scale) + + def createInternalCudfDecimal(precision: Int, scale: Int): Option[String] = { if (precision <= GpuOverrides.DECIMAL32_MAX_PRECISION) { Some("DECIMAL32") } else if (precision <= GpuOverrides.DECIMAL64_MAX_PRECISION) { @@ -59,13 +62,16 @@ object DecimalUtil { // Decimal supportable check has been conducted in the GPU plan overriding stage. // So, we don't have to handle decimal-supportable problem at here. val dt = dtype.asInstanceOf[DecimalType] - createCudfDecimal(dt.precision, dt.scale) + createCudfDecimal(dt) + case _: GpuUnsignedIntegerType => Some("UINT32") + case _: GpuUnsignedLongType => Some("UINT64") case _ => None } } /** * Get the number of decimal places needed to hold the integral type held by this column + * Keep here for Spark 2.x so we don't need CUDF */ def getPrecisionForIntegralType(input: String): Int = input match { case "INT8" => 3 // -128 to 127 @@ -74,12 +80,14 @@ object DecimalUtil { case "INT64" => 19 // -9223372036854775808 to 9223372036854775807 case t => throw new IllegalArgumentException(s"Unsupported type $t") } + // The following types were copied from Spark's DecimalType class private val BooleanDecimal = DecimalType(1, 0) def optionallyAsDecimalType(t: DataType): Option[DecimalType] = t match { case dt: DecimalType => Some(dt) case ByteType | ShortType | IntegerType | LongType => + // Spark 2.x diff using string for type so don't need CUDF val prec = DecimalUtil.getPrecisionForIntegralType(getNonNestedRapidsType(t)) Some(DecimalType(prec, 0)) case BooleanType => Some(BooleanDecimal) diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCastMeta.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCastMeta.scala index 52cfb45f095..7e4f169355c 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCastMeta.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCastMeta.scala @@ -134,6 +134,10 @@ final class CastExprMeta[INPUT <: Cast]( recursiveTagExprForGpuCheck(keyFrom, keyTo, depth + 1) recursiveTagExprForGpuCheck(valueFrom, valueTo, depth + 1) + case (MapType(keyFrom, valueFrom, _), StringType) => + recursiveTagExprForGpuCheck(keyFrom, StringType, depth + 1) + recursiveTagExprForGpuCheck(valueFrom, StringType, depth + 1) + case _ => } } diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataTypes.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataTypes.scala new file mode 100644 index 00000000000..8e81df94aaf --- /dev/null +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuDataTypes.scala @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.spark.rapids + +import org.apache.spark.sql.types.DataType + +/** + * An unsigned, 32-bit integer type that maps to DType.UINT32 in cudf. + * @note This type should NOT be used in Catalyst plan nodes that could be exposed to + * CPU expressions. + */ +class GpuUnsignedIntegerType private() extends DataType { + // The companion object and this class are separated so the companion object also subclasses + // this type. Otherwise the companion object would be of type "UnsignedIntegerType$" in + // byte code. + // Defined with a private constructor so the companion object is the only possible instantiation. + override def defaultSize: Int = 4 + + override def simpleString: String = "uint" + + override def asNullable: DataType = this +} + +case object GpuUnsignedIntegerType extends GpuUnsignedIntegerType + + +/** + * An unsigned, 64-bit integer type that maps to DType.UINT64 in cudf. + * @note This type should NOT be used in Catalyst plan nodes that could be exposed to + * CPU expressions. + */ +class GpuUnsignedLongType private() extends DataType { + // The companion object and this class are separated so the companion object also subclasses + // this type. Otherwise the companion object would be of type "UnsignedIntegerType$" in + // byte code. + // Defined with a private constructor so the companion object is the only possible instantiation. + override def defaultSize: Int = 8 + + override def simpleString: String = "ulong" + + override def asNullable: DataType = this +} + +case object GpuUnsignedLongType extends GpuUnsignedLongType diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 72f9593effa..29ed4e2e274 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -23,12 +23,13 @@ import scala.reflect.ClassTag import scala.util.control.NonFatal import com.nvidia.spark.rapids.RapidsConf.{SUPPRESS_PLANNING_FAILURE, TEST_CONF} -import com.nvidia.spark.rapids.shims._ +import com.nvidia.spark.rapids.shims.{GpuBroadcastHashJoinMeta, GpuShuffledHashJoinMeta, GpuSortMergeJoinMeta, GpuSpecifiedWindowFrameMeta, GpuWindowExpressionMeta, GpuWindowSpecDefinitionMeta, OffsetWindowFunctionMeta} import org.apache.spark.internal.Logging import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ +import org.apache.spark.sql.catalyst.expressions.rapids.TimeStamp import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.ArrayData @@ -394,6 +395,10 @@ object JsonFormatType extends FileFormatType { override def toString = "JSON" } +object AvroFormatType extends FileFormatType { + override def toString = "Avro" +} + sealed trait FileFormatOp object ReadFileOp extends FileFormatOp { override def toString = "read" @@ -464,11 +469,6 @@ object GpuOverrides extends Logging { listeners.clear() } - def canRegexpBeTreatedLikeARegularString(strLit: UTF8String): Boolean = { - val s = strLit.toString - !regexList.exists(pattern => s.contains(pattern)) - } - @scala.annotation.tailrec def extractLit(exp: Expression): Option[Literal] = exp match { case l: Literal => Some(l) @@ -674,9 +674,9 @@ object GpuOverrides extends Logging { .map(r => r.wrap(expr, conf, parent, r).asInstanceOf[BaseExprMeta[INPUT]]) .getOrElse(new RuleNotFoundExprMeta(expr, conf, parent)) - lazy val fileFormats: Map[FileFormatType, Map[FileFormatOp, FileFormatChecks]] = Map( + lazy val basicFormats: Map[FileFormatType, Map[FileFormatOp, FileFormatChecks]] = Map( (CsvFormatType, FileFormatChecks( - cudfRead = TypeSig.commonCudfTypes, + cudfRead = TypeSig.commonCudfTypes + TypeSig.DECIMAL_128, cudfWrite = TypeSig.none, sparkSig = TypeSig.cpuAtomics)), (ParquetFormatType, FileFormatChecks( @@ -695,11 +695,20 @@ object GpuOverrides extends Logging { sparkSig = (TypeSig.cpuAtomics + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP + TypeSig.UDT).nested())), (JsonFormatType, FileFormatChecks( - cudfRead = TypeSig.commonCudfTypes, + cudfRead = TypeSig.commonCudfTypes + TypeSig.DECIMAL_128, + cudfWrite = TypeSig.none, + sparkSig = (TypeSig.cpuAtomics + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP + + TypeSig.UDT).nested())), + (AvroFormatType, FileFormatChecks( + cudfRead = TypeSig.BOOLEAN + TypeSig.BYTE + TypeSig.SHORT + TypeSig.INT + TypeSig.LONG + + TypeSig.FLOAT + TypeSig.DOUBLE + TypeSig.STRING, cudfWrite = TypeSig.none, sparkSig = (TypeSig.cpuAtomics + TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP + TypeSig.UDT).nested()))) + // the only shim changin is Spark 3.3.X so leave off for now + lazy val fileFormats = basicFormats // ++ SparkShimImpl.getFileFormats + val commonExpressions: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq( expr[Literal]( "Holds a static value from the query", @@ -951,6 +960,15 @@ object GpuOverrides extends Logging { TypeSig.all))), (denseRank, conf, p, r) => new ExprMeta[DenseRank](denseRank, conf, p, r) { }), + expr[PercentRank]( + "Window function that returns the percent rank value within the aggregation window", + ExprChecks.windowOnly(TypeSig.DOUBLE, TypeSig.DOUBLE, + repeatingParamCheck = + Some(RepeatingParamCheck("ordering", + TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL, + TypeSig.all))), + (percentRank, conf, p, r) => new ExprMeta[PercentRank](percentRank, conf, p, r) { + }), expr[Lead]( "Window function that returns N entries ahead of this one", ExprChecks.windowOnly( @@ -1074,6 +1092,15 @@ object GpuOverrides extends Logging { ExprChecks.mathUnaryWithAst, (a, conf, p, r) => new UnaryAstExprMeta[Cbrt](a, conf, p, r) { }), + expr[Hypot]( + "Pythagorean addition (Hypotenuse) of real numbers", + ExprChecks.binaryProject( + TypeSig.DOUBLE, + TypeSig.DOUBLE, + ("lhs", TypeSig.DOUBLE, TypeSig.DOUBLE), + ("rhs", TypeSig.DOUBLE, TypeSig.DOUBLE)), + (a, conf, p, r) => new BinaryExprMeta[Hypot](a, conf, p, r) { + }), expr[Floor]( "Floor of a number", ExprChecks.unaryProjectInputMatchesOutput( @@ -1472,9 +1499,9 @@ object GpuOverrides extends Logging { }), expr[Pmod]( "Pmod", - ExprChecks.binaryProject(TypeSig.integral + TypeSig.fp, TypeSig.cpuNumeric, - ("lhs", TypeSig.integral + TypeSig.fp, TypeSig.cpuNumeric), - ("rhs", TypeSig.integral + TypeSig.fp, TypeSig.cpuNumeric)), + ExprChecks.binaryProject(TypeSig.gpuNumeric, TypeSig.cpuNumeric, + ("lhs", TypeSig.gpuNumeric, TypeSig.cpuNumeric), + ("rhs", TypeSig.gpuNumeric, TypeSig.cpuNumeric)), (a, conf, p, r) => new BinaryExprMeta[Pmod](a, conf, p, r) { }), expr[Add]( @@ -1681,9 +1708,9 @@ object GpuOverrides extends Logging { expr[Remainder]( "Remainder or modulo", ExprChecks.binaryProject( - TypeSig.integral + TypeSig.fp, TypeSig.cpuNumeric, - ("lhs", TypeSig.integral + TypeSig.fp, TypeSig.cpuNumeric), - ("rhs", TypeSig.integral + TypeSig.fp, TypeSig.cpuNumeric)), + TypeSig.gpuNumeric, TypeSig.cpuNumeric, + ("lhs", TypeSig.gpuNumeric, TypeSig.cpuNumeric), + ("rhs", TypeSig.gpuNumeric, TypeSig.cpuNumeric)), (a, conf, p, r) => new BinaryExprMeta[Remainder](a, conf, p, r) { }), expr[AggregateExpression]( @@ -2044,9 +2071,13 @@ object GpuOverrides extends Logging { (in, conf, p, r) => new GpuGetArrayItemMeta(in, conf, p, r)), expr[GetMapValue]( "Gets Value from a Map based on a key", - ExprChecks.binaryProject(TypeSig.STRING, TypeSig.all, - ("map", TypeSig.MAP.nested(TypeSig.STRING), TypeSig.MAP.nested(TypeSig.all)), - ("key", TypeSig.lit(TypeEnum.STRING), TypeSig.all)), + ExprChecks.binaryProject( + (TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.NULL + + TypeSig.DECIMAL_128 + TypeSig.MAP).nested(), + TypeSig.all, + ("map", TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.MAP), TypeSig.MAP.nested(TypeSig.all)), + ("key", TypeSig.commonCudfTypesLit() + TypeSig.lit(TypeEnum.DECIMAL), TypeSig.all)), (in, conf, p, r) => new GpuGetMapValueMeta(in, conf, p, r)), expr[ElementAt]( "Returns element of array at given(1-based) index in value if column is array. " + @@ -2056,14 +2087,12 @@ object GpuOverrides extends Logging { TypeSig.DECIMAL_128 + TypeSig.MAP).nested(), TypeSig.all, ("array/map", TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.MAP) + - TypeSig.MAP.nested(TypeSig.STRING) - .withPsNote(TypeEnum.MAP ,"If it's map, only string is supported."), + TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.MAP) + .withPsNote(TypeEnum.MAP ,"If it's map, only primitive key types are supported."), TypeSig.ARRAY.nested(TypeSig.all) + TypeSig.MAP.nested(TypeSig.all)), - ("index/key", (TypeSig.lit(TypeEnum.INT) + TypeSig.lit(TypeEnum.STRING)) - .withPsNote(TypeEnum.INT, "ints are only supported as array indexes, " + - "not as maps keys") - .withPsNote(TypeEnum.STRING, "strings are only supported as map keys, " + - "not array indexes"), + ("index/key", (TypeSig.INT + TypeSig.commonCudfTypesLit() + TypeSig.lit(TypeEnum.DECIMAL)) + .withPsNote(TypeEnum.INT, "Only ints are supported as array indexes"), TypeSig.all)), (in, conf, p, r) => new BinaryExprMeta[ElementAt](in, conf, p, r) { override def tagExprForGpu(): Unit = { @@ -2071,9 +2100,15 @@ object GpuOverrides extends Logging { val checks = in.left.dataType match { case _: MapType => // Match exactly with the checks for GetMapValue - ExprChecks.binaryProject(TypeSig.STRING, TypeSig.all, - ("map", TypeSig.MAP.nested(TypeSig.STRING), TypeSig.MAP.nested(TypeSig.all)), - ("key", TypeSig.lit(TypeEnum.STRING), TypeSig.all)) + ExprChecks.binaryProject( + (TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.NULL + + TypeSig.DECIMAL_128 + TypeSig.MAP).nested(), + TypeSig.all, + ("map", + TypeSig.MAP.nested(TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.MAP), + TypeSig.MAP.nested(TypeSig.all)), + ("key", TypeSig.commonCudfTypesLit() + TypeSig.lit(TypeEnum.DECIMAL), TypeSig.all)) case _: ArrayType => // Match exactly with the checks for GetArrayItem ExprChecks.binaryProject( @@ -2083,7 +2118,7 @@ object GpuOverrides extends Logging { ("array", TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.MAP), TypeSig.ARRAY.nested(TypeSig.all)), - ("ordinal", TypeSig.lit(TypeEnum.INT), TypeSig.INT)) + ("ordinal", TypeSig.INT, TypeSig.INT)) case _ => throw new IllegalStateException("Only Array or Map is supported as input.") } checks.tag(this) @@ -2111,6 +2146,13 @@ object GpuOverrides extends Logging { TypeSig.MAP.nested(TypeSig.all)), (in, conf, p, r) => new UnaryExprMeta[MapValues](in, conf, p, r) { }), + expr[StringToMap]( + "Creates a map after splitting the input string into pairs of key-value strings", + ExprChecks.projectOnly(TypeSig.MAP.nested(TypeSig.STRING), TypeSig.MAP.nested(TypeSig.STRING), + Seq(ParamCheck("str", TypeSig.STRING, TypeSig.STRING), + ParamCheck("pairDelim", TypeSig.lit(TypeEnum.STRING), TypeSig.lit(TypeEnum.STRING)), + ParamCheck("keyValueDelim", TypeSig.lit(TypeEnum.STRING), TypeSig.lit(TypeEnum.STRING)))), + (in, conf, p, r) => new GpuStringToMapMeta(in, conf, p, r)), expr[ArrayMin]( "Returns the minimum value in the array", ExprChecks.unaryProject( @@ -2182,10 +2224,11 @@ object GpuOverrides extends Logging { expr[SortArray]( "Returns a sorted array with the input array and the ascending / descending order", ExprChecks.binaryProject( - TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128), + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + + TypeSig.STRUCT), TypeSig.ARRAY.nested(TypeSig.all), ("array", TypeSig.ARRAY.nested( - TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128), + TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.STRUCT), TypeSig.ARRAY.nested(TypeSig.all)), ("ascendingOrder", TypeSig.lit(TypeEnum.BOOLEAN), TypeSig.lit(TypeEnum.BOOLEAN))), (sortExpression, conf, p, r) => new BinaryExprMeta[SortArray](sortExpression, conf, p, r) { @@ -2257,6 +2300,17 @@ object GpuOverrides extends Logging { TypeSig.all))), (in, conf, p, r) => new ExprMeta[ArrayTransform](in, conf, p, r) { }), + expr[ArrayExists]( + "Return true if any element satisfies the predicate LambdaFunction", + ExprChecks.projectOnly(TypeSig.BOOLEAN, TypeSig.BOOLEAN, + Seq( + ParamCheck("argument", + TypeSig.ARRAY.nested(TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL + + TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.MAP), + TypeSig.ARRAY.nested(TypeSig.all)), + ParamCheck("function", TypeSig.BOOLEAN, TypeSig.BOOLEAN))), + (in, conf, p, r) => new ExprMeta[ArrayExists](in, conf, p, r) { + }), expr[StringLocate]( "Substring search operator", ExprChecks.projectOnly(TypeSig.INT, TypeSig.INT, @@ -2390,24 +2444,19 @@ object GpuOverrides extends Logging { (a, conf, p, r) => new BinaryExprMeta[Like](a, conf, p, r) { }), expr[RLike]( - "RLike", + "Regular expression version of Like", ExprChecks.binaryProject(TypeSig.BOOLEAN, TypeSig.BOOLEAN, ("str", TypeSig.STRING, TypeSig.STRING), ("regexp", TypeSig.lit(TypeEnum.STRING), TypeSig.STRING)), - (a, conf, p, r) => new GpuRLikeMeta(a, conf, p, r)).disabledByDefault( - "the implementation is not 100% compatible. " + - "See the compatibility guide for more information."), + (a, conf, p, r) => new GpuRLikeMeta(a, conf, p, r)), expr[RegExpExtract]( - "RegExpExtract", + "Extract a specific group identified by a regular expression", ExprChecks.projectOnly(TypeSig.STRING, TypeSig.STRING, Seq(ParamCheck("str", TypeSig.STRING, TypeSig.STRING), ParamCheck("regexp", TypeSig.lit(TypeEnum.STRING), TypeSig.STRING), ParamCheck("idx", TypeSig.lit(TypeEnum.INT), TypeSig.lit(TypeEnum.INT)))), - (a, conf, p, r) => new GpuRegExpExtractMeta(a, conf, p, r)) - .disabledByDefault( - "the implementation is not 100% compatible. " + - "See the compatibility guide for more information."), + (a, conf, p, r) => new GpuRegExpExtractMeta(a, conf, p, r)), expr[Length]( "String character length or binary byte length", ExprChecks.unaryProject(TypeSig.INT, TypeSig.INT, @@ -2475,6 +2524,7 @@ object GpuOverrides extends Logging { TypeSig.all))), (a, conf, p, r) => new ReplicateRowsExprMeta[ReplicateRows](a, conf, p, r) { }), + // spark 2.x CollectList and CollectSet use TypedImperative which isn't in 2.x expr[StddevPop]( "Aggregation computing population standard deviation", ExprChecks.groupByOnly( @@ -2560,13 +2610,15 @@ object GpuOverrides extends Logging { (a, conf, p, r) => new BinaryExprMeta[GetJsonObject](a, conf, p, r) { } ), - expr[ScalarSubquery]( + expr[org.apache.spark.sql.execution.ScalarSubquery]( "Subquery that will return only one row and one column", ExprChecks.projectOnly( - TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128, - TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128, + (TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + + TypeSig.ARRAY + TypeSig.MAP + TypeSig.STRUCT).nested(), + TypeSig.all, Nil, None), - (a, conf, p, r) => new ExprMeta[ScalarSubquery](a, conf, p, r) { + (a, conf, p, r) => + new ExprMeta[org.apache.spark.sql.execution.ScalarSubquery](a, conf, p, r) { }), expr[CreateMap]( desc = "Create a map", @@ -2584,12 +2636,40 @@ object GpuOverrides extends Logging { TypeSig.DATE)), Some(RepeatingParamCheck("step", TypeSig.integral, TypeSig.integral + TypeSig.CALENDAR))), (a, conf, p, r) => new GpuSequenceMeta(a, conf, p, r) - ) + ), + expr[BitLength]( + "The bit length of string data", + ExprChecks.unaryProject( + TypeSig.INT, TypeSig.INT, + TypeSig.STRING, TypeSig.STRING + TypeSig.BINARY), + (a, conf, p, r) => new UnaryExprMeta[BitLength](a, conf, p, r) { + }), + expr[OctetLength]( + "The byte length of string data", + ExprChecks.unaryProject( + TypeSig.INT, TypeSig.INT, + TypeSig.STRING, TypeSig.STRING + TypeSig.BINARY), + (a, conf, p, r) => new UnaryExprMeta[OctetLength](a, conf, p, r) { + }), + expr[GetArrayStructFields]( + "Extracts the `ordinal`-th fields of all array elements for the data with the type of" + + " array of struct", + ExprChecks.unaryProject( + TypeSig.ARRAY.nested(TypeSig.commonCudfTypesWithNested), + TypeSig.ARRAY.nested(TypeSig.all), + // we should allow all supported types for the children types signature of the nested + // struct, even only a struct child is allowed for the array here. Since TypeSig supports + // only one level signature for nested type. + TypeSig.ARRAY.nested(TypeSig.commonCudfTypesWithNested), + TypeSig.ARRAY.nested(TypeSig.all)), + (e, conf, p, r) => new GpuGetArrayStructFieldsMeta(e, conf, p, r) + ) ).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap // Shim expressions should be last to allow overrides with shim-specific versions val expressions: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = - commonExpressions ++ GpuHiveOverrides.exprs ++ ShimGpuOverrides.shimExpressions + commonExpressions ++ TimeStamp.getExprs ++ GpuHiveOverrides.exprs ++ + ShimGpuOverrides.shimExpressions def wrapPart[INPUT <: Partitioning]( part: INPUT, @@ -2855,7 +2935,10 @@ object GpuOverrides extends Logging { TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.MAP).nested(), TypeSig.all, Map("partitionSpec" -> - InputCheck(TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_64, TypeSig.all))), + InputCheck( + TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128 + + TypeSig.STRUCT.nested(TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128), + TypeSig.all))), (windowOp, conf, p, r) => new GpuWindowExecMeta(windowOp, conf, p, r) ), diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index 9b54774f174..92010fda514 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -338,6 +338,13 @@ object RapidsConf { .checkValue(v => v >= 0 && v <= 1, "The fraction value must be in [0, 1].") .createWithDefault(1) + val RMM_EXACT_ALLOC = conf("spark.rapids.memory.gpu.allocSize") + .doc("The exact size in byte that RMM should allocate. This is intended to only be " + + "used for testing.") + .internal() // If this becomes public we need to add in checks for the value when it is used. + .bytesConf(ByteUnit.BYTE) + .createOptional + val RMM_ALLOC_MAX_FRACTION = conf(RMM_ALLOC_MAX_FRACTION_KEY) .doc("The fraction of total GPU memory that limits the maximum size of the RMM pool. " + s"The value must be greater than or equal to the setting for $RMM_ALLOC_FRACTION. " + @@ -404,8 +411,7 @@ object RapidsConf { "\"ASYNC\", and \"NONE\". With \"DEFAULT\", the RMM pool allocator is used; with " + "\"ARENA\", the RMM arena allocator is used; with \"ASYNC\", the new CUDA stream-ordered " + "memory allocator in CUDA 11.2+ is used. If set to \"NONE\", pooling is disabled and RMM " + - "just passes through to CUDA memory allocation directly. Note: \"ARENA\" is the " + - "recommended pool allocator if CUDF is built with Per-Thread Default Stream (PTDS).") + "just passes through to CUDA memory allocation directly.") .stringConf .createWithDefault("ARENA") @@ -470,6 +476,16 @@ object RapidsConf { .booleanConf .createWithDefault(false) + val SHUFFLED_HASH_JOIN_OPTIMIZE_SHUFFLE = + conf("spark.rapids.sql.shuffledHashJoin.optimizeShuffle") + .doc("Enable or disable an optimization where shuffled build side batches are kept " + + "on the host while the first stream batch is loaded onto the GPU. The optimization " + + "increases off-heap host memory usage to avoid holding onto the GPU semaphore while " + + "waiting for stream side IO.") + .internal() + .booleanConf + .createWithDefault(true) + val STABLE_SORT = conf("spark.rapids.sql.stableSort.enabled") .doc("Enable or disable stable sorting. Apache Spark's sorting is typically a stable " + "sort, but sort stability cannot be guaranteed in distributed work loads because the " + @@ -686,6 +702,11 @@ object RapidsConf { .booleanConf .createWithDefault(true) + val ENABLE_EXISTENCE_JOIN = conf("spark.rapids.sql.join.existence.enabled") + .doc("When set to true existence joins are enabled on the GPU") + .booleanConf + .createWithDefault(true) + val ENABLE_PROJECT_AST = conf("spark.rapids.sql.projectAstEnabled") .doc("Enable project operations to use cudf AST expressions when possible.") .internal() @@ -874,52 +895,6 @@ object RapidsConf { .booleanConf .createWithDefault(false) - val ENABLE_READ_CSV_DATES = conf("spark.rapids.sql.csv.read.date.enabled") - .doc("Parsing invalid CSV dates produces different results from Spark") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_BOOLS = conf("spark.rapids.sql.csv.read.bool.enabled") - .doc("Parsing an invalid CSV boolean value produces true instead of null") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_BYTES = conf("spark.rapids.sql.csv.read.byte.enabled") - .doc("Parsing CSV bytes is much more lenient and will return 0 for some " + - "malformed values instead of null") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_SHORTS = conf("spark.rapids.sql.csv.read.short.enabled") - .doc("Parsing CSV shorts is much more lenient and will return 0 for some " + - "malformed values instead of null") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_INTEGERS = conf("spark.rapids.sql.csv.read.integer.enabled") - .doc("Parsing CSV integers is much more lenient and will return 0 for some " + - "malformed values instead of null") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_LONGS = conf("spark.rapids.sql.csv.read.long.enabled") - .doc("Parsing CSV longs is much more lenient and will return 0 for some " + - "malformed values instead of null") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_FLOATS = conf("spark.rapids.sql.csv.read.float.enabled") - .doc("Parsing CSV floats has some issues at the min and max values for floating" + - "point numbers and can be more lenient on parsing inf and -inf values") - .booleanConf - .createWithDefault(false) - - val ENABLE_READ_CSV_DOUBLES = conf("spark.rapids.sql.csv.read.double.enabled") - .doc("Parsing CSV double has some issues at the min and max values for floating" + - "point numbers and can be more lenient on parsing inf and -inf values") - .booleanConf - .createWithDefault(false) - val ENABLE_JSON = conf("spark.rapids.sql.format.json.enabled") .doc("When set to true enables all json input and output acceleration. " + "(only input is currently supported anyways)") @@ -931,6 +906,17 @@ object RapidsConf { .booleanConf .createWithDefault(false) + val ENABLE_AVRO = conf("spark.rapids.sql.format.avro.enabled") + .doc("When set to true enables all avro input and output acceleration. " + + "(only input is currently supported anyways)") + .booleanConf + .createWithDefault(false) + + val ENABLE_AVRO_READ = conf("spark.rapids.sql.format.avro.read.enabled") + .doc("When set to true enables avro input acceleration") + .booleanConf + .createWithDefault(false) + val ENABLE_RANGE_WINDOW_BYTES = conf("spark.rapids.sql.window.range.byte.enabled") .doc("When the order-by column of a range based window is byte type and " + "the range boundary calculated for a value has overflow, CPU and GPU will get " + @@ -963,6 +949,14 @@ object RapidsConf { .booleanConf .createWithDefault(true) + val ENABLE_REGEXP = conf("spark.rapids.sql.regexp.enabled") + .doc("Specifies whether regular expressions should be evaluated on GPU. Complex expressions " + + "can cause out of memory issues so this is disabled by default. Setting this config to " + + "true will make supported regular expressions run on the GPU. See the compatibility " + + "guide for more information about which regular expressions are supported on the GPU.") + .booleanConf + .createWithDefault(false) + // INTERNAL TEST AND DEBUG CONFIGS val TEST_CONF = conf("spark.rapids.sql.test.enabled") @@ -1197,9 +1191,9 @@ object RapidsConf { .internal() .doc("Overrides the automatic Spark shim detection logic and forces a specific shims " + "provider class to be used. Set to the fully qualified shims provider class to use. " + - "If you are using a custom Spark version such as Spark 3.0.1.0 then this can be used to " + - "specify the shims provider that matches the base Spark version of Spark 3.0.1, i.e.: " + - "com.nvidia.spark.rapids.shims.spark301.SparkShimServiceProvider. If you modified Spark " + + "If you are using a custom Spark version such as Spark 3.1.1.0 then this can be used to " + + "specify the shims provider that matches the base Spark version of Spark 3.1.1, i.e.: " + + "com.nvidia.spark.rapids.shims.spark311.SparkShimServiceProvider. If you modified Spark " + "then there is no guarantee the RAPIDS Accelerator will function properly." + "When tested in a combined jar with other Shims, it's expected that the provided " + "implementation follows the same convention as existing Spark shims. If its class" + @@ -1479,6 +1473,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val exportColumnarRdd: Boolean = get(EXPORT_COLUMNAR_RDD) + lazy val shuffledHashJoinOptimizeShuffle: Boolean = get(SHUFFLED_HASH_JOIN_OPTIMIZE_SHUFFLE) + lazy val stableSort: Boolean = get(STABLE_SORT) lazy val isIncompatEnabled: Boolean = get(INCOMPATIBLE_OPS) @@ -1507,7 +1503,13 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val isPooledMemEnabled: Boolean = get(POOLED_MEM) - lazy val rmmPool: String = get(RMM_POOL) + // Spark 2.x doesn't have access to Cuda in CUDF so just allow + lazy val rmmPool: String = { + var pool = get(RMM_POOL) + pool + } + + lazy val rmmExactAlloc: Option[Long] = get(RMM_EXACT_ALLOC) lazy val rmmAllocFraction: Double = get(RMM_ALLOC_FRACTION) @@ -1571,6 +1573,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val areLeftAntiJoinsEnabled: Boolean = get(ENABLE_LEFT_ANTI_JOIN) + lazy val areExistenceJoinsEnabled: Boolean = get(ENABLE_EXISTENCE_JOIN) + lazy val isCastDecimalToFloatEnabled: Boolean = get(ENABLE_CAST_DECIMAL_TO_FLOAT) lazy val isCastFloatToDecimalEnabled: Boolean = get(ENABLE_CAST_FLOAT_TO_DECIMAL) @@ -1587,22 +1591,6 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val isCsvTimestampReadEnabled: Boolean = get(ENABLE_CSV_TIMESTAMPS) - lazy val isCsvDateReadEnabled: Boolean = get(ENABLE_READ_CSV_DATES) - - lazy val isCsvBoolReadEnabled: Boolean = get(ENABLE_READ_CSV_BOOLS) - - lazy val isCsvByteReadEnabled: Boolean = get(ENABLE_READ_CSV_BYTES) - - lazy val isCsvShortReadEnabled: Boolean = get(ENABLE_READ_CSV_SHORTS) - - lazy val isCsvIntReadEnabled: Boolean = get(ENABLE_READ_CSV_INTEGERS) - - lazy val isCsvLongReadEnabled: Boolean = get(ENABLE_READ_CSV_LONGS) - - lazy val isCsvFloatReadEnabled: Boolean = get(ENABLE_READ_CSV_FLOATS) - - lazy val isCsvDoubleReadEnabled: Boolean = get(ENABLE_READ_CSV_DOUBLES) - lazy val isCastDecimalToStringEnabled: Boolean = get(ENABLE_CAST_DECIMAL_TO_STRING) lazy val isProjectAstEnabled: Boolean = get(ENABLE_PROJECT_AST) @@ -1661,6 +1649,10 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val isJsonReadEnabled: Boolean = get(ENABLE_JSON_READ) + lazy val isAvroEnabled: Boolean = get(ENABLE_AVRO) + + lazy val isAvroReadEnabled: Boolean = get(ENABLE_AVRO_READ) + lazy val shuffleManagerEnabled: Boolean = get(SHUFFLE_MANAGER_ENABLED) lazy val shuffleTransportEnabled: Boolean = get(SHUFFLE_TRANSPORT_ENABLE) @@ -1759,6 +1751,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging { lazy val isRangeWindowLongEnabled: Boolean = get(ENABLE_RANGE_WINDOW_LONG) + lazy val isRegExpEnabled: Boolean = get(ENABLE_REGEXP) + lazy val getSparkGpuResourceName: String = get(SPARK_GPU_RESOURCE_NAME) lazy val isCpuBasedUDFEnabled: Boolean = get(ENABLE_CPU_BASED_UDF) diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala index 5f4ee4d54d0..705faa446a8 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala @@ -277,14 +277,14 @@ class RegexParser(pattern: String) { // word boundaries consumeExpected(ch) RegexEscaped(ch) - case '[' | '\\' | '^' | '$' | '.' | '⎮' | '?' | '*' | '+' | '(' | ')' | '{' | '}' => + case '[' | '\\' | '^' | '$' | '.' | '|' | '?' | '*' | '+' | '(' | ')' | '{' | '}' => // escaped metacharacter consumeExpected(ch) RegexEscaped(ch) case 'x' => consumeExpected(ch) parseHexDigit - case _ if Character.isDigit(ch) => + case '0' => parseOctalDigit case other => throw new RegexUnsupportedException( @@ -302,13 +302,18 @@ class RegexParser(pattern: String) { // \x{h...h} The character with hexadecimal value 0xh...h // (Character.MIN_CODE_POINT <= 0xh...h <= Character.MAX_CODE_POINT) + val varHex = pattern.charAt(pos) == '{' + if (varHex) { + consumeExpected('{') + } val start = pos while (!eof() && isHexDigit(pattern.charAt(pos))) { pos += 1 } val hexDigit = pattern.substring(start, pos) - - if (hexDigit.length < 2) { + if (varHex) { + consumeExpected('}') + } else if (hexDigit.length != 2) { throw new RegexUnsupportedException(s"Invalid hex digit: $hexDigit") } @@ -337,7 +342,12 @@ class RegexParser(pattern: String) { if (pos + 1 < pattern.length && isOctalDigit(pattern.charAt(pos + 1))) { if (pos + 2 < pattern.length && isOctalDigit(pattern.charAt(pos + 2)) && pattern.charAt(pos) <= '3') { - parseOctalDigits(3) + if (pos + 3 < pattern.length && isOctalDigit(pattern.charAt(pos + 3)) + && pattern.charAt(pos+1) <= '3' && pattern.charAt(pos) == '0') { + parseOctalDigits(4) + } else { + parseOctalDigits(3) + } } else { parseOctalDigits(2) } @@ -404,6 +414,35 @@ class RegexParser(pattern: String) { } +object RegexParser { + private val regexpChars = Set('\u0000', '\\', '.', '^', '$', '\f') + + def isRegExpString(s: String): Boolean = { + + def isRegExpString(ast: RegexAST): Boolean = ast match { + case RegexChar(ch) => regexpChars.contains(ch) + case RegexEscaped(_) => true + case RegexSequence(parts) => parts.exists(isRegExpString) + case _ => true + } + + try { + val parser = new RegexParser(s) + val ast = parser.parse() + isRegExpString(ast) + } catch { + case _: RegexUnsupportedException => + // if we cannot parse it then assume that it might be valid regexp + true + } + } +} + +sealed trait RegexMode +object RegexFindMode extends RegexMode +object RegexReplaceMode extends RegexMode +object RegexSplitMode extends RegexMode + /** * Transpile Java/Spark regular expression to a format that cuDF supports, or throw an exception * if this is not possible. @@ -411,7 +450,8 @@ class RegexParser(pattern: String) { * @param replace True if performing a replacement (regexp_replace), false * if matching only (rlike) */ -class CudfRegexTranspiler(replace: Boolean) { +class CudfRegexTranspiler(mode: RegexMode) { + private val regexMetaChars = ".$^[]\\|?*+(){}" // cuDF throws a "nothing to repeat" exception for many of the edge cases that are // rejected by the transpiler @@ -433,6 +473,69 @@ class CudfRegexTranspiler(replace: Boolean) { cudfRegex.toRegexString } + def transpileToSplittableString(e: RegexAST): Option[String] = { + e match { + case RegexEscaped(ch) if regexMetaChars.contains(ch) => Some(ch.toString) + case RegexChar(ch) if !regexMetaChars.contains(ch) => Some(ch.toString) + case RegexSequence(parts) => + parts.foldLeft[Option[String]](Some("")) { (all, x) => + all match { + case Some(current) => + transpileToSplittableString(x) match { + case Some(y) => Some(current + y) + case _ => None + } + case _ => None + } + } + case _ => None + } + } + + def transpileToSplittableString(pattern: String): Option[String] = { + try { + val regex = new RegexParser(pattern).parse() + transpileToSplittableString(regex) + } catch { + // treat as regex if we can't parse it + case _: RegexUnsupportedException => + None + } + } + + private def isRepetition(e: RegexAST): Boolean = { + e match { + case RegexRepetition(_, _) => true + case RegexGroup(_, term) => isRepetition(term) + case RegexSequence(parts) if parts.nonEmpty => isRepetition(parts.last) + case _ => false + } + } + + private def isSupportedRepetitionBase(e: RegexAST): Boolean = { + e match { + case RegexEscaped(ch) if ch != 'd' && ch != 'w' => // example: "\B?" + false + + case RegexChar(a) if "$^".contains(a) => + // example: "$*" + false + + case RegexRepetition(_, _) => + // example: "a*+" + false + + case RegexSequence(parts) => + parts.forall(isSupportedRepetitionBase) + + case RegexGroup(_, term) => + isSupportedRepetitionBase(term) + + case _ => true + } + } + + private def rewrite(regex: RegexAST): RegexAST = { regex match { @@ -443,19 +546,33 @@ class CudfRegexTranspiler(replace: Boolean) { case '$' => // see https://github.com/NVIDIA/spark-rapids/issues/4533 throw new RegexUnsupportedException("line anchor $ is not supported") + case '^' if mode == RegexSplitMode => + throw new RegexUnsupportedException("line anchor ^ is not supported in split mode") case _ => regex } - case RegexOctalChar(_) => - // see https://github.com/NVIDIA/spark-rapids/issues/4288 - throw new RegexUnsupportedException( - s"cuDF does not support octal digits consistently with Spark") + case RegexOctalChar(digits) => + val octal = if (digits.charAt(0) == '0' && digits.length == 4) { + digits.substring(1) + } else { + digits + } + if (Integer.parseInt(octal, 8) >= 128) { + // see https://github.com/NVIDIA/spark-rapids/issues/4746 + throw new RegexUnsupportedException( + "cuDF does not support octal digits 0o177 < n <= 0o377") + } + RegexOctalChar(octal) - case RegexHexDigit(_) => - // see https://github.com/NVIDIA/spark-rapids/issues/4486 - throw new RegexUnsupportedException( - s"cuDF does not support hex digits consistently with Spark") + case RegexHexDigit(digits) => + val codePoint = Integer.parseInt(digits, 16) + if (codePoint >= 128) { + // see https://github.com/NVIDIA/spark-rapids/issues/4866 + throw new RegexUnsupportedException( + "cuDF does not support hex digits > 0x7F") + } + RegexHexDigit(String.format("%02x", Int.box(codePoint))) case RegexEscaped(ch) => ch match { case 'D' => @@ -470,8 +587,14 @@ class CudfRegexTranspiler(replace: Boolean) { case 's' | 'S' => // see https://github.com/NVIDIA/spark-rapids/issues/4528 throw new RegexUnsupportedException("whitespace classes are not supported") + case 'A' if mode == RegexSplitMode => + throw new RegexUnsupportedException("string anchor \\A is not supported in split mode") + case 'Z' if mode == RegexSplitMode => + throw new RegexUnsupportedException("string anchor \\Z is not supported in split mode") + case 'z' if mode == RegexSplitMode => + throw new RegexUnsupportedException("string anchor \\z is not supported in split mode") case 'z' => - if (replace) { + if (mode == RegexReplaceMode) { // see https://github.com/NVIDIA/spark-rapids/issues/4425 throw new RegexUnsupportedException( "string anchor \\z is not supported in replace mode") @@ -495,6 +618,17 @@ class CudfRegexTranspiler(replace: Boolean) { // - "[a[]" should match the literal characters "a" and "[" // - "[a-b[c-d]]" is supported by Java but not cuDF throw new RegexUnsupportedException("nested character classes are not supported") + case RegexEscaped(ch) if ch == '0' => + // see https://github.com/NVIDIA/spark-rapids/issues/4862 + // examples + // - "[\02] should match the character with code point 2" + throw new RegexUnsupportedException( + "cuDF does not support octal digits in character classes") + case RegexEscaped(ch) if ch == 'x' => + // examples + // - "[\x02] should match the character with code point 2" + throw new RegexUnsupportedException( + "cuDF does not support hex digits in character classes") case _ => } val components: Seq[RegexCharacterClassComponent] = characters @@ -566,7 +700,7 @@ class CudfRegexTranspiler(replace: Boolean) { RegexSequence(parts.map(rewrite)) case RegexRepetition(base, quantifier) => (base, quantifier) match { - case (_, SimpleQuantifier(ch)) if replace && "?*".contains(ch) => + case (_, SimpleQuantifier(ch)) if mode == RegexReplaceMode && "?*".contains(ch) => // example: pattern " ?", input "] b[", replace with "X": // java: X]XXbX[X // cuDF: XXXX] b[ @@ -574,20 +708,43 @@ class CudfRegexTranspiler(replace: Boolean) { throw new RegexUnsupportedException( "regexp_replace on GPU does not support repetition with ? or *") - case (RegexEscaped(ch), _) if ch != 'd' && ch != 'w' => - // example: "\B?" - throw new RegexUnsupportedException(nothingToRepeat) + case (_, SimpleQuantifier(ch)) if mode == RegexSplitMode && "?*".contains(ch) => + // example: pattern " ?", input "] b[", replace with "X": + // java: X]XXbX[X + // cuDF: XXXX] b[ + // see https://github.com/NVIDIA/spark-rapids/issues/4884 + throw new RegexUnsupportedException( + "regexp_split on GPU does not support repetition with ? or * consistently with Spark") - case (RegexChar(a), _) if "$^".contains(a) => - // example: "$*" - throw new RegexUnsupportedException(nothingToRepeat) + case (_, QuantifierVariableLength(0, _)) if mode == RegexReplaceMode => + // see https://github.com/NVIDIA/spark-rapids/issues/4468 + throw new RegexUnsupportedException( + "regexp_replace on GPU does not support repetition with {0,} or {0,n}") - case (RegexRepetition(_, _), _) => - // example: "a*+" - throw new RegexUnsupportedException(nothingToRepeat) + case (_, QuantifierVariableLength(0, _)) if mode == RegexSplitMode => + // see https://github.com/NVIDIA/spark-rapids/issues/4884 + throw new RegexUnsupportedException( + "regexp_split on GPU does not support repetition with {0,} or {0,n} " + + "consistently with Spark") - case _ => + case (_, QuantifierFixedLength(0)) + if mode != RegexFindMode => + throw new RegexUnsupportedException( + "regex_replace and regex_split on GPU do not support repetition with {0}") + + case (RegexGroup(_, term), SimpleQuantifier(ch)) + if "+*".contains(ch) && !isSupportedRepetitionBase(term) => + throw new RegexUnsupportedException(nothingToRepeat) + case (RegexGroup(_, term), QuantifierVariableLength(_, None)) + if !isSupportedRepetitionBase(term) => + // specifically this variable length repetition: \A{2,} + throw new RegexUnsupportedException(nothingToRepeat) + case (RegexGroup(_, _), SimpleQuantifier(ch)) if ch == '?' => + RegexRepetition(rewrite(base), quantifier) + case _ if isSupportedRepetitionBase(base) => RegexRepetition(rewrite(base), quantifier) + case _ => + throw new RegexUnsupportedException(nothingToRepeat) } @@ -596,14 +753,6 @@ class CudfRegexTranspiler(replace: Boolean) { val rr = rewrite(r) // cuDF does not support repetition on one side of a choice, such as "a*|a" - def isRepetition(e: RegexAST): Boolean = { - e match { - case RegexRepetition(_, _) => true - case RegexGroup(_, term) => isRepetition(term) - case RegexSequence(parts) if parts.nonEmpty => isRepetition(parts.last) - case _ => false - } - } if (isRepetition(ll) || isRepetition(rr)) { throw new RegexUnsupportedException(nothingToRepeat) } @@ -613,8 +762,9 @@ class CudfRegexTranspiler(replace: Boolean) { def endsWithLineAnchor(e: RegexAST): Boolean = { e match { case RegexSequence(parts) if parts.nonEmpty => - isBeginOrEndLineAnchor(parts.last) - case _ => false + endsWithLineAnchor(parts.last) + case RegexEscaped('A') => true + case _ => isBeginOrEndLineAnchor(e) } } if (endsWithLineAnchor(ll) || endsWithLineAnchor(rr)) { @@ -708,7 +858,13 @@ sealed trait RegexCharacterClassComponent extends RegexAST sealed case class RegexHexDigit(a: String) extends RegexCharacterClassComponent { override def children(): Seq[RegexAST] = Seq.empty - override def toRegexString: String = s"\\x$a" + override def toRegexString: String = { + if (a.length == 2) { + s"\\x$a" + } else { + s"\\x{$a}" + } + } } sealed case class RegexOctalChar(a: String) extends RegexCharacterClassComponent { @@ -716,9 +872,9 @@ sealed case class RegexOctalChar(a: String) extends RegexCharacterClassComponent override def toRegexString: String = s"\\$a" } -sealed case class RegexChar(a: Char) extends RegexCharacterClassComponent { +sealed case class RegexChar(ch: Char) extends RegexCharacterClassComponent { override def children(): Seq[RegexAST] = Seq.empty - override def toRegexString: String = s"$a" + override def toRegexString: String = s"$ch" } sealed case class RegexEscaped(a: Char) extends RegexCharacterClassComponent{ diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala index 71621d3fa76..e5967cc4575 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimGpuOverrides.scala @@ -43,7 +43,7 @@ object ShimGpuOverrides extends Logging { "Convert a column of one type of data into another type", new CastChecks(), (cast, conf, p, r) => new CastExprMeta[Cast](cast, false, conf, p, r, - doFloatToIntCheck = false, stringToAnsiDate = false)), + doFloatToIntCheck = true, stringToAnsiDate = false)), GpuOverrides.expr[Average]( "Average aggregate operator", ExprChecks.fullAgg( @@ -67,7 +67,7 @@ object ShimGpuOverrides extends Logging { s"a precision large than 23. The current precision is ${dt.precision}") } else { logWarning("Decimal overflow guarantees disabled for " + - s"Average(${a.child.dataType}) produces $dt with an " + + s"Average(${a.child.dataType}) produces ${dt} with an " + s"intermediate precision of ${dt.precision + 15}") } } @@ -85,32 +85,15 @@ object ShimGpuOverrides extends Logging { (a, conf, p, r) => new UnaryAstExprMeta[Abs](a, conf, p, r) { }), GpuOverrides.expr[RegExpReplace]( - "RegExpReplace support for string literal input patterns", + "String replace using a regular expression pattern", ExprChecks.projectOnly(TypeSig.STRING, TypeSig.STRING, Seq(ParamCheck("str", TypeSig.STRING, TypeSig.STRING), ParamCheck("regex", TypeSig.lit(TypeEnum.STRING), TypeSig.STRING), - ParamCheck("rep", TypeSig.lit(TypeEnum.STRING), TypeSig.STRING))), - (a, conf, p, r) => new GpuRegExpReplaceMeta(a, conf, p, r)).disabledByDefault( - "the implementation is not 100% compatible. " + - "See the compatibility guide for more information."), - GpuOverrides.expr[TimeSub]( - "Subtracts interval from timestamp", - ExprChecks.binaryProject(TypeSig.TIMESTAMP, TypeSig.TIMESTAMP, - ("start", TypeSig.TIMESTAMP, TypeSig.TIMESTAMP), - ("interval", TypeSig.lit(TypeEnum.CALENDAR) - .withPsNote(TypeEnum.CALENDAR, "months not supported"), TypeSig.CALENDAR)), - (timeSub, conf, p, r) => new BinaryExprMeta[TimeSub](timeSub, conf, p, r) { - override def tagExprForGpu(): Unit = { - timeSub.interval match { - case Literal(intvl: CalendarInterval, DataTypes.CalendarIntervalType) => - if (intvl.months != 0) { - willNotWorkOnGpu("interval months isn't supported") - } - case _ => - } - checkTimeZoneId(timeSub.timeZoneId) - } - }), + ParamCheck("rep", TypeSig.lit(TypeEnum.STRING), TypeSig.STRING), + ParamCheck("pos", TypeSig.lit(TypeEnum.INT) + .withPsNote(TypeEnum.INT, "only a value of 1 is supported"), + TypeSig.lit(TypeEnum.INT)))), + (a, conf, p, r) => new GpuRegExpReplaceMeta(a, conf, p, r)), GpuOverrides.expr[ScalaUDF]( "User Defined Function, the UDF can choose to implement a RAPIDS accelerated interface " + "to get better performance.", diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala index 8e44ba50b58..0da3a6834e0 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala @@ -527,6 +527,30 @@ object TypeSig { def lit(dataType: TypeEnum.Value): TypeSig = TypeSig.none.withLit(dataType) + /** + * Create a TypeSig that only supports literals of certain given types. + */ + def lit(dataTypes: TypeEnum.ValueSet): TypeSig = + new TypeSig(dataTypes) + + /** + * Create a TypeSig that supports only literals of common primitive CUDF types. + */ + def commonCudfTypesLit(): TypeSig = { + lit(TypeEnum.ValueSet( + TypeEnum.BOOLEAN, + TypeEnum.BYTE, + TypeEnum.SHORT, + TypeEnum.INT, + TypeEnum.LONG, + TypeEnum.FLOAT, + TypeEnum.DOUBLE, + TypeEnum.DATE, + TypeEnum.TIMESTAMP, + TypeEnum.STRING + )) + } + /** * Create a TypeSig that has partial support for the given type. */ @@ -668,6 +692,12 @@ object TypeSig { TIMESTAMP + STRING + DECIMAL_128 + NULL + BINARY + CALENDAR + ARRAY + STRUCT + UDT).nested() + /** + * commonCudfTypes plus decimal, null and nested types. + */ + val commonCudfTypesWithNested: TypeSig = (commonCudfTypes + DECIMAL_128 + NULL + + ARRAY + STRUCT + MAP).nested() + /** * Different types of Pandas UDF support different sets of output type. Please refer to * https://github.com/apache/spark/blob/master/python/pyspark/sql/udf.py#L98 @@ -884,7 +914,7 @@ object FileFormatChecks { * The namedChecks map can be used to provide checks for specific groups of expressions. */ class ExecChecks private( - check: TypeSig, + val check: TypeSig, sparkSig: TypeSig, val namedChecks: Map[String, InputCheck], override val shown: Boolean = true) @@ -1097,7 +1127,7 @@ object CaseWhenCheck extends ExprChecks { } /** - * This is specific to WidowSpec, because it does not follow the typical parameter convention. + * This is specific to WindowSpec, because it does not follow the typical parameter convention. */ object WindowSpecCheck extends ExprChecks { val check: TypeSig = @@ -1281,7 +1311,8 @@ class CastChecks extends ExprChecks { val mapChecks: TypeSig = MAP.nested(commonCudfTypes + DECIMAL_128 + NULL + ARRAY + BINARY + STRUCT + MAP) + psNote(TypeEnum.MAP, "the map's key and value must also support being cast to the " + - "desired child types") + "desired child types") + + psNote(TypeEnum.STRING, "the map's key and value must also support being cast to string") val sparkMapSig: TypeSig = STRING + MAP.nested(all) val structChecks: TypeSig = psNote(TypeEnum.STRING, "the struct's children must also support " + @@ -1715,14 +1746,12 @@ object SupportedOpsDocs { println() println("# General limitations") println("## `Decimal`") - println("The `Decimal` type in Spark supports a precision") - println("up to 38 digits (128-bits). The RAPIDS Accelerator in most cases stores values up to") - println("64-bits and will support 128-bit in the future. As such the accelerator currently only") - println(s"supports a precision up to ${GpuOverrides.DECIMAL64_MAX_PRECISION} digits. Note that") - println("decimals are disabled by default in the plugin, because it is supported by a relatively") - println("small number of operations presently. This can result in a lot of data movement to and") - println("from the GPU, slowing down processing in some cases.") - println("Result `Decimal` precision and scale follow the same rule as CPU mode in Apache Spark:") + println("The `Decimal` type in Spark supports a precision up to 38 digits (128-bits). ") + println("The RAPIDS Accelerator supports 128-bit starting from version 21.12 and decimals are ") + println("enabled by default.") + println("Please check [Decimal Support](compatibility.md#decimal-support) for more details.") + println() + println("`Decimal` precision and scale follow the same rule as CPU mode in Apache Spark:") println() println("```") println(" * In particular, if we have expressions e1 and e2 with precision/scale p1/s1 and p2/s2") @@ -2117,6 +2146,7 @@ object SupportedOpsForTools { case "parquet" => conf.isParquetEnabled && conf.isParquetReadEnabled case "orc" => conf.isOrcEnabled && conf.isOrcReadEnabled case "json" => conf.isJsonEnabled && conf.isJsonReadEnabled + case "avro" => conf.isAvroEnabled && conf.isAvroReadEnabled case _ => throw new IllegalArgumentException("Format is unknown we need to add it here!") } @@ -2125,15 +2155,7 @@ object SupportedOpsForTools { val readOps = types.map { t => val typeEnabled = if (format.toString.toLowerCase.equals("csv")) { t.toString match { - case "BOOLEAN" => conf.isCsvBoolReadEnabled - case "BYTE" => conf.isCsvByteReadEnabled - case "SHORT" => conf.isCsvShortReadEnabled - case "INT" => conf.isCsvIntReadEnabled - case "LONG" => conf.isCsvLongReadEnabled - case "FLOAT" => conf.isCsvFloatReadEnabled - case "DOUBLE" => conf.isCsvDoubleReadEnabled case "TIMESTAMP" => conf.isCsvTimestampReadEnabled - case "DATE" => conf.isCsvDateReadEnabled case _ => true } } else { diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuCSVScan.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuCSVScan.scala index 02d5bca1656..61c3813d0dd 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuCSVScan.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuCSVScan.scala @@ -21,8 +21,10 @@ import java.nio.charset.StandardCharsets import com.nvidia.spark.rapids._ import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.csv.GpuCsvUtils import org.apache.spark.sql.catalyst.util.PermissiveMode import org.apache.spark.sql.execution.datasources.csv.CSVOptions +import org.apache.spark.sql.rapids.LegacyTimeParserPolicy import org.apache.spark.sql.types._ object GpuCSVScan { @@ -163,50 +165,14 @@ object GpuCSVScan { // parsedOptions.maxColumns was originally a performance optimization but is not used any more if (readSchema.map(_.dataType).contains(DateType)) { - if (!meta.conf.isCsvDateReadEnabled) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading dates. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_DATES} to true.") + if (GpuOverrides.getTimeParserPolicy == LegacyTimeParserPolicy) { + // Spark's CSV parser will parse the string "2020-50-16" to the date 2024/02/16 when + // timeParserPolicy is set to LEGACY mode and we would reject this as an invalid date + // so we fall back to CPU + meta.willNotWorkOnGpu(s"GpuCSVScan does not support timeParserPolicy=LEGACY") } - dateFormatInRead(parsedOptions).foreach { dateFormat => - if (!supportedDateFormats.contains(dateFormat)) { - meta.willNotWorkOnGpu(s"the date format '${dateFormat}' is not supported'") - } - } - } - - if (!meta.conf.isCsvBoolReadEnabled && readSchema.map(_.dataType).contains(BooleanType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading boolean. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_BOOLS} to true.") - } - - if (!meta.conf.isCsvByteReadEnabled && readSchema.map(_.dataType).contains(ByteType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading bytes. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_BYTES} to true.") - } - - if (!meta.conf.isCsvShortReadEnabled && readSchema.map(_.dataType).contains(ShortType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading shorts. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_SHORTS} to true.") - } - - if (!meta.conf.isCsvIntReadEnabled && readSchema.map(_.dataType).contains(IntegerType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading integers. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_INTEGERS} to true.") - } - - if (!meta.conf.isCsvLongReadEnabled && readSchema.map(_.dataType).contains(LongType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading longs. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_LONGS} to true.") - } - - if (!meta.conf.isCsvFloatReadEnabled && readSchema.map(_.dataType).contains(FloatType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading floats. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_FLOATS} to true.") - } - - if (!meta.conf.isCsvDoubleReadEnabled && readSchema.map(_.dataType).contains(DoubleType)) { - meta.willNotWorkOnGpu("CSV reading is not 100% compatible when reading doubles. " + - s"To enable it please set ${RapidsConf.ENABLE_READ_CSV_DOUBLES} to true.") + DateUtils.tagAndGetCudfFormat(meta, + GpuCsvUtils.dateFormatInRead(parsedOptions), parseString = true) } if (readSchema.map(_.dataType).contains(TimestampType)) { diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuRegExpReplaceMeta.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuRegExpReplaceMeta.scala index df07b72fa39..f9ac0c1be55 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuRegExpReplaceMeta.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/GpuRegExpReplaceMeta.scala @@ -38,7 +38,7 @@ class GpuRegExpReplaceMeta( // use GpuStringReplace } else { try { - pattern = Some(new CudfRegexTranspiler(replace = true).transpile(s.toString)) + pattern = Some(new CudfRegexTranspiler(RegexReplaceMode).transpile(s.toString)) } catch { case e: RegexUnsupportedException => willNotWorkOnGpu(e.getMessage) @@ -100,4 +100,10 @@ object GpuRegExpUtils { b.toString } + def tagForRegExpEnabled(meta: ExprMeta[_]): Unit = { + if (!meta.conf.isRegExpEnabled) { + meta.willNotWorkOnGpu(s"regular expression support is disabled. " + + s"Set ${RapidsConf.ENABLE_REGEXP}=true to enable it") + } + } } diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala index 0265874d2a2..2619fd1f91d 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/OffsetWindowFunctionMeta.scala @@ -18,43 +18,58 @@ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids.{BaseExprMeta, DataFromReplacementRule, ExprMeta, GpuOverrides, RapidsConf, RapidsMeta} -import org.apache.spark.sql.catalyst.expressions.{Lag, Lead, Literal, OffsetWindowFunction} +import org.apache.spark.sql.catalyst.expressions.{Expression, Lag, Lead, Literal, OffsetWindowFunction} import org.apache.spark.sql.types.IntegerType +/** + * Spark 3.1.1-specific replacement for com.nvidia.spark.rapids.OffsetWindowFunctionMeta. + * This is required primarily for two reasons: + * 1. com.nvidia.spark.rapids.OffsetWindowFunctionMeta (compiled against Spark 3.0.x) + * fails class load in Spark 3.1.x. (`expr.input` is not recognized as an Expression.) + * 2. The semantics of offsets in LAG() are reversed/negated in Spark 3.1.1. + * E.g. The expression `LAG(col, 5)` causes Lag.offset to be set to `-5`, + * as opposed to `5`, in prior versions of Spark. + * This class adjusts the LAG offset to use similar semantics to Spark 3.0.x. + */ abstract class OffsetWindowFunctionMeta[INPUT <: OffsetWindowFunction] ( expr: INPUT, conf: RapidsConf, parent: Option[RapidsMeta[_, _]], rule: DataFromReplacementRule) - extends ExprMeta[INPUT](expr, conf, parent, rule) { + extends ExprMeta[INPUT](expr, conf, parent, rule) { lazy val input: BaseExprMeta[_] = GpuOverrides.wrapExpr(expr.input, conf, Some(this)) - lazy val offset: BaseExprMeta[_] = { + lazy val adjustedOffset: Expression = { expr match { - case _: Lead => // Supported. - case _: Lag => // Supported. + case lag: Lag => + GpuOverrides.extractLit(lag.offset) match { + case Some(Literal(offset: Int, IntegerType)) => + Literal(-offset, IntegerType) + case _ => + throw new IllegalStateException( + s"Only integer literal offsets are supported for LAG. Found:${lag.offset}") + } + case lead: Lead => + GpuOverrides.extractLit(lead.offset) match { + case Some(Literal(offset: Int, IntegerType)) => + Literal(offset, IntegerType) + case _ => + throw new IllegalStateException( + s"Only integer literal offsets are supported for LEAD. Found:${lead.offset}") + } case other => - throw new IllegalStateException( - s"Only LEAD/LAG offset window functions are supported. Found: $other") - } - - val literalOffset = GpuOverrides.extractLit(expr.offset) match { - case Some(Literal(offset: Int, IntegerType)) => - Literal(offset, IntegerType) - case _ => - throw new IllegalStateException( - s"Only integer literal offsets are supported for LEAD/LAG. Found: ${expr.offset}") + throw new IllegalStateException(s"$other is not a supported window function") } - - GpuOverrides.wrapExpr(literalOffset, conf, Some(this)) } + lazy val offset: BaseExprMeta[_] = + GpuOverrides.wrapExpr(adjustedOffset, conf, Some(this)) lazy val default: BaseExprMeta[_] = GpuOverrides.wrapExpr(expr.default, conf, Some(this)) override val childExprs: Seq[BaseExprMeta[_]] = Seq(input, offset, default) override def tagExprForGpu(): Unit = { expr match { - case _: Lead => // Supported. - case _: Lag => // Supported. + case Lead(_,_,_) => // Supported. + case Lag(_,_,_) => // Supported. case other => willNotWorkOnGpu( s"Only LEAD/LAG offset window functions are supported. Found: $other") } diff --git a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala index fda779c5cc7..ab8f6b04f85 100644 --- a/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala +++ b/spark2-sql-plugin/src/main/scala/com/nvidia/spark/rapids/shims/TypeSigUtil.scala @@ -20,7 +20,7 @@ import com.nvidia.spark.rapids.{TypeEnum, TypeSig} import org.apache.spark.sql.types.DataType -/** TypeSig Support for [3.0.1, 3.2.0) */ +/** TypeSig Support for [3.1.1, 3.2.0) */ object TypeSigUtil extends com.nvidia.spark.rapids.TypeSigUtilBase { /** diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala new file mode 100644 index 00000000000..d5cb9580e4f --- /dev/null +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/csv/GpuCsvUtils.scala @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.csv + +import org.apache.spark.sql.execution.datasources.csv.CSVOptions + +object GpuCsvUtils { + // spark 2.x uses FastDateFormat, use getPattern + def dateFormatInRead(options: CSVOptions): String = options.dateFormat.getPattern +} diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/expressions/rapids/Timestamp.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/expressions/rapids/Timestamp.scala new file mode 100644 index 00000000000..1f0751e626d --- /dev/null +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/expressions/rapids/Timestamp.scala @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.spark.sql.catalyst.expressions.rapids + +import com.nvidia.spark.rapids.ExprRule + +import org.apache.spark.sql.catalyst.expressions.Expression + +/** + * GetTimestamp is marked as private so we had to put it in a place that could access it. + */ +object TimeStamp { + + def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Map.empty +} diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/json/rapids/GpuJsonScan.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/json/rapids/GpuJsonScan.scala index b9918be7e0d..c338749ef2e 100644 --- a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/json/rapids/GpuJsonScan.scala +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/json/rapids/GpuJsonScan.scala @@ -30,12 +30,8 @@ import org.apache.spark.sql.types.{DateType, StringType, StructType, TimestampTy object GpuJsonScan { - def dateFormatInRead(fileOptions: Serializable): Option[String] = { - fileOptions match { - case jsonOpts: JSONOptions => Option(jsonOpts.dateFormat.getPattern) - case _ => throw new RuntimeException("Wrong file options.") - } - } + // spark 2.x uses FastDateFormat, use getPattern + def dateFormatInRead(options: JSONOptions): String = options.dateFormat.getPattern def timestampFormatInRead(fileOptions: Serializable): Option[String] = { fileOptions match { @@ -136,11 +132,8 @@ object GpuJsonScan { }) if (readSchema.map(_.dataType).contains(DateType)) { - dateFormatInRead(parsedOptions).foreach { dateFormat => - if (!supportedDateFormats.contains(dateFormat)) { - meta.willNotWorkOnGpu(s"the date format '${dateFormat}' is not supported'") - } - } + DateUtils.tagAndGetCudfFormat(meta, + dateFormatInRead(parsedOptions), parseString = true) } if (readSchema.map(_.dataType).contains(TimestampType)) { diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala index 4141642338d..25297497c4b 100644 --- a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileSourceScanExec.scala @@ -31,6 +31,7 @@ object GpuFileSourceScanExec { case f if GpuOrcFileFormat.isSparkOrcFormat(f) => GpuReadOrcFileFormat.tagSupport(meta) case _: ParquetFileFormat => GpuReadParquetFileFormat.tagSupport(meta) case _: JsonFileFormat => GpuReadJsonFileFormat.tagSupport(meta) + // SPARK 2.x - We leave off Avro here since its a datasource v2 thing and off by default case f => meta.willNotWorkOnGpu(s"unsupported file format: ${f.getClass.getCanonicalName}") } diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala index 757efd4a37f..92e36f09ac5 100644 --- a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala @@ -16,9 +16,9 @@ package org.apache.spark.sql.rapids -import com.nvidia.spark.rapids.{BinaryExprMeta, DataFromReplacementRule, DataTypeUtils, GpuOverrides, RapidsConf, RapidsMeta} +import com.nvidia.spark.rapids.{BinaryExprMeta, DataFromReplacementRule, DataTypeUtils, GpuOverrides, RapidsConf, RapidsMeta, UnaryExprMeta} -import org.apache.spark.sql.catalyst.expressions.{GetArrayItem, GetMapValue} +import org.apache.spark.sql.catalyst.expressions.{GetArrayItem, GetArrayStructFields, GetMapValue} class GpuGetArrayItemMeta( expr: GetArrayItem, @@ -26,17 +26,6 @@ class GpuGetArrayItemMeta( parent: Option[RapidsMeta[_, _]], rule: DataFromReplacementRule) extends BinaryExprMeta[GetArrayItem](expr, conf, parent, rule) { - import GpuOverrides._ - - override def tagExprForGpu(): Unit = { - extractLit(expr.ordinal).foreach { litOrd => - // Once literal array/struct types are supported this can go away - val ord = litOrd.value - if ((ord == null || ord.asInstanceOf[Int] < 0) && DataTypeUtils.isNestedType(expr.dataType)) { - willNotWorkOnGpu("negative and null indexes are not supported for nested types") - } - } - } } class GpuGetMapValueMeta( @@ -46,3 +35,11 @@ class GpuGetMapValueMeta( rule: DataFromReplacementRule) extends BinaryExprMeta[GetMapValue](expr, conf, parent, rule) { } + +class GpuGetArrayStructFieldsMeta( + expr: GetArrayStructFields, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _]], + rule: DataFromReplacementRule) + extends UnaryExprMeta[GetArrayStructFields](expr, conf, parent, rule) { +} diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/datetimeExpressionsMeta.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/datetimeExpressionsMeta.scala index f0dcb5addb3..a87586cf1fa 100644 --- a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/datetimeExpressionsMeta.scala +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/datetimeExpressionsMeta.scala @@ -51,8 +51,17 @@ object GpuToTimestamp { "dd-MM" -> ParseFormatMeta('-', isTimestamp = false, raw"\A\d{2}-\d{2}\Z"), "dd/MM" -> ParseFormatMeta('/', isTimestamp = false, - raw"\A\d{2}/\d{2}\Z") + raw"\A\d{2}/\d{2}\Z"), + "MM/yyyy" -> ParseFormatMeta('/', isTimestamp = false, + raw"\A\d{2}/\d{4}\Z"), + "MM-yyyy" -> ParseFormatMeta('-', isTimestamp = false, + raw"\A\d{2}-\d{4}\Z"), + "MM/dd/yyyy" -> ParseFormatMeta('/', isTimestamp = false, + raw"\A\d{2}/\d{2}/\d{4}\Z"), + "MM-dd-yyyy" -> ParseFormatMeta('-', isTimestamp = false, + raw"\A\d{2}-\d{2}-\d{4}\Z") ) + // We are compatible with Spark for these formats when the timeParserPolicy is LEGACY. It // is possible that other formats may be supported but these are the only ones that we have // tests for. @@ -94,54 +103,8 @@ abstract class UnixTimeExprMeta[A <: BinaryExpression with TimeZoneAwareExpressi extractStringLit(expr.right) match { case Some(rightLit) => sparkFormat = rightLit - if (GpuOverrides.getTimeParserPolicy == LegacyTimeParserPolicy) { - try { - // try and convert the format to cuDF format - this will throw an exception if - // the format contains unsupported characters or words - strfFormat = DateUtils.toStrf(sparkFormat, - expr.left.dataType == DataTypes.StringType) - // format parsed ok but we have no 100% compatible formats in LEGACY mode - if (GpuToTimestamp.LEGACY_COMPATIBLE_FORMATS.contains(sparkFormat)) { - // LEGACY support has a number of issues that mean we cannot guarantee - // compatibility with CPU - // - we can only support 4 digit years but Spark supports a wider range - // - we use a proleptic Gregorian calender but Spark uses a hybrid Julian+Gregorian - // calender in LEGACY mode - // Spark 2.x - ansi not available - /* - if (SQLConf.get.ansiEnabled) { - willNotWorkOnGpu("LEGACY format in ANSI mode is not supported on the GPU") - } else */ - if (!conf.incompatDateFormats) { - willNotWorkOnGpu(s"LEGACY format '$sparkFormat' on the GPU is not guaranteed " + - s"to produce the same results as Spark on CPU. Set " + - s"${RapidsConf.INCOMPATIBLE_DATE_FORMATS.key}=true to force onto GPU.") - } - } else { - willNotWorkOnGpu(s"LEGACY format '$sparkFormat' is not supported on the GPU.") - } - } catch { - case e: TimestampFormatConversionException => - willNotWorkOnGpu(s"Failed to convert ${e.reason} ${e.getMessage}") - } - } else { - try { - // try and convert the format to cuDF format - this will throw an exception if - // the format contains unsupported characters or words - strfFormat = DateUtils.toStrf(sparkFormat, - expr.left.dataType == DataTypes.StringType) - // format parsed ok, so it is either compatible (tested/certified) or incompatible - if (!GpuToTimestamp.CORRECTED_COMPATIBLE_FORMATS.contains(sparkFormat) && - !conf.incompatDateFormats) { - willNotWorkOnGpu(s"CORRECTED format '$sparkFormat' on the GPU is not guaranteed " + - s"to produce the same results as Spark on CPU. Set " + - s"${RapidsConf.INCOMPATIBLE_DATE_FORMATS.key}=true to force onto GPU.") - } - } catch { - case e: TimestampFormatConversionException => - willNotWorkOnGpu(s"Failed to convert ${e.reason} ${e.getMessage}") - } - } + strfFormat = DateUtils.tagAndGetCudfFormat(this, + sparkFormat, expr.left.dataType == DataTypes.StringType) case None => willNotWorkOnGpu("format has to be a string literal") } diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuHashJoin.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuHashJoin.scala index c65c5b62460..283e1e7b108 100644 --- a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuHashJoin.scala +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuHashJoin.scala @@ -48,6 +48,9 @@ object JoinTypeChecks { case LeftAnti if !conf.areLeftAntiJoinsEnabled => meta.willNotWorkOnGpu("left anti joins have been disabled. To enable set " + s"${RapidsConf.ENABLE_LEFT_ANTI_JOIN.key} to true") + case ExistenceJoin(_) if !conf.areExistenceJoinsEnabled => + meta.willNotWorkOnGpu("existence joins have been disabled. To enable set " + + s"${RapidsConf.ENABLE_EXISTENCE_JOIN.key} to true") case _ => // not disabled } } @@ -102,25 +105,19 @@ object GpuHashJoin { conditionMeta: Option[BaseExprMeta[_]]): Unit = { val keyDataTypes = (leftKeys ++ rightKeys).map(_.dataType) - def unSupportNonEqualCondition(): Unit = if (conditionMeta.isDefined) { - meta.willNotWorkOnGpu(s"$joinType joins currently do not support conditions") - } - def unSupportStructKeys(): Unit = if (keyDataTypes.exists(_.isInstanceOf[StructType])) { - meta.willNotWorkOnGpu(s"$joinType joins currently do not support with struct keys") - } JoinTypeChecks.tagForGpu(joinType, meta) joinType match { case _: InnerLike => - case RightOuter | LeftOuter => + case RightOuter | LeftOuter | LeftSemi | LeftAnti | ExistenceJoin(_) => conditionMeta.foreach(meta.requireAstForGpuOn) - case LeftSemi | LeftAnti => - unSupportNonEqualCondition() case FullOuter => conditionMeta.foreach(meta.requireAstForGpuOn) // FullOuter join cannot support with struct keys as two issues below // * https://github.com/NVIDIA/spark-rapids/issues/2126 // * https://github.com/rapidsai/cudf/issues/7947 - unSupportStructKeys() + if (keyDataTypes.exists(_.isInstanceOf[StructType])) { + meta.willNotWorkOnGpu(s"$joinType joins currently do not support with struct keys") + } case _ => meta.willNotWorkOnGpu(s"$joinType currently is not supported") } diff --git a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringMeta.scala b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringMeta.scala index 85814e5cb47..cf35b01d743 100644 --- a/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringMeta.scala +++ b/spark2-sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringMeta.scala @@ -19,9 +19,9 @@ package org.apache.spark.sql.rapids import scala.collection.mutable.ArrayBuffer import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.ShimExpression +import com.nvidia.spark.rapids.shims.{GpuRegExpUtils, ShimExpression} -import org.apache.spark.sql.catalyst.expressions.{Literal, RegExpExtract, RLike, StringSplit, SubstringIndex} +import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, Literal, RegExpExtract, RLike, StringSplit, StringToMap, SubstringIndex, TernaryExpression} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String @@ -34,11 +34,12 @@ class GpuRLikeMeta( private var pattern: Option[String] = None override def tagExprForGpu(): Unit = { + GpuRegExpUtils.tagForRegExpEnabled(this) expr.right match { case Literal(str: UTF8String, DataTypes.StringType) if str != null => try { // verify that we support this regex and can transpile it to cuDF format - pattern = Some(new CudfRegexTranspiler(replace = false).transpile(str.toString)) + pattern = Some(new CudfRegexTranspiler(RegexFindMode).transpile(str.toString)) } catch { case e: RegexUnsupportedException => willNotWorkOnGpu(e.getMessage) @@ -60,6 +61,7 @@ class GpuRegExpExtractMeta( private var numGroups = 0 override def tagExprForGpu(): Unit = { + GpuRegExpUtils.tagForRegExpEnabled(this) def countGroups(regexp: RegexAST): Int = { regexp match { @@ -73,7 +75,7 @@ class GpuRegExpExtractMeta( try { val javaRegexpPattern = str.toString // verify that we support this regex and can transpile it to cuDF format - val cudfRegexPattern = new CudfRegexTranspiler(replace = false) + val cudfRegexPattern = new CudfRegexTranspiler(RegexFindMode) .transpile(javaRegexpPattern) pattern = Some(cudfRegexPattern) numGroups = countGroups(new RegexParser(javaRegexpPattern).parse()) @@ -175,37 +177,152 @@ object GpuSubstringIndex { } } -class GpuStringSplitMeta( - expr: StringSplit, +// StringSplit is BinaryExpression in spark 2.x so needs to be separate +abstract class StringSplitRegBinaryExpMeta[INPUT <: BinaryExpression](expr: INPUT, conf: RapidsConf, parent: Option[RapidsMeta[_, _]], rule: DataFromReplacementRule) - extends BinaryExprMeta[StringSplit](expr, conf, parent, rule) { + extends BinaryExprMeta[INPUT](expr, conf, parent, rule) { import GpuOverrides._ - override def tagExprForGpu(): Unit = { - // 2.x uses expr.pattern not expr.regex - val regexp = extractLit(expr.pattern) - if (regexp.isEmpty) { - willNotWorkOnGpu("only literal regexp values are supported") + /** + * Return the cudf transpiled regex pattern, and a boolean flag indicating whether the input + * delimiter is really a regex patter or just a literal string. + */ + def checkRegExp(delimExpr: Expression): Option[(String, Boolean)] = { + var pattern: String = "" + var isRegExp: Boolean = false + + val delim = extractLit(delimExpr) + if (delim.isEmpty) { + willNotWorkOnGpu("Only literal delimiter patterns are supported") } else { - val str = regexp.get.value.asInstanceOf[UTF8String] - if (str != null) { - if (!canRegexpBeTreatedLikeARegularString(str)) { - willNotWorkOnGpu("regular expressions are not supported yet") + val utf8Str = delim.get.value.asInstanceOf[UTF8String] + if (utf8Str == null) { + willNotWorkOnGpu("Delimiter pattern is null") + } else { + if (utf8Str.numChars() == 0) { + willNotWorkOnGpu("An empty delimiter pattern is not supported") } - if (str.numChars() == 0) { - willNotWorkOnGpu("An empty regex is not supported yet") + val transpiler = new CudfRegexTranspiler(RegexSplitMode) + transpiler.transpileToSplittableString(utf8Str.toString) match { + case Some(simplified) => + pattern = simplified + case None => + try { + pattern = transpiler.transpile(utf8Str.toString) + isRegExp = true + } catch { + case e: RegexUnsupportedException => + willNotWorkOnGpu(e.getMessage) + } } + } + } + + Some((pattern, isRegExp)) + } + + def throwUncheckedDelimiterException() = + throw new IllegalStateException("Delimiter expression has not been checked for regex pattern") +} + +abstract class StringSplitRegExpMeta[INPUT <: TernaryExpression](expr: INPUT, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _]], + rule: DataFromReplacementRule) + extends TernaryExprMeta[INPUT](expr, conf, parent, rule) { + import GpuOverrides._ + + /** + * Return the cudf transpiled regex pattern, and a boolean flag indicating whether the input + * delimiter is really a regex patter or just a literal string. + */ + def checkRegExp(delimExpr: Expression): Option[(String, Boolean)] = { + var pattern: String = "" + var isRegExp: Boolean = false + + val delim = extractLit(delimExpr) + if (delim.isEmpty) { + willNotWorkOnGpu("Only literal delimiter patterns are supported") + } else { + val utf8Str = delim.get.value.asInstanceOf[UTF8String] + if (utf8Str == null) { + willNotWorkOnGpu("Delimiter pattern is null") } else { - willNotWorkOnGpu("null regex is not supported yet") + if (utf8Str.numChars() == 0) { + willNotWorkOnGpu("An empty delimiter pattern is not supported") + } + val transpiler = new CudfRegexTranspiler(RegexSplitMode) + transpiler.transpileToSplittableString(utf8Str.toString) match { + case Some(simplified) => + pattern = simplified + case None => + try { + pattern = transpiler.transpile(utf8Str.toString) + isRegExp = true + } catch { + case e: RegexUnsupportedException => + willNotWorkOnGpu(e.getMessage) + } + } } } + + Some((pattern, isRegExp)) + } + + def throwUncheckedDelimiterException() = + throw new IllegalStateException("Delimiter expression has not been checked for regex pattern") +} + +class GpuStringSplitMeta( + expr: StringSplit, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _]], + rule: DataFromReplacementRule) + extends StringSplitRegBinaryExpMeta[StringSplit](expr, conf, parent, rule) { + import GpuOverrides._ + + private var delimInfo: Option[(String, Boolean)] = None + + override def tagExprForGpu(): Unit = { + // 2.x uses expr.pattern not expr.regex + delimInfo = checkRegExp(expr.pattern) + // 2.x has no limit parameter /* - if (!isLit(expr.limit)) { - willNotWorkOnGpu("only literal limit is supported") + extractLit(expr.limit) match { + case Some(Literal(n: Int, _)) => + if (n == 0 || n == 1) { + // https://github.com/NVIDIA/spark-rapids/issues/4720 + willNotWorkOnGpu("limit of 0 or 1 is not supported") + } + case _ => + willNotWorkOnGpu("only literal limit is supported") } */ } } + +class GpuStringToMapMeta(expr: StringToMap, + conf: RapidsConf, + parent: Option[RapidsMeta[_, _]], + rule: DataFromReplacementRule) + extends StringSplitRegExpMeta[StringToMap](expr, conf, parent, rule) { + + private def checkFoldable(children: Seq[Expression]): Unit = { + if (children.forall(_.foldable)) { + willNotWorkOnGpu("result can be compile-time evaluated") + } + } + + private var pairDelimInfo: Option[(String, Boolean)] = None + private var keyValueDelimInfo: Option[(String, Boolean)] = None + + override def tagExprForGpu(): Unit = { + checkFoldable(expr.children) + pairDelimInfo = checkRegExp(expr.pairDelim) + keyValueDelimInfo = checkRegExp(expr.keyValueDelim) + } +}