fail if ignoreOrdering and multiple partitions

NVIDIA · Sep 24, 2020 · fde56bd · fde56bd
1 parent c78b8fc
commit fde56bd
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/common/BenchUtils.scala b/integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/common/BenchUtils.scala
@@ -400,6 +400,12 @@ object BenchUtils {
 
     if (count1 == count2) {
       println(s"Both DataFrames contain $count1 rows")
+
+      if (!ignoreOrdering && (df1.rdd.getNumPartitions > 1 || df2.rdd.getNumPartitions > 1)) {
+        throw new IllegalStateException(
+          "Cannot ignore ordering because one or more inputs have multiple partitions")
+      }
+
       val result1 = collectResults(df1, ignoreOrdering, useIterator)
       val result2 = collectResults(df2, ignoreOrdering, useIterator)