add a spark 3.3 compat test in ci

Signed-off-by: Erik Ordentlich <eordentlich@gmail.com>
NVIDIA · Jul 29, 2023 · 5b9f0fc · 5b9f0fc
1 parent f84d032
commit 5b9f0fc
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/ci/test.sh b/ci/test.sh
@@ -43,3 +43,10 @@ pip install -r requirements_dev.txt && pip install -e .
 
 # benchmark
 ./run_benchmark.sh $bench_args
+
+# check compatibility with Spark 3.3 in nightly run
+if [[ $type == "nightly" ]]; then
+    pip uninstall pyspark -y
+    pip install pyspark~=3.3.0
+    ./run_benchmark.sh $bench_args
+fi