Skip to content

Commit

Permalink
Fixing the failing test test_window on DB (#2484)
Browse files Browse the repository at this point in the history
By making the 'outReference' be lazy. Because on Databricks, it
will get some invalid expressions in the 'projectList' if binding
the reference on driver side. E.g. 'none#0L', 'none@1L'.

Signed-off-by: Firestarman <firestarmanllc@gmail.com>
  • Loading branch information
firestarman authored May 24, 2021
1 parent 9d6cb6e commit 31b168e
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 5 deletions.
4 changes: 1 addition & 3 deletions integration_tests/src/main/python/udf_cudf_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import pytest

from conftest import is_at_least_precommit_run, is_databricks_runtime
from conftest import is_at_least_precommit_run

from pyspark.sql.pandas.utils import require_minimum_pyarrow_version, require_minimum_pandas_version
try:
Expand Down Expand Up @@ -275,8 +275,6 @@ def gpu_run(spark):

# ======= Test Window In Pandas =======
@cudf_udf
@pytest.mark.xfail(condition=is_databricks_runtime(),
reason='https://github.com/NVIDIA/spark-rapids/issues/2372')
def test_window(enable_cudf_udf):
@pandas_udf("int")
def _sum_cpu_func(v: pd.Series) -> int:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ case class GpuWindowInPandasExec(
}
}

private val outReferences = {
private lazy val outReferences = {
val allExpressions = windowFramesWithExpressions.map(_._2).flatten
val references = allExpressions.zipWithIndex.map { case (e, i) =>
// Results of window expressions will be on the right side of child's output
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ case class GpuWindowInPandasExec(
}
}

private val outReferences = {
private lazy val outReferences = {
val allExpressions = windowFramesWithExpressions.map(_._2).flatten
val references = allExpressions.zipWithIndex.map { case (e, i) =>
// Results of window expressions will be on the right side of child's output
Expand Down

0 comments on commit 31b168e

Please sign in to comment.