From 06e1522c03ee3994d1b357a457f495e7a1ec74d4 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Thu, 20 Aug 2020 14:50:28 +0800 Subject: [PATCH] Keep only tests for exsisting features --- .../scala/com/nvidia/spark/OpcodeSuite.scala | 61 ------------------- 1 file changed, 61 deletions(-) diff --git a/udf-compiler/src/test/scala/com/nvidia/spark/OpcodeSuite.scala b/udf-compiler/src/test/scala/com/nvidia/spark/OpcodeSuite.scala index e4690a78ae7..ff0d4dfb7b9 100644 --- a/udf-compiler/src/test/scala/com/nvidia/spark/OpcodeSuite.scala +++ b/udf-compiler/src/test/scala/com/nvidia/spark/OpcodeSuite.scala @@ -18,7 +18,6 @@ package com.nvidia.spark import com.nvidia.spark.rapids.RapidsConf import org.scalatest.FunSuite - import org.apache.spark.SparkConf import org.apache.spark.sql.{Dataset, SparkSession} import org.apache.spark.sql.expressions.UserDefinedFunction @@ -57,10 +56,6 @@ class OpcodeSuite extends FunSuite { ds2.show } - def udfIsCompiled[T](ds: Dataset[T]) : Boolean = { - !ds.queryExecution.analyzed.toString().contains("UDF") - } - // START OF TESTS test("not boolean") { @@ -1886,60 +1881,4 @@ class OpcodeSuite extends FunSuite { val ref = dataset.withColumn("new",lit("this is the string".getBytes("US-ASCII"))) checkEquiv(result, ref) } - - test("Float - isNaN - True") { - val myudf: (Float) => Boolean = a => { - a.isNaN() - } - val u = makeUdf(myudf) - val dataset = List(Float.NaN).toDF("x").repartition(1) - val result = dataset.withColumn("new", u(col("x"))) - val ref = dataset.withColumn("new", lit(Float.NaN.isNaN)) - assert(udfIsCompiled(result)) - checkEquiv(result, ref) - } - - test("Float - isNaN - False") { - val myudf: (Float) => Boolean = a => { - a.isNaN() - } - val u = makeUdf(myudf) - val dataset = List(2f).toDF("x").repartition(1) - val result = dataset.withColumn("new", u(col("x"))) - val ref = dataset.withColumn("new", lit(false)) - assert(udfIsCompiled(result)) - checkEquiv(result, ref) - } - - test("Double - isNaN - True") { - val myudf: (Double) => Boolean = a => { - a.isNaN() - } - val u = makeUdf(myudf) - val dataset = List(Double.NaN).toDF("x").repartition(1) - val result = dataset.withColumn("new", u(col("x"))) - val ref = dataset.withColumn("new", lit(Double.NaN.isNaN)) - assert(udfIsCompiled(result)) - checkEquiv(result, ref) - } - - test("Double - isNaN - False") { - val myudf: (Float) => Boolean = a => { - a.isNaN() - } - val u = makeUdf(myudf) - val dataset = List(2D).toDF("x").repartition(1) - val result = dataset.withColumn("new", u(col("x"))) - val ref = dataset.withColumn("new", lit(false)) - assert(udfIsCompiled(result)) - checkEquiv(result, ref) - } - - test("NaNvl") { - val dataset = List((2,Float.NaN),(3,Float.NaN)).toDF("x", "y").repartition(1) - val result = dataset.withColumn("new", nanvl(dataset.col("x"), dataset.col("y"))) - dataset.show() - result.show() - result.explain(true) - } }