Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Better GPU Cast type checks #1402

Merged
merged 2 commits into from
Dec 16, 2020
Merged

Better GPU Cast type checks #1402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7bc3055
Better GPU Cast type checks
revans2 Dec 15, 2020
046893a
Addressed review comments
revans2 Dec 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
959 changes: 615 additions & 344 deletions docs/supported_ops.md
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions shims/spark300/src/main/scala/com/nvidia/spark/rapids/shims/spark300/Spark300Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ class Spark300Shims extends SparkShims {

override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
Seq(
GpuOverrides.expr[AnsiCast](
"Convert a column of one type of data into another type",
new CastChecks(),
(cast, conf, p, r) => new CastExprMeta[AnsiCast](cast, true, conf, p, r)),
GpuOverrides.expr[TimeSub](
"Subtracts interval from timestamp",
ExprChecks.binaryProjectNotLambda(TypeSig.TIMESTAMP, TypeSig.TIMESTAMP,
Expand Down
42 changes: 42 additions & 0 deletions shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,48 @@ class Spark310Shims extends Spark301Shims {
}

def exprs310: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq(
GpuOverrides.expr[AnsiCast](
"Convert a column of one type of data into another type",
new CastChecks() {
import TypeSig._
// nullChecks are the same

override val booleanChecks: TypeSig = integral + fp + BOOLEAN + STRING
override val sparkBooleanSig: TypeSig = numeric + BOOLEAN + STRING

override val integralChecks: TypeSig = integral + fp + BOOLEAN + STRING
override val sparkIntegralSig: TypeSig = numeric + BOOLEAN + STRING

override val fpChecks: TypeSig = integral + fp + BOOLEAN + STRING
override val sparkFpSig: TypeSig = numeric + BOOLEAN + STRING

override val dateChecks: TypeSig = TIMESTAMP + DATE + STRING
override val sparkDateSig: TypeSig = TIMESTAMP + DATE + STRING

override val timestampChecks: TypeSig = TIMESTAMP + DATE + STRING
override val sparkTimestampSig: TypeSig = TIMESTAMP + DATE + STRING

// stringChecks are the same
// binaryChecks are the same

override val decimalChecks: TypeSig = none
override val sparkDecimalSig: TypeSig = numeric + BOOLEAN + STRING

// calendarChecks are the same

override val arrayChecks: TypeSig = none
override val sparkArraySig: TypeSig = ARRAY.nested(all)

override val mapChecks: TypeSig = none
override val sparkMapSig: TypeSig = MAP.nested(all)

override val structChecks: TypeSig = none
override val sparkStructSig: TypeSig = STRUCT.nested(all)

override val udtChecks: TypeSig = none
override val sparkUdtSig: TypeSig = UDT
},
(cast, conf, p, r) => new CastExprMeta[AnsiCast](cast, true, conf, p, r)),
GpuOverrides.expr[RegExpReplace](
"RegExpReplace support for string literal input patterns",
ExprChecks.projectNotLambda(TypeSig.STRING, TypeSig.STRING,
Expand Down
72 changes: 0 additions & 72 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCast.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,6 @@ class CastExprMeta[INPUT <: CastBase](
private val toType = cast.dataType

override def tagExprForGpu(): Unit = {
if (!GpuCast.canCast(fromType, toType)) {
willNotWorkOnGpu(s"$castExpr from $fromType " +
s"to $toType is not currently supported on the GPU")
}
if (!conf.isCastFloatToStringEnabled && toType == DataTypes.StringType &&
(fromType == DataTypes.FloatType || fromType == DataTypes.DoubleType)) {
willNotWorkOnGpu("the GPU will use different precision than Java's toString method when " +
Expand Down Expand Up @@ -116,74 +112,6 @@ object GpuCast {
"required range"

val INVALID_FLOAT_CAST_MSG = "At least one value is either null or is an invalid number"


/**
* Returns true iff we can cast `from` to `to` using the GPU.
*/
def canCast(from: DataType, to: DataType): Boolean = {
if (from == to) {
return true
}
from match {
case NullType => to match {
// The only thing we really need is that we can use a null scalar to create a vector
case BooleanType | ByteType | ShortType | IntegerType | LongType | FloatType |
DoubleType | TimestampType | DateType | StringType => true
case _ => false
}
case BooleanType => to match {
case ByteType | ShortType | IntegerType | LongType => true
case FloatType | DoubleType => true
case TimestampType => true
case StringType => true
case _ => false
}
case ByteType | ShortType | IntegerType | LongType => to match {
case BooleanType => true
case ByteType | ShortType | IntegerType | LongType => true
case FloatType | DoubleType => true
case StringType => true
case TimestampType => true
case BinaryType => true
case _ => false
}
case FloatType | DoubleType => to match {
case BooleanType => true
case ByteType | ShortType | IntegerType | LongType => true
case FloatType | DoubleType => true
case TimestampType => true
case StringType => true
case _ => false
}
case DateType => to match {
case BooleanType => true
case ByteType | ShortType | IntegerType | LongType => true
case FloatType | DoubleType => true
case TimestampType => true
case StringType => true
case _ => false
}
case TimestampType => to match {
case BooleanType => true
case ByteType | ShortType | IntegerType => true
case LongType => true
case FloatType | DoubleType => true
case DateType => true
case StringType => true
case _ => false
}
case StringType => to match {
case BooleanType => true
case ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType => true
case DateType => true
case TimestampType => true
case BinaryType => true
case _ => false
}
case _ => false
}
}
}

/**
Expand Down
12 changes: 2 additions & 10 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,17 +706,9 @@ object GpuOverrides {
}),
expr[Cast](
"Convert a column of one type of data into another type",
ExprChecks.unaryProjectNotLambdaInputMatchesOutput(
TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.BINARY,
TypeSig.all),
new CastChecks(),
(cast, conf, p, r) => new CastExprMeta[Cast](cast, SparkSession.active.sessionState.conf
.ansiEnabled, conf, p, r)),
expr[AnsiCast](
"Convert a column of one type of data into another type",
ExprChecks.unaryProjectNotLambdaInputMatchesOutput(
TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.BINARY,
TypeSig.all),
(cast, conf, p, r) => new CastExprMeta[AnsiCast](cast, true, conf, p, r)),
.ansiEnabled, conf, p, r)),
expr[ToDegrees](
"Converts radians to degrees",
ExprChecks.mathUnary,
Expand Down
Loading