Skip to content

Commit

Permalink
Use the APIs for creation from utf8 string. (NVIDIA#2506)
Browse files Browse the repository at this point in the history
Signed-off-by: Firestarman <firestarmanllc@gmail.com>
  • Loading branch information
firestarman authored May 26, 2021
1 parent 5595042 commit f038749
Showing 1 changed file with 6 additions and 12 deletions.
18 changes: 6 additions & 12 deletions sql-plugin/src/main/scala/com/nvidia/spark/rapids/literals.scala
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,7 @@ object GpuScalar extends Arm with Logging {
/** Converts an element for nested lists */
private def convertElementTo(element: Any, elementType: DataType): Any = elementType match {
case _ if element == null => null
// StructData does not support utf8 string yet, so parses it here instead of calling the
// `convertUTF8StringTo`. Tracked by https://github.com/rapidsai/cudf/issues/8137
case StringType => element.asInstanceOf[UTF8String].toString
case StringType => element.asInstanceOf[UTF8String].getBytes
case dt: DecimalType => convertDecimalTo(element.asInstanceOf[Decimal], dt) match {
case Left(i) => i
case Right(l) => l
Expand Down Expand Up @@ -159,13 +157,10 @@ object GpuScalar extends Arm with Logging {
case TimestampType =>
ColumnVector.timestampMicroSecondsFromBoxedLongs(seq.asInstanceOf[Seq[JLong]]: _*)
case StringType =>
// To be updated. https://github.com/rapidsai/cudf/issues/8137
ColumnVector.build(DType.STRING, seq.length, b => {
seq.asInstanceOf[Seq[UTF8String]].foreach {
case null => b.appendNull()
case s => b.appendUTF8String(s.getBytes)
}
})
ColumnVector.fromUTF8Strings(seq.asInstanceOf[Seq[UTF8String]].map {
case null => null
case us => us.getBytes
}: _*)
case dt: DecimalType =>
val decs = seq.asInstanceOf[Seq[Decimal]]
if (DecimalType.is32BitDecimalType(dt)) {
Expand Down Expand Up @@ -290,8 +285,7 @@ object GpuScalar extends Arm with Logging {
}
case StringType => v match {
case s: String => Scalar.fromString(s)
// TODO JNI supports creating a scalar from UTF8 String bytes directly.
case us: UTF8String => Scalar.fromString(us.toString)
case us: UTF8String => Scalar.fromUTF8String(us.getBytes)
case _ => throw new IllegalArgumentException(s"'$v: ${v.getClass}' is not supported" +
s" for StringType, expecting String or UTF8String.")
}
Expand Down

0 comments on commit f038749

Please sign in to comment.