[BUG] Host column vectors are being leaked during tests

[jlowe]

Seeing many occurrences of host column vectors being leaked in scala-test-detailed-output.log during the nightly build, For example:

21/12/22 14:53:35.098 Cleaner Thread ERROR HostColumnVector: A HOST COLUMN VECTOR WAS LEAKED (ID: 47963)
21/12/22 14:53:35.106 Cleaner Thread ERROR MemoryCleaner: Leaked vector (ID: 47963): 2021-12-22 14:53:33.0890 UTC: INC
java.lang.Thread.getStackTrace(Thread.java:1559)
ai.rapids.cudf.MemoryCleaner$RefCountDebugItem.<init>(MemoryCleaner.java:301)
ai.rapids.cudf.MemoryCleaner$Cleaner.addRef(MemoryCleaner.java:82)
ai.rapids.cudf.HostColumnVector.incRefCountInternal(HostColumnVector.java:147)
ai.rapids.cudf.HostColumnVector.<init>(HostColumnVector.java:78)
ai.rapids.cudf.ColumnView.copyToHost(ColumnView.java:4220)
com.nvidia.spark.rapids.GpuColumnVector.copyToHost(GpuColumnVector.java:1113)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$toHost$2(GpuColumnarToRowExec.scala:191)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
scala.collection.TraversableLike.map(TraversableLike.scala:286)
scala.collection.TraversableLike.map$(TraversableLike.scala:279)
scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$2(GpuColumnarToRowExec.scala:211)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$2$adapted(GpuColumnarToRowExec.scala:209)
com.nvidia.spark.rapids.Arm.withResource(Arm.scala:28)
com.nvidia.spark.rapids.Arm.withResource$(Arm.scala:26)
com.nvidia.spark.rapids.ColumnarToRowIterator.withResource(GpuColumnarToRowExec.scala:178)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$1(GpuColumnarToRowExec.scala:209)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$1$adapted(GpuColumnarToRowExec.scala:208)
scala.Option.foreach(Option.scala:407)
com.nvidia.spark.rapids.ColumnarToRowIterator.loadNextBatch(GpuColumnarToRowExec.scala:208)
com.nvidia.spark.rapids.ColumnarToRowIterator.next(GpuColumnarToRowExec.scala:255)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.$anonfun$new$7(GpuColumnarToRowSuite.scala:61)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.$anonfun$new$6(GpuColumnarToRowSuite.scala:60)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.$anonfun$new$6$adapted(GpuColumnarToRowSuite.scala:57)
com.nvidia.spark.rapids.Arm.closeOnExcept(Arm.scala:87)
com.nvidia.spark.rapids.Arm.closeOnExcept$(Arm.scala:85)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.closeOnExcept(GpuColumnarToRowSuite.scala:25)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.$anonfun$new$4(GpuColumnarToRowSuite.scala:57)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.$anonfun$new$4$adapted(GpuColumnarToRowSuite.scala:54)
com.nvidia.spark.rapids.Arm.withResource(Arm.scala:28)
com.nvidia.spark.rapids.Arm.withResource$(Arm.scala:26)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.withResource(GpuColumnarToRowSuite.scala:25)
com.nvidia.spark.rapids.GpuColumnarToRowSuite.$anonfun$new$3(GpuColumnarToRowSuite.scala:54)
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[...]

[...]

21/12/22 14:53:35.106 Cleaner Thread ERROR HostColumnVector: A HOST COLUMN VECTOR WAS LEAKED (ID: 47621)
1/12/22 14:53:35.106 Cleaner Thread ERROR MemoryCleaner: Leaked vector (ID: 47621): 2021-12-22 14:53:33.0623 UTC: INC
java.lang.Thread.getStackTrace(Thread.java:1559)
ai.rapids.cudf.MemoryCleaner$RefCountDebugItem.<init>(MemoryCleaner.java:301)
ai.rapids.cudf.MemoryCleaner$Cleaner.addRef(MemoryCleaner.java:82)
ai.rapids.cudf.HostColumnVector.incRefCountInternal(HostColumnVector.java:147)
ai.rapids.cudf.HostColumnVector.<init>(HostColumnVector.java:93)
ai.rapids.cudf.ColumnView.copyToHost(ColumnView.java:4196)
com.nvidia.spark.rapids.GpuColumnVector.copyToHost(GpuColumnVector.java:1113)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$toHost$2(GpuColumnarToRowExec.scala:191)
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
scala.collection.TraversableLike.map(TraversableLike.scala:286)
scala.collection.TraversableLike.map$(TraversableLike.scala:279)
scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$2(GpuColumnarToRowExec.scala:211)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$2$adapted(GpuColumnarToRowExec.scala:209)
com.nvidia.spark.rapids.Arm.withResource(Arm.scala:28)
com.nvidia.spark.rapids.Arm.withResource$(Arm.scala:26)
com.nvidia.spark.rapids.ColumnarToRowIterator.withResource(GpuColumnarToRowExec.scala:178)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$1(GpuColumnarToRowExec.scala:209)
com.nvidia.spark.rapids.ColumnarToRowIterator.$anonfun$loadNextBatch$1$adapted(GpuColumnarToRowExec.scala:208)
scala.Option.foreach(Option.scala:407)
com.nvidia.spark.rapids.ColumnarToRowIterator.loadNextBatch(GpuColumnarToRowExec.scala:208)
com.nvidia.spark.rapids.ColumnarToRowIterator.next(GpuColumnarToRowExec.scala:255)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$10(InternalColumnarRDDConverterSuite.scala:98)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$10$adapted(InternalColumnarRDDConverterSuite.scala:97)
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$9(InternalColumnarRDDConverterSuite.scala:97)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$9$adapted(InternalColumnarRDDConverterSuite.scala:94)
com.nvidia.spark.rapids.Arm.closeOnExcept(Arm.scala:87)
com.nvidia.spark.rapids.Arm.closeOnExcept$(Arm.scala:85)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.closeOnExcept(InternalColumnarRDDConverterSuite.scala:28)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$7(InternalColumnarRDDConverterSuite.scala:94)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$7$adapted(InternalColumnarRDDConverterSuite.scala:91)
com.nvidia.spark.rapids.Arm.withResource(Arm.scala:28)
com.nvidia.spark.rapids.Arm.withResource$(Arm.scala:26)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.withResource(InternalColumnarRDDConverterSuite.scala:28)
org.apache.spark.sql.rapids.execution.InternalColumnarRDDConverterSuite.$anonfun$new$6(InternalColumnarRDDConverterSuite.scala:91)
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[...]