Plug-in Build Failing for Databricks 11.3

[razajafri]

Describe the bug
build is failing on Databricks 11.3. Possibly related to #10274

INFO] Compiling 449 Scala sources and 58 Java sources to /home/ubuntu/spark-rapids/sql-plugin/target/spark330db/classes...
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala:29: object PythonWorker is not a member of package org.apache.spark.api.python
[ERROR] import org.apache.spark.api.python.{ChainedPythonFunctions, PythonRDD, PythonWorker}
[ERROR]        ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala:41: not found: type Writer
[ERROR]       writer: Writer,
[ERROR]               ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala:44: not found: type PythonWorker
[ERROR]       worker: PythonWorker,
[ERROR]               ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala:55: not found: type Writer
[ERROR]       context: TaskContext): Writer = {
[ERROR]                              ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala:52: not found: type PythonWorker
[ERROR]       worker: PythonWorker,
[ERROR]               ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala:56: not found: type Writer
[ERROR]     new Writer(env, worker, inputIterator, partitionIndex, context) {
[ERROR]         ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala:70: private method writeNextInputToStream in <$anon: <error>> is never used
[ERROR]       override def writeNextInputToStream(dataOut: DataOutputStream): Boolean = {
[ERROR]                    ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala:60: not found: type Writer
[ERROR]       context: TaskContext): Writer = {
[ERROR]                              ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala:57: not found: type PythonWorker
[ERROR]       worker: PythonWorker,  // Changed from "Socket" to this "PythonWorker" from db341
[ERROR]               ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala:61: not found: type Writer
[ERROR]     new Writer(env, worker, inputIterator, partitionIndex, context) {
[ERROR]         ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala:29: Unused import
[ERROR] import org.apache.spark.api.python.{ChainedPythonFunctions, PythonRDD, PythonWorker}
[ERROR]                                                                        ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala:73: private method writeNextInputToStream in <$anon: <error>> is never used
[ERROR]       override def writeNextInputToStream(dataOut: DataOutputStream): Boolean = {
[ERROR]                    ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala:68: not found: type Writer
[ERROR]       context: TaskContext): Writer = {
[ERROR]                              ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala:65: not found: type PythonWorker
[ERROR]       worker: PythonWorker, // From DB341, changed from Socket to PythonWorker
[ERROR]               ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala:69: not found: type Writer
[ERROR]     new Writer(env, worker, inputIterator, partitionIndex, context) {
[ERROR]         ^
[ERROR] /home/ubuntu/spark-rapids/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala:81: private method writeNextInputToStream in <$anon: <error>> is never used
[ERROR]       override def writeNextInputToStream(dataOut: DataOutputStream): Boolean = {

[razajafri]

Fixed by #10439