remove spark311db shim layer (#4094)

Signed-off-by: Peixin Li <pxli@nyu.edu>

jenkins/databricks/build.sh

@@ -26,7 +26,6 @@ BUILD_PROFILES=${BUILD_PROFILES:-'databricks312,!snapshot-shims'}
 BASE_SPARK_VERSION=${BASE_SPARK_VERSION:-'3.1.2'}
 BUILDVER=$(echo ${BASE_SPARK_VERSION} | sed 's/\.//g')db
 # the version of Spark used when we install the Databricks jars in .m2
-# 3.1.0-databricks is add because its actually based on Spark 3.1.1
 BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=${BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS:-$BASE_SPARK_VERSION}
 SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=$BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS-databricks
 # something like spark_3_1 or spark_3_0

jenkins/databricks/params.py

@@ -23,10 +23,6 @@
 source_tgz = 'spark-rapids-ci.tgz'
 tgz_dest = '/home/ubuntu/spark-rapids-ci.tgz'
 base_spark_pom_version = '3.0.1'
-# this is odd but with Databricks 8.2 it reports Spark version 3.1.0
-# but its really 3.1.1, so base Spark pom will be 3.1.1 but we want to
-# install them as 3.1.0-databricks. Other Databricks versions
-# this is the same as base_spark_pom_version.
 base_spark_version_to_install_databricks_jars = base_spark_pom_version
 clusterid = ''
 build_profiles = 'databricks,!snapshot-shims'

pom.xml

@@ -357,63 +357,6 @@
                 <module>aggregator</module>
             </modules>
         </profile>
-        <profile>
-            <!-- Note Databricks requires 2 properties -Ddatabricks and -Dbuildver=311db -->
-            <id>release311db</id>
-            <activation>
-                <property>
-                    <name>buildver</name>
-                    <value>311db</value>
-                </property>
-            </activation>
-            <properties>
-                <!-- Downgrade scala plugin version due to: https://github.com/sbt/sbt/issues/4305 -->
-                <scala.plugin.version>3.4.4</scala.plugin.version>
-                <shim.module.name>spark311db</shim.module.name>
-                <spark.version.classifier>spark311db</spark.version.classifier>
-                <!--
-                     Note that we are using the Spark version for all of the Databricks dependencies as well.
-                     The jenkins/databricks/build.sh script handles installing the jars as maven artifacts.
-                     This is to make it easier and not have to change version numbers for each individual dependency
-                     and deal with differences between Databricks versions
-                -->
-                <spark.version>${spark311db.version}</spark.version>
-                <spark.test.version>${spark311db.version}</spark.test.version>
-                <rat.consoleOutput>true</rat.consoleOutput>
-                <hive.storage.api.version>${spark311db.version}</hive.storage.api.version>
-                <protobuf.java.version>${spark311db.version}</protobuf.java.version>
-            </properties>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.codehaus.mojo</groupId>
-                        <artifactId>build-helper-maven-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <id>add-profile-src-31+</id>
-                                <goals><goal>add-source</goal></goals>
-                                <phase>generate-sources</phase>
-                                <configuration>
-                                    <sources>
-                                        <source>${project.basedir}/src/main/301until320-all/scala</source>
-                                        <source>${project.basedir}/src/main/311+-all/scala</source>
-                                        <source>${project.basedir}/src/main/31xdb/scala</source>
-                                        <source>${project.basedir}/src/main/311db/scala</source>
-                                        <source>${project.basedir}/src/main/311until320-all/scala</source>
-                                        <source>${project.basedir}/src/main/311until320-apache/scala</source>
-                                        <source>${project.basedir}/src/main/301until330-all/scala</source>
-                                        <source>${project.basedir}/src/main/pre320-treenode/scala</source>
-                                    </sources>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-            <modules>
-                <module>aggregator</module>
-            </modules>
-        </profile>
         <profile>
             <!-- Note Databricks requires 2 properties -Ddatabricks and -Dbuildver=312db -->
             <id>release312db</id>
@@ -823,7 +766,6 @@
         <spark302.version>3.0.2</spark302.version>
         <spark303.version>3.0.3</spark303.version>
         <spark304.version>3.0.4-SNAPSHOT</spark304.version>
-        <spark311db.version>3.1.1-databricks</spark311db.version>
         <spark311.version>3.1.1</spark311.version>
         <spark311cdh.version>3.1.1.3.1.7270.0-253</spark311cdh.version>
         <spark312.version>3.1.2</spark312.version>