diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
index 92701221..efc8288b 100644
--- a/.github/workflows/gradle.yml
+++ b/.github/workflows/gradle.yml
@@ -10,13 +10,19 @@ on:
 
 jobs:
   build_pa_pkg:
+    strategy:
+      matrix:
+        java:
+          - 8
+          - 11
+          - 14
     runs-on: [ubuntu-latest]
     name: Building PA package
     steps:
     - name: Set up JDK
       uses: actions/setup-java@v1
       with:
-        java-version: 14
+        java-version: ${{matrix.java}}
     # Performance Analyzer in ./tmp/performance-analyzer
     - name: Checkout Performance Analyzer package
       uses: actions/checkout@v2
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index 9f096361..23fc6e2d 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -1,7 +1,7 @@
 - [Developer Guide](#developer-guide)
   - [Forking and Cloning](#forking-and-cloning)
   - [Install Prerequisites](#install-prerequisites)
-    - [JDK 14](#jdk-14)
+    - [JDK 11](#jdk-11)
   - [Building](#building)
   - [Using IntelliJ IDEA](#using-intellij-idea)
   - [Submitting Changes](#submitting-changes)
@@ -16,9 +16,9 @@ Fork this repository on GitHub, and clone locally with `git clone`.
 
 ### Install Prerequisites
 
-#### JDK 14
+#### JDK 11
 
-OpenSearch components build using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`.
+OpenSearch components build using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`.
 
 ### Building
 
diff --git a/build.gradle b/build.gradle
index 0fc0112c..86dd785e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -417,7 +417,6 @@ bundlePlugin {
     from('build/resources/main/') {
         include "plugin-security.policy"
     }
-    exclude('tools.jar')
     from("config/opensearch-performance-analyzer") {
         into "config"
     }
@@ -644,7 +643,10 @@ task bwcTestSuite(type: StandaloneRestIntegTestTask) {
     exclude '**/*Test*'
     exclude '**/*IT*'
     dependsOn tasks.named("${baseName}#mixedClusterTask")
-    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+    if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+        // Only running this for > jdk8 versions as of now.
+        dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+    }
     dependsOn tasks.named("${baseName}#fullRestartClusterTask")
 }