Issue #23507 Jenkins optimizations

- memory and cpu settings tested for current hardware usually used on jenkins,
  respecting limits
- compilation without test with parallelized maven execution
- upgraded jnlp
- separate settings for maven and ant tests
- bundles content with fast compression, excluded what we don't need

Jenkinsfile

@@ -19,8 +19,11 @@
 // without this, the agent could be using a pod created from a different descriptor
 env.label = "glassfish-ci-pod-${UUID.randomUUID().toString()}"
 
-// list of test ids
+// Docker image defined in this project in [glassfish]/etc/docker/Dockerfile
+env.gfImage = "ee4jglassfish/ci:tini-jdk-11.0.10"
+
 def jobs = [
+  "verifyPhase",
   "cdi_all",
   "ql_gf_full_profile_all",
   "ql_gf_web_profile_all",
@@ -31,136 +34,118 @@ def jobs = [
   "ejb_group_embedded"
 ]
 
-
-def jobs_all = [
-  "cdi_all",
-  "ql_gf_full_profile_all",
-  "ql_gf_web_profile_all",
-  "web_jsp",
-  "ejb_group_1",
-  "ejb_group_2",
-  "ejb_group_3",
-  "ejb_group_embedded",
-  "ejb_web_all",
-  "ejb_all",
-
-  "deployment_all",
-  "ql_gf_nucleus_all",
-  "nucleus_admin_all",
-  "jdbc_all",
-  "batch_all",
-  "persistence_all",
-  "connector_group_1",
-  "connector_group_2",
-  "connector_group_3",
-  "connector_group_4"
-]
-
 def parallelStagesMap = jobs.collectEntries {
   ["${it}": generateStage(it)]
 }
 
 def generateStage(job) {
-    return {
-        podTemplate(label: env.label) {
-            node(label) {
-                stage("${job}") {
-                    container('glassfish-ci') {
-                      // do the scm checkout
-                      retry(10) {
-                        sleep 60
-                        checkout scm
-                      }
-
-                      // run the test
-                      unstash 'build-bundles'
-
-                      try {
-                          retry(3) {
-                              timeout(time: 2, unit: 'HOURS') {
-                                sh """
-                                  export CLASSPATH=$WORKSPACE/glassfish6/javadb
-                                  ./appserver/tests/gftest.sh run_test ${job}
-                                """
-                              }
-                          }
-                      } finally {
-                        // archive what we can...
-                        archiveArtifacts artifacts: "${job}-results.tar.gz"
-                        junit testResults: 'results/junitreports/*.xml', allowEmptyResults: false
-                      }
-                    }
-                }
+  if (job == 'verifyPhase') {
+    return generateMvnPodTemplate(job)
+  } else {
+    return generateAntPodTemplate(job)
+  }
+}
+
+def generateMvnPodTemplate(job) {
+  return {
+    podTemplate(
+      inheritFrom: "${env.label}",
+      containers: [
+        containerTemplate(
+          name: "glassfish-build",
+          image: "${env.gfImage}",
+          resourceRequestMemory: "7Gi",
+          resourceRequestCpu: "2650m"
+        )
+      ]
+    ) {
+      node(label) {
+        stage("${job}") {
+          container('glassfish-build') {
+            retry(5) {
+              sleep 1
+              checkout scm
             }
+            timeout(time: 1, unit: 'HOURS') {
+              sh """
+                mvn clean install
+              """
+              junit testResults: '**/*-reports/*.xml', allowEmptyResults: false
+            }
+          }
         }
+      }
     }
+  }
 }
 
-pipeline {
-
-  options {
-    // keep at most 50 builds
-    buildDiscarder(logRotator(numToKeepStr: '10'))
-
-    // preserve the stashes to allow re-running a test stage
-    preserveStashes()
-
-    // issue related to default 'implicit' checkout, disable it
-    skipDefaultCheckout()
-
-    // abort pipeline if previous stage is unstable
-    skipStagesAfterUnstable()
-
-    // show timestamps in logs
-    timestamps()
-
-    // global timeout, abort after 6 hours
-    timeout(time: 6, unit: 'HOURS')
+def generateAntPodTemplate(job) {
+  return {
+    podTemplate(
+      inheritFrom: "${env.label}",
+      containers: [
+        containerTemplate(
+          name: "glassfish-build",
+          image: "${env.gfImage}",
+          resourceRequestMemory: "4Gi",
+          resourceRequestCpu: "2650m"
+        )
+      ]
+    ) {
+      node(label) {
+        stage("${job}") {
+          container('glassfish-build') {
+            retry(5) {
+              sleep 1
+              checkout scm
+            }
+            unstash 'build-bundles'
+            try {
+              timeout(time: 1, unit: 'HOURS') {
+                sh """
+                  export CLASSPATH=$WORKSPACE/glassfish6/javadb
+                  ./appserver/tests/gftest.sh run_test ${job}
+                """
+              }
+            } finally {
+              archiveArtifacts artifacts: "${job}-results.tar.gz"
+              junit testResults: 'results/junitreports/*.xml', allowEmptyResults: false
+            }
+          }
+        }
+      }
+    }
   }
+}
+
+pipeline {
 
   agent {
     kubernetes {
       label "${env.label}"
-      defaultContainer 'glassfish-ci'
       yaml """
 apiVersion: v1
 kind: Pod
 metadata:
 spec:
-  volumes:
-    - name: "jenkins-home"
-      emptyDir: {}
-    - name: maven-repo-shared-storage
-      persistentVolumeClaim:
-       claimName: glassfish-maven-repo-storage
-    - name: settings-xml
-      secret:
-        secretName: m2-secret-dir
-        items:
-        - key: settings.xml
-          path: settings.xml
-    - name: settings-security-xml
-      secret:
-        secretName: m2-secret-dir
-        items:
-        - key: settings-security.xml
-          path: settings-security.xml
-    - name: maven-repo-local-storage
-      emptyDir: {}
   containers:
   - name: jnlp
-    image: jenkins/jnlp-slave:alpine
+    image: jenkins/inbound-agent:4.11-1-alpine-jdk11
     imagePullPolicy: IfNotPresent
     env:
       - name: JAVA_TOOL_OPTIONS
-        value: -Xmx1G
+        value: "-Xmx768m -Xss768k"
     resources:
+      # fixes random failure: minimum cpu usage per Pod is 200m, but request is 100m.
+      # affects performance on large repositories
       limits:
-        memory: "1Gi"
-        cpu: "1"
-  - name: glassfish-ci
-    # Docker image defined in this project in [glassfish]/etc/docker/Dockerfile
-    image: ee4jglassfish/ci:tini-jdk-11.0.10
+        memory: "1200Mi"
+        cpu: "300m"
+      requests:
+        memory: "1200Mi"
+        cpu: "300m"
+  - name: glassfish-build
+    image: ${env.gfImage}
     args:
     - cat
     tty: true
@@ -183,13 +168,38 @@ spec:
         mountPath: "/home/jenkins/.m2/repository/org/glassfish/main"
     env:
       - name: "MAVEN_OPTS"
-        value: "-Duser.home=/home/jenkins"
+        value: "-Duser.home=/home/jenkins -Xmx2500m -Xss768k -XX:+UseStringDeduplication"
       - name: "MVN_EXTRA"
         value: "--batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+      - name: JAVA_TOOL_OPTIONS
+        value: "-Xmx2g -Xss768k -XX:+UseStringDeduplication"
     resources:
       limits:
+        memory: "12Gi"
+        cpu: "8000m"
+      requests:
         memory: "7Gi"
-        cpu: "3"
+        cpu: "4000m"
+  volumes:
+    - name: "jenkins-home"
+      emptyDir: {}
+    - name: maven-repo-shared-storage
+      persistentVolumeClaim:
+        claimName: glassfish-maven-repo-storage
+    - name: settings-xml
+      secret:
+        secretName: m2-secret-dir
+        items:
+        - key: settings.xml
+          path: settings.xml
+    - name: settings-security-xml
+      secret:
+        secretName: m2-secret-dir
+        items:
+        - key: settings-security.xml
+          path: settings-security.xml
+    - name: maven-repo-local-storage
+      emptyDir: {}
 """
     }
   }
@@ -204,21 +214,37 @@ spec:
     PORT_HTTPS=8181
   }
 
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '10'))
+
+    // to allow re-running a test stage
+    preserveStashes()
+
+    // issue related to default 'implicit' checkout, disable it
+    skipDefaultCheckout()
+
+    // abort pipeline if previous stage is unstable
+    skipStagesAfterUnstable()
+
+    // show timestamps in logs
+    timestamps()
+
+    // global timeout, abort after 6 hours
+    timeout(time: 6, unit: 'HOURS')
+  }
+
   stages {
+
     stage('build') {
       agent {
         kubernetes {
           label "${env.label}"
         }
       }
       steps {
-        container('glassfish-ci') {
+        container('glassfish-build') {
           timeout(time: 1, unit: 'HOURS') {
-
-            // do the scm checkout
             checkout scm
-
-            // do the build
             sh '''
               echo Maven version
               mvn -v
@@ -229,10 +255,12 @@ spec:
               echo Uname
               uname -a
 
-              bash -xe ./gfbuild.sh build_re_dev
+              # Until we fix ANTLR in cmp-support-sqlstore, broken in parallel builds. Just -Pfast after the fix.
+              mvn clean install -Pfastest,staging -T4C
+              ./gfbuild.sh archive_bundles
+              ls -la ./bundles
             '''
             archiveArtifacts artifacts: 'bundles/*.zip'
-            // junit testResults: 'test-results/build-unit-tests/results/junitreports/test_results_junit.xml'
             stash includes: 'bundles/*', name: 'build-bundles'
           }
         }

appserver/tests/gftest.sh

@@ -49,7 +49,7 @@ if [ ! -z "${JENKINS_HOME}" ] ; then
 
   # setup the local repository
   # with the archived chunk from the pipeline build stage
-  cat ${WORKSPACE}/bundles/_maven-repo* | tar -xvz -f - --overwrite -m -p -C ${HOME}/.m2/repository
+  tar -xzf ${WORKSPACE}/bundles/maven-repo.tar.gz --overwrite -m -p -C ${HOME}/.m2/repository
   echo "Removing old glassfish directory: ${S1AS_HOME}";
   rm -rf "${S1AS_HOME}";
 fi

appserver/tests/quicklook/run_test.sh

@@ -26,7 +26,7 @@ copy_ql_results(){
         cp ${WORKSPACE}/nucleus/domains/domain1/logs/server.log* ${WORKSPACE}/results
     fi
     cp ${TEST_RUN_LOG} ${WORKSPACE}/results/
-    tar -cf ${WORKSPACE}/${1}-results.tar.gz ${WORKSPACE}/results
+    tar -czf ${WORKSPACE}/${1}-results.tar.gz ${WORKSPACE}/results
     change_junit_report_class_names
 }
 

etc/docker/Dockerfile

@@ -56,7 +56,7 @@ ENV JAVA_HOME /usr/lib/jvm/jdk11
 ENV MAVEN_HOME /usr/share/maven
 ENV M2_HOME /usr/share/maven
 ENV ANT_HOME /usr/share/ant
-ENV JAVA_TOOL_OPTIONS "-Xmx2G"
+ENV JAVA_TOOL_OPTIONS "-Xmx2G -Xss768k"
 
 ENV HOME /home/jenkins
 WORKDIR /home/jenkins

gfbuild.sh

@@ -67,7 +67,7 @@ archive_bundles(){
 }
 
 dev_build(){
-  mvn -U clean install -Pstaging ${MVN_EXTRA}
+  mvn -U clean install -Pstaging,fastest -T2C ${MVN_EXTRA}
 }
 
 build_re_dev(){
@@ -83,7 +83,5 @@ fi
 "$@"
 
 if [ ! -z "${JENKINS_HOME}" ] ; then
-  # archive the local repository org.glassfish.main
-  # the output is a tar archive split into 1MB chunks.
-  tar -cz -f - -C ${HOME}/.m2/repository org/glassfish/main | split -b 1m - ${WORKSPACE}/bundles/_maven-repo
+  tar -c --exclude='*.zip' --exclude='*/main/tests/*' --exclude='*/main/distributions/*' --exclude='*/main/extras/*' --exclude='*/main/admingui/*' --newer-mtime '1 day ago' -C ${HOME}/.m2/repository org/glassfish/main | gzip --fast > ${WORKSPACE}/bundles/maven-repo.tar.gz
 fi