Merge pull request #445 from lemeurherve/refactor-jenkinsfile

refactor(build): factorize Jenkinsfile steps
jenkinsci · Jul 7, 2023 · 4f712d2 · 4f712d2
2 parents 6e3201e + 19c1a1f
commit 4f712d2
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 135 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -6,49 +6,83 @@ pipeline {
     }
 
     stages {
-        stage('Build') {
+        stage('docker-agent') {
             failFast true
-            parallel {
-                stage('Windows') {
-                    agent {
-                        label "docker-windows"
+            matrix {
+                axes {
+                    axis {
+                        name 'AGENT_TYPE'
+                        values 'linux', 'windows-2019'
                     }
-                    options {
-                        timeout(time: 60, unit: 'MINUTES')
-                    }
-                    environment {
-                        DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
-                    }
-                    stages {
-                        stage('Build and Test') {
-                            // This stage is the "CI" and should be run on all code changes triggered by a code change
-                            when {
-                                not { buildingTag() }
-                            }
-                            steps {
-                                powershell '& ./build.ps1 test'
-                            }
-                            post {
-                                always {
-                                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'target/**/junit-results.xml')
+                }
+                stages {
+                    stage('Main') {
+                        agent {
+                            label env.AGENT_TYPE
+                        }
+                        options {
+                            timeout(time: 30, unit: 'MINUTES')
+                        }
+                        environment {
+                            DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
+                        }
+                        stages {
+                            stage('Prepare Docker') {
+                                when {
+                                    environment name: 'AGENT_TYPE', value: 'linux'
+                                }
+                                steps {
+                                    sh '''
+                                    docker buildx create --use
+                                    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+                                    '''
                                 }
                             }
-                        }
-                        stage('Deploy to DockerHub') {
-                            // This stage is the "CD" and should only be run when a tag triggered the build
-                            when {
-                                buildingTag()
+                            stage('Build and Test') {
+                                // This stage is the "CI" and should be run on all code changes triggered by a code change
+                                when {
+                                    not { buildingTag() }
+                                }
+                                steps {
+                                    script {
+                                        if(isUnix()) {
+                                            sh './build.sh'
+                                            sh './build.sh test'
+                                            // If the tests are passing for Linux AMD64, then we can build all the CPU architectures
+                                            sh 'docker buildx bake --file docker-bake.hcl linux'
+                                        } else {
+                                            powershell "& ./build.ps1 test"
+                                        }
+                                    }
+                                }
+                                post {
+                                    always {
+                                        junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'target/**/junit-results.xml')
+                                    }
+                                }
                             }
-                            steps {
-                                script {
-                                    if(env.TAG_NAME != null) {
+                            stage('Deploy to DockerHub') {
+                                // This stage is the "CD" and should only be run when a tag triggered the build
+                                when {
+                                    buildingTag()
+                                }
+                                steps {
+                                    script {
                                         def tagItems = env.TAG_NAME.split('-')
                                         if(tagItems.length == 2) {
                                             def remotingVersion = tagItems[0]
                                             def buildNumber = tagItems[1]
                                             // This function is defined in the jenkins-infra/pipeline-library
                                             infra.withDockerCredentials {
-                                                powershell "& ./build.ps1 -PushVersions -RemotingVersion $remotingVersion -BuildNumber $buildNumber -DisableEnvProps publish"
+                                                if (isUnix()) {
+                                                    sh """
+                                                    docker buildx create --use
+                                                    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+                                                    ./build.sh -r ${remotingVersion} -b ${buildNumber} -d publish
+                                                    """
+                                                } else {
+                                                    powershell "& ./build.ps1 -PushVersions -RemotingVersion $remotingVersion -BuildNumber $buildNumber -DisableEnvProps publish"
+                                                }
                                             }
                                         }
                                     }
@@ -57,67 +91,6 @@ pipeline {
                         }
                     }
                 }
-                stage('Linux') {
-                    agent {
-                        label "docker && linux"
-                    }
-                    options {
-                        timeout(time: 30, unit: 'MINUTES')
-                    }
-                    environment {
-                        DOCKERHUB_ORGANISATION = "${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"
-                    }
-                    stages {
-                        stage('Prepare Docker') {
-                            steps {
-                                sh '''
-                                docker buildx create --use
-                                docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-                                '''
-                            }
-                        }
-                        stage('Build and Test') {
-                            // This stage is the "CI" and should be run on all code changes triggered by a code change
-                            when {
-                                not { buildingTag() }
-                            }
-                            steps {
-                                sh './build.sh'
-                                sh './build.sh test'
-                                // If the tests are passing for Linux AMD64, then we can build all the CPU architectures
-                                sh 'docker buildx bake --file docker-bake.hcl linux'
-                            }
-                            post {
-                                always {
-                                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'target/*.xml')
-                                }
-                            }
-                        }
-                        stage('Deploy to DockerHub') {
-                            // This stage is the "CD" and should only be run when a tag triggered the build
-                            when {
-                                buildingTag()
-                            }
-                            steps {
-                                script {
-                                    def tagItems = env.TAG_NAME.split('-')
-                                    if(tagItems.length == 2) {
-                                        def remotingVersion = tagItems[0]
-                                        def buildNumber = tagItems[1]
-                                        // This function is defined in the jenkins-infra/pipeline-library
-                                        infra.withDockerCredentials {
-                                            sh """
-                                            docker buildx create --use
-                                            docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-                                            ./build.sh -r ${remotingVersion} -b ${buildNumber} -d publish
-                                            """
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
             }
         }
     }

diff --git a/build-windows.yaml b/build-windows.yaml
@@ -1,33 +1,37 @@
 services:
-  jdk11-nanoserver-1809:
-    image: jdk11-nanoserver-1809
+  jdk11-nanoserver:
+    image: jdk11-nanoserver-${NANOSERVER_VERSION_NAME}
     build:
-      context: ./windows/nanoserver-1809/
+      context: ./windows/nanoserver/
       args:
         JAVA_HOME: "C:/openjdk-11"
         JAVA_VERSION: "11.0.19_7"
         VERSION: ${REMOTING_VERSION}
-  jdk17-nanoserver-1809:
-    image: jdk17-nanoserver-1809
+        WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
+  jdk17-nanoserver:
+    image: jdk17-nanoserver-${NANOSERVER_VERSION_NAME}
     build:
-      context: ./windows/nanoserver-1809/
+      context: ./windows/nanoserver/
       args:
         JAVA_HOME: "C:/openjdk-17"
         JAVA_VERSION: "17.0.7_7"
         VERSION: ${REMOTING_VERSION}
-  jdk11-windowsservercore-ltsc2019:
-    image: jdk11-windowsservercore-ltsc2019
+        WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
+  jdk11-windowsservercore:
+    image: jdk11-windowsservercore-${WINDOWS_VERSION_NAME}
     build:
-      context: ./windows/windowsservercore-ltsc2019/
+      context: ./windows/windowsservercore/
       args:
         JAVA_HOME: "C:/openjdk-11"
         JAVA_VERSION: "11.0.19_7"
         VERSION: ${REMOTING_VERSION}
-  jdk17-windowsservercore-ltsc2019:
-    image: jdk17-windowsservercore-ltsc2019
+        WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
+  jdk17-windowsservercore:
+    image: jdk17-windowsservercore-${WINDOWS_VERSION_NAME}
     build:
-      context: ./windows/windowsservercore-ltsc2019/
+      context: ./windows/windowsservercore/
       args:
         JAVA_HOME: "C:/openjdk-17"
         JAVA_VERSION: "17.0.7_7"
         VERSION: ${REMOTING_VERSION}
+        WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
diff --git a/build.ps1 b/build.ps1
@@ -9,9 +9,10 @@ Param(
     [switch] $DisableEnvProps = $false
 )
 
-$ErrorActionPreference = "Stop"
+$ErrorActionPreference = 'Stop'
 $Repository = 'agent'
 $Organization = 'jenkins'
+$AgentType = 'windows-2019'
 
 if(!$DisableEnvProps) {
     Get-Content env.props | ForEach-Object {
@@ -36,6 +37,10 @@ if(![String]::IsNullOrWhiteSpace($env:REMOTING_VERSION)) {
     $RemotingVersion = $env:REMOTING_VERSION
 }
 
+if(![String]::IsNullOrWhiteSpace($env:AGENT_TYPE)) {
+    $AgentType = $env:AGENT_TYPE
+}
+
 # Check for required commands
 Function Test-CommandExists {
     # From https://devblogs.microsoft.com/scripting/use-a-powershell-function-to-see-if-a-command-exists/
@@ -62,6 +67,15 @@ Function Test-CommandExists {
 $defaultJdk = '11'
 $builds = @{}
 $env:REMOTING_VERSION = "$RemotingVersion"
+$env:WINDOWS_VERSION_NAME = $AgentType.replace('windows-', 'ltsc')
+$env:NANOSERVER_VERSION_NAME = $env:WINDOWS_VERSION_NAME
+$env:WINDOWS_VERSION_TAG = $env:WINDOWS_VERSION_NAME
+# Unconsistent naming for the 2019 version, needed as while nanoserver-ltsc2019 and windowsserver-ltsc2019 tags exist eclipse-temurin:<...>-ltsc2019 does not
+# We also need to keep the `jdkN-nanoserver-1809` images for now, cf https://github.com/jenkinsci/docker-agent/issues/451
+if ($AgentType -eq 'windows-2019') {
+    $env:WINDOWS_VERSION_TAG = 1809
+    $env:NANOSERVER_VERSION_NAME = 1809
+}
 $ProgressPreference = 'SilentlyContinue' # Disable Progress bar for faster downloads
 
 Test-CommandExists "docker"
@@ -72,7 +86,9 @@ $baseDockerCmd = 'docker-compose --file=build-windows.yaml'
 $baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
 
 Invoke-Expression "$baseDockerCmd config --services" 2>$null | ForEach-Object {
-    $image = $_
+    $image = '{0}-{1}' -f $_, $env:WINDOWS_VERSION_NAME
+    # Special case for nanoserver-1809 images
+    $image = $image.replace('nanoserver-ltsc2019', 'nanoserver-1809')
     $items = $image.Split("-")
     $jdkMajorVersion = $items[0].Remove(0,3)
     $windowsType = $items[1]
@@ -113,7 +129,8 @@ function Test-Image {
     Write-Host "= TEST: Testing image ${ImageName}:"
 
     $env:AGENT_IMAGE = $ImageName
-    $env:IMAGE_FOLDER = Invoke-Expression "$baseDockerCmd config" 2>$null |  yq -r ".services.${ImageName}.build.context"
+    $serviceName = $ImageName.SubString(0, $ImageName.LastIndexOf('-'))
+    $env:IMAGE_FOLDER = Invoke-Expression "$baseDockerCmd config" 2>$null |  yq -r ".services.${serviceName}.build.context"
     $env:VERSION = "$RemotingVersion-$BuildNumber"
 
 
@@ -166,6 +183,7 @@ if($target -eq "test") {
     if(![System.String]::IsNullOrWhiteSpace($Build) -and $builds.ContainsKey($Build)) {
         Test-Image $Build
     } else {
+        Write-Host "= TEST: Testing all images"
         foreach($image in $builds.Keys) {
             Test-Image $image
         }

diff --git a/tests/test_helpers.psm1 b/tests/test_helpers.psm1
@@ -115,7 +115,7 @@ function Run-Program($cmd, $params) {
     $stderr = $proc.StandardError.ReadToEnd()
     $proc.WaitForExit()
     if($proc.ExitCode -ne 0) {
-        Write-Host "`n`nstdout:`n$stdout`n`nstderr:`n$stderr`n`n"
+        Write-Host "`n`nstdout:`n$stdout`n`nstderr:`n$stderr`n$cmd`n`nparams:`n$param`n`n"
     }
 
     return $proc.ExitCode, $stdout, $stderr

diff --git a/updatecli/updatecli.d/git-windows.yaml b/updatecli/updatecli.d/git-windows.yaml
@@ -30,56 +30,54 @@ sources:
       - trimprefix: "v"
 
 targets:
-  ############# Nanoserver 18.09
-  setGitVersionWindowsNanoserver1809:
-    name: Update the Git Windows version for Windows Nanoserver 1809
+  setGitVersionNanoserver:
+    name: Update the Git Windows version for Windows Nanoserver
     transformers:
       - findsubmatch:
           pattern: '(.*).windows\.(\d*)$'
           captureindex: 1
     kind: dockerfile
     spec:
-      file: windows/nanoserver-1809/Dockerfile
+      file: windows/nanoserver/Dockerfile
       instruction:
         keyword: ARG
         matcher: GIT_VERSION
     scmid: default
-  setGitPackagePatchWindowsNanoserver1809:
-    name: Update the Git Package Windows patch for Windows Nanoserver 1809
+  setGitPackagePatchNanoserver:
+    name: Update the Git Package Windows patch for Windows Nanoserver
     transformers:
       - findsubmatch:
           pattern: '(.*).windows\.(\d*)$'
           captureindex: 2
     kind: dockerfile
     spec:
-      file: windows/nanoserver-1809/Dockerfile
+      file: windows/nanoserver/Dockerfile
       instruction:
         keyword: ARG
         matcher: GIT_PATCH_VERSION
     scmid: default
-  ############# Windows Server Core LTSC2019
-  setGitVersionWindowsServer2019:
-    name: Update the Git Windows version for Windows Server Core LTSC2019
+  setGitVersionWindowsServerCore:
+    name: Update the Git Windows version for Windows Server Core
     transformers:
       - findsubmatch:
           pattern: '(.*).windows\.(\d*)$'
           captureindex: 1
     kind: dockerfile
     spec:
-      file: windows/windowsservercore-ltsc2019/Dockerfile
+      file: windows/windowsservercore/Dockerfile
       instruction:
         keyword: ARG
         matcher: GIT_VERSION
     scmid: default
-  setGitPackagePatchWindowsServer2019:
-    name: Update the Git Package Windows patch for Windows Server Core LTSC2019
+  setGitPackagePatchWindowsServerCore:
+    name: Update the Git Package Windows patch for Windows Server Core
     transformers:
       - findsubmatch:
           pattern: '(.*).windows\.(\d*)$'
           captureindex: 2
     kind: dockerfile
     spec:
-      file: windows/windowsservercore-ltsc2019/Dockerfile
+      file: windows/windowsservercore/Dockerfile
       instruction:
         keyword: ARG
         matcher: GIT_PATCH_VERSION

diff --git a/updatecli/updatecli.d/jdk11.yaml b/updatecli/updatecli.d/jdk11.yaml
@@ -54,7 +54,15 @@ conditions:
         - arm/v7
       image: eclipse-temurin
       tag: '{{source "lastVersion" }}-jdk-focal'
-  checkTemurinWindowsCoreDockerImage:
+  checkTemurinNanocore2019DockerImage:
+    kind: dockerimage
+    name: Check if the container image "eclipse-temurin:<lastVersion>-jdk-windowsservercore-1809" is available
+    disablesourceinput: true
+    spec:
+      architecture: amd64
+      image: eclipse-temurin
+      tag: '{{source "lastVersion" }}-jdk-nanoserver-1809'
+  checkTemurinWindowsCore2019DockerImage:
     kind: dockerimage
     name: Check if the container image "eclipse-temurin:<lastVersion>-jdk-windowsservercore-1809" is available
     disablesourceinput: true