Merge remote-tracking branch 'upstream/master' into feature/windows-7-64

* upstream/master: [JJBB] Add 6.8+ branches (elastic#22321) [CI] Support Windows-8 (elastic#22307) Change cloud.provider from googlecloud to gcp in billing metricset (elastic#22287) [packaging][beats-tester] use commit id binaries (elastic#22027) [CI] Report error in the catch section (elastic#22297)
v1v · Oct 30, 2020 · 75dc72f · 75dc72f
2 parents cb080ee + fc4d009
commit 75dc72f
Show file tree

Hide file tree

Showing 21 changed files with 227 additions and 28 deletions.
diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy
@@ -54,15 +54,13 @@ pipeline {
           options { skipDefaultCheckout() }
           when { branch 'master' }
           steps {
-            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }
         stage('Build *.x branch') {
           options { skipDefaultCheckout() }
           when { branch '*.x' }
           steps {
-            // TODO: to use the git commit that triggered the upstream build
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
           }
         }
@@ -79,7 +77,7 @@ pipeline {
           options { skipDefaultCheckout() }
           when {
             not {
-              allOf {
+              anyOf {
                 branch comparator: 'REGEXP', pattern: '(master|.*x)'
                 changeRequest()
               }
@@ -96,14 +94,28 @@ pipeline {
 }
 
 def runBeatsTesterJob(Map args = [:]) {
-  if (args.apm && args.beats) {
+  def apm = args.get('apm', '')
+  def beats = args.get('beats', '')
+  def version = args.version
+
+  if (isUpstreamTrigger()) {
+    copyArtifacts(filter: 'beats-tester.properties',
+                  flatten: true,
+                  projectName: "Beats/packaging/${env.JOB_BASE_NAME}",
+                  selector: upstream(fallbackToLastSuccessful: true))
+    def props = readProperties(file: 'beats-tester.properties')
+    apm = props.get('APM_URL_BASE', '')
+    beats = props.get('BEATS_URL_BASE', '')
+    version = props.get('VERSION', '8.0.0-SNAPSHOT')
+  }
+  if (apm?.trim() || beats?.trim()) {
     build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,
           parameters: [
-            string(name: 'APM_URL_BASE', value: args.apm),
-            string(name: 'BEATS_URL_BASE', value: args.beats),
-            string(name: 'VERSION', value: args.version)
+            string(name: 'APM_URL_BASE', value: apm),
+            string(name: 'BEATS_URL_BASE', value: beats),
+            string(name: 'VERSION', value: version)
           ])
   } else {
-    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ])
+    build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ])
   }
-}
+}
diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml
@@ -1,8 +1,8 @@
 ---
 - job:
     name: Beats/beats-tester
-    display-name: Beats Tester
-    description: Run the beats-tester
+    display-name: Beats Tester orchestrator
+    description: Orchestrate the beats-tester when packaging finished successfully
     view: Beats
     disabled: false
     project-type: multibranch

diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -35,6 +35,9 @@
             - exact-name:
                 name: 'master'
                 case-sensitive: true
+            - regex-name:
+                regex: '6\.[x89]'
+                case-sensitive: true
             - regex-name:
                 regex: '7\.[x789]'
                 case-sensitive: true

diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -65,8 +65,29 @@ pipeline {
           options { skipDefaultCheckout() }
           steps {
             deleteDir()
-            gitCheckout(basedir: "${BASE_DIR}")
+            script {
+              if(isUpstreamTrigger()) {
+                try {
+                  copyArtifacts(filter: 'packaging.properties',
+                                flatten: true,
+                                projectName: "Beats/beats/${env.JOB_BASE_NAME}",
+                                selector: upstream(fallbackToLastSuccessful: true))
+                  def props = readProperties(file: 'packaging.properties')
+                  gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT)
+                } catch(err) {
+                  // Fallback to the head of the branch as used to be.
+                  gitCheckout(basedir: "${BASE_DIR}")
+                }
+              } else {
+                gitCheckout(basedir: "${BASE_DIR}")
+              }
+            }
             setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+            withMageEnv(){
+              dir("${BASE_DIR}"){
+                setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
+              }
+            }
             stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
           }
         }
@@ -167,6 +188,17 @@ pipeline {
           }
         }
       }
+      post {
+        success {
+          writeFile(file: 'beats-tester.properties',
+                    text: """\
+                    ## To be consumed by the beats-tester pipeline
+                    COMMIT=${env.GIT_BASE_COMMIT}
+                    BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}
+                    VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/
+          archiveArtifacts artifacts: 'beats-tester.properties'
+        }
+      }
     }
   }
 }
@@ -192,7 +224,7 @@ def pushCIDockerImages(){
 }
 
 def tagAndPush(beatName){
-  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+  def libbetaVer = env.BEAT_VERSION
   def aliasVersion = ""
   if("${env.SNAPSHOT}" == "true"){
     aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
@@ -346,7 +378,7 @@ def publishPackages(baseDir){
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 
   // Copy those files to another location with the sha commit to test them
-  // aftewords.
+  // afterward.
   bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}"
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 }

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -77,6 +77,7 @@ pipeline {
         withGithubNotify(context: "Lint") {
           withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
+            setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
             cmd(label: "make check-python", script: "make check-python")
             cmd(label: "make check-go", script: "make check-go")
             cmd(label: "Check for changes", script: "make check-no-changes")
@@ -123,6 +124,12 @@ pipeline {
     }
   }
   post {
+    success {
+      writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline
+COMMIT=${env.GIT_BASE_COMMIT}
+VERSION=${env.VERSION}-SNAPSHOT""")
+      archiveArtifacts artifacts: 'packaging.properties'
+    }
     always {
       deleteDir()
       unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
@@ -344,6 +351,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
       } catch(err) {
         // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage
         upload = true
+        error("Error '${err.toString()}'")
       } finally {
         if (archive) {
           archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)

diff --git a/Makefile b/Makefile
@@ -191,6 +191,11 @@ python-env:
 test-apm:
 	sh ./script/test_apm.sh
 
+## get-version : Get the libbeat version
+.PHONY: get-version
+get-version:
+	@mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " "
+
 ### Packaging targets ####
 
 ## snapshot : Builds a snapshot release.

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -86,7 +86,18 @@ stages:
                 - "windows-10"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
-    # windows-7:  See https://github.com/elastic/beats/issues/19831
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test auditbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
+    #windows-7:  See https://github.com/elastic/beats/issues/19831
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.
     #        - "windows-7"

diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -74,7 +74,18 @@ stages:
                 - "windows-10"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
-    # windows-7:  See https://github.com/elastic/beats/issues/22317
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test filebeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
+    #windows-7:  See https://github.com/elastic/beats/issues/22317
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.
     #        - "windows-7"

diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -94,6 +94,17 @@ stages:
                 - "windows-2008"
             branches: true     ## for all the branches
             tags: true         ## for all the tag
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test heartbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tag
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
@@ -79,6 +79,17 @@ stages:
                 - "windows-10"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test metricbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
@@ -94,6 +94,17 @@ stages:
                 - "windows-2008"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test packetbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml
@@ -47,7 +47,7 @@ stages:
                 - "windows-2012"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
-    # windows-10:  See https://github.com/elastic/beats/issues/22046
+    #windows-10:  See https://github.com/elastic/beats/issues/22046
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.
     #        - "windows-10"
@@ -58,7 +58,7 @@ stages:
     #            - "windows-10"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
-    # windows-7:  See https://github.com/elastic/beats/issues/22302
+    #windows-7:  See https://github.com/elastic/beats/issues/22302
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.
     #        - "windows-7"
@@ -69,3 +69,14 @@ stages:
     #            - "windows-7"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test winlogbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
@@ -95,6 +95,17 @@ stages:
                 - "windows-2008"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/auditbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -94,6 +94,17 @@ stages:
                 - "windows-2008"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/elastic-agent for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
@@ -95,6 +95,17 @@ stages:
                 - "windows-2008"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/filebeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.

diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
@@ -92,6 +92,17 @@ stages:
                 - "windows-2008"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+    windows-8:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-8"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/functionbeat for windows-8"
+            labels:
+                - "windows-8"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     windows-7:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.