From ed576aa0945c38c84d6656a374bc30ac19c9b9bf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 30 Oct 2020 13:24:06 +0000 Subject: [PATCH 1/5] [CI] Report error in the catch section (#22297) --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 6fff22bfa09..ace40657f2c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -344,6 +344,7 @@ def withBeatsEnv(Map args = [:], Closure body) { } catch(err) { // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage upload = true + error("Error '${err.toString()}'") } finally { if (archive) { archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) From 2ad0b8823bf15cbaf8f6078b68e915aeec92fd9a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 30 Oct 2020 13:25:30 +0000 Subject: [PATCH 2/5] [packaging][beats-tester] use commit id binaries (#22027) --- .ci/beats-tester.groovy | 30 +++++++++++++++++++++--------- .ci/jobs/beats-tester.yml | 4 ++-- .ci/packaging.groovy | 38 +++++++++++++++++++++++++++++++++++--- Jenkinsfile | 7 +++++++ Makefile | 5 +++++ 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy index 91781a98d31..3650246e966 100644 --- a/.ci/beats-tester.groovy +++ b/.ci/beats-tester.groovy @@ -54,7 +54,6 @@ pipeline { options { skipDefaultCheckout() } when { branch 'master' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -62,7 +61,6 @@ pipeline { options { skipDefaultCheckout() } when { branch '*.x' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -79,7 +77,7 @@ pipeline { options { skipDefaultCheckout() } when { not { - allOf { + anyOf { branch comparator: 'REGEXP', pattern: '(master|.*x)' changeRequest() } @@ -96,14 +94,28 @@ pipeline { } def runBeatsTesterJob(Map args = [:]) { - if (args.apm && args.beats) { + def apm = args.get('apm', '') + def beats = args.get('beats', '') + def version = args.version + + if (isUpstreamTrigger()) { + copyArtifacts(filter: 'beats-tester.properties', + flatten: true, + projectName: "Beats/packaging/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'beats-tester.properties') + apm = props.get('APM_URL_BASE', '') + beats = props.get('BEATS_URL_BASE', '') + version = props.get('VERSION', '8.0.0-SNAPSHOT') + } + if (apm?.trim() || beats?.trim()) { build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ - string(name: 'APM_URL_BASE', value: args.apm), - string(name: 'BEATS_URL_BASE', value: args.beats), - string(name: 'VERSION', value: args.version) + string(name: 'APM_URL_BASE', value: apm), + string(name: 'BEATS_URL_BASE', value: beats), + string(name: 'VERSION', value: version) ]) } else { - build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ]) + build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ]) } -} \ No newline at end of file +} diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 808123a225e..14cc1007c56 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -1,8 +1,8 @@ --- - job: name: Beats/beats-tester - display-name: Beats Tester - description: Run the beats-tester + display-name: Beats Tester orchestrator + description: Orchestrate the beats-tester when packaging finished successfully view: Beats disabled: false project-type: multibranch diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index e86906fd8bb..02adc1a06b8 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -65,8 +65,29 @@ pipeline { options { skipDefaultCheckout() } steps { deleteDir() - gitCheckout(basedir: "${BASE_DIR}") + script { + if(isUpstreamTrigger()) { + try { + copyArtifacts(filter: 'packaging.properties', + flatten: true, + projectName: "Beats/beats/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'packaging.properties') + gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT) + } catch(err) { + // Fallback to the head of the branch as used to be. + gitCheckout(basedir: "${BASE_DIR}") + } + } else { + gitCheckout(basedir: "${BASE_DIR}") + } + } setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + withMageEnv(){ + dir("${BASE_DIR}"){ + setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) + } + } stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } @@ -167,6 +188,17 @@ pipeline { } } } + post { + success { + writeFile(file: 'beats-tester.properties', + text: """\ + ## To be consumed by the beats-tester pipeline + COMMIT=${env.GIT_BASE_COMMIT} + BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT} + VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/ + archiveArtifacts artifacts: 'beats-tester.properties' + } + } } } } @@ -192,7 +224,7 @@ def pushCIDockerImages(){ } def tagAndPush(beatName){ - def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim() + def libbetaVer = env.BEAT_VERSION def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version @@ -346,7 +378,7 @@ def publishPackages(baseDir){ uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) // Copy those files to another location with the sha commit to test them - // aftewords. + // afterward. bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}" uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) } diff --git a/Jenkinsfile b/Jenkinsfile index ace40657f2c..82bd3e0cef2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,6 +77,7 @@ pipeline { withGithubNotify(context: "Lint") { withBeatsEnv(archive: false, id: "lint") { dumpVariables() + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) cmd(label: "make check-python", script: "make check-python") cmd(label: "make check-go", script: "make check-go") cmd(label: "Check for changes", script: "make check-no-changes") @@ -123,6 +124,12 @@ pipeline { } } post { + success { + writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline +COMMIT=${env.GIT_BASE_COMMIT} +VERSION=${env.VERSION}-SNAPSHOT""") + archiveArtifacts artifacts: 'packaging.properties' + } always { deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") diff --git a/Makefile b/Makefile index b00cded4f9f..bd0da94f129 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,11 @@ python-env: test-apm: sh ./script/test_apm.sh +## get-version : Get the libbeat version +.PHONY: get-version +get-version: + @mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " " + ### Packaging targets #### ## snapshot : Builds a snapshot release. From f23f05a33fb570f7100a80d9ae31ab720ad96d1d Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 30 Oct 2020 07:26:39 -0600 Subject: [PATCH 3/5] Change cloud.provider from googlecloud to gcp in billing metricset (#22287) --- .../module/googlecloud/billing/_meta/data.json | 14 +++++--------- .../module/googlecloud/billing/billing.go | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json b/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json index 92ee77fb15b..5000711030a 100644 --- a/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/billing/_meta/data.json @@ -1,12 +1,8 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "cloud": { - "account": { - "id": "elastic-bi", - "name": "elastic-bi" - }, - "provider": "googlecloud" - }, + "cloud.account.id": "elastic-bi", + "cloud.account.name": "elastic-bi", + "cloud.provider": "gcp", "event": { "dataset": "googlecloud.billing", "duration": 115000, @@ -15,9 +11,9 @@ "googlecloud": { "billing": { "cost_type": "regular", - "invoice_month": "202008", + "invoice_month": "202010", "project_id": "elastic-fin-bi", - "total": 170.811692 + "total": 77.897328 } }, "metricset": { diff --git a/x-pack/metricbeat/module/googlecloud/billing/billing.go b/x-pack/metricbeat/module/googlecloud/billing/billing.go index a314df78100..16cdbccf090 100644 --- a/x-pack/metricbeat/module/googlecloud/billing/billing.go +++ b/x-pack/metricbeat/module/googlecloud/billing/billing.go @@ -278,7 +278,7 @@ func createEvents(rowItems []bigquery.Value, accountID string) mb.Event { } event.RootFields = common.MapStr{ - "cloud.provider": "googlecloud", + "cloud.provider": "gcp", "cloud.account.id": accountID, "cloud.account.name": accountID, } From 1900632eabb95f7d4ec154bd51f3e685bd23d631 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 30 Oct 2020 15:48:43 +0000 Subject: [PATCH 4/5] [CI] Support Windows-8 (#22307) --- auditbeat/Jenkinsfile.yml | 11 +++++++++++ filebeat/Jenkinsfile.yml | 11 +++++++++++ heartbeat/Jenkinsfile.yml | 11 +++++++++++ metricbeat/Jenkinsfile.yml | 11 +++++++++++ packetbeat/Jenkinsfile.yml | 11 +++++++++++ winlogbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/auditbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/elastic-agent/Jenkinsfile.yml | 11 +++++++++++ x-pack/filebeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/functionbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/metricbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/packetbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/winlogbeat/Jenkinsfile.yml | 11 +++++++++++ 13 files changed, 143 insertions(+) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 925b9ff0adc..7d272e68479 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -86,3 +86,14 @@ stages: - "windows-10" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test auditbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 322cea302c3..6cc3850fab7 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -73,4 +73,15 @@ stages: labels: - "windows-10" branches: true ## for all the branches + tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test filebeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches tags: true ## for all the tags \ No newline at end of file diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 9cebe5bbde2..9a261f40f1d 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -94,3 +94,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tag + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test heartbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tag diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index bdb9fd1a2bf..ffdef94a920 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -79,3 +79,14 @@ stages: - "windows-10" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test metricbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 96aff7bbfb2..e4e6ffb4a25 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -94,3 +94,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test packetbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index 3514c68c81e..43dd2b78ebb 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -58,3 +58,14 @@ stages: # - "windows-10" # branches: true ## for all the branches # tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test winlogbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 8b5f5d298dc..b2a270ef643 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -95,3 +95,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/auditbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index e3f33e109a9..a350f6a4923 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -94,3 +94,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/elastic-agent for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index e677f15d225..9dd1e17207b 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -95,3 +95,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/filebeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 7339a700f41..612feea132c 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -92,3 +92,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/functionbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 4d00157b2e6..0d7f1d3870e 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -84,3 +84,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/metricbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index 5f1ff2ce6b2..94adf826dc3 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -95,3 +95,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/packetbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 966e00f4999..49c1aaff301 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -69,3 +69,14 @@ stages: - "windows-2008" branches: true ## for all the branches tags: true ## for all the tags + windows-8: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-8" + when: ## Override the top-level when. + comments: + - "/test x-pack/winlogbeat for windows-8" + labels: + - "windows-8" + branches: true ## for all the branches + tags: true ## for all the tags From fc4d0093de7965fc330891a074e2aa7830630c27 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 30 Oct 2020 16:22:33 +0000 Subject: [PATCH 5/5] [JJBB] Add 6.8+ branches (#22321) --- .ci/jobs/beats.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index 27095b2fecb..54952b06e7c 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -35,6 +35,9 @@ - exact-name: name: 'master' case-sensitive: true + - regex-name: + regex: '6\.[x89]' + case-sensitive: true - regex-name: regex: '7\.[x789]' case-sensitive: true