From 7d2e42c038220d23e6f87504722b93d29efc015e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 9 Nov 2020 09:15:42 +0000 Subject: [PATCH] [packaging][beats-tester] use commit id binaries (#22027) (#22471) --- .ci/beats-tester.groovy | 30 +++++++++++++++++++--------- .ci/jobs/beats-tester.yml | 4 ++-- .ci/packaging.groovy | 41 ++++++++++++++++++++++++++++++++++----- Jenkinsfile | 7 +++++++ Makefile | 5 +++++ 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy index 91781a98d31..3650246e966 100644 --- a/.ci/beats-tester.groovy +++ b/.ci/beats-tester.groovy @@ -54,7 +54,6 @@ pipeline { options { skipDefaultCheckout() } when { branch 'master' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -62,7 +61,6 @@ pipeline { options { skipDefaultCheckout() } when { branch '*.x' } steps { - // TODO: to use the git commit that triggered the upstream build runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") } } @@ -79,7 +77,7 @@ pipeline { options { skipDefaultCheckout() } when { not { - allOf { + anyOf { branch comparator: 'REGEXP', pattern: '(master|.*x)' changeRequest() } @@ -96,14 +94,28 @@ pipeline { } def runBeatsTesterJob(Map args = [:]) { - if (args.apm && args.beats) { + def apm = args.get('apm', '') + def beats = args.get('beats', '') + def version = args.version + + if (isUpstreamTrigger()) { + copyArtifacts(filter: 'beats-tester.properties', + flatten: true, + projectName: "Beats/packaging/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'beats-tester.properties') + apm = props.get('APM_URL_BASE', '') + beats = props.get('BEATS_URL_BASE', '') + version = props.get('VERSION', '8.0.0-SNAPSHOT') + } + if (apm?.trim() || beats?.trim()) { build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ - string(name: 'APM_URL_BASE', value: args.apm), - string(name: 'BEATS_URL_BASE', value: args.beats), - string(name: 'VERSION', value: args.version) + string(name: 'APM_URL_BASE', value: apm), + string(name: 'BEATS_URL_BASE', value: beats), + string(name: 'VERSION', value: version) ]) } else { - build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ]) + build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ]) } -} \ No newline at end of file +} diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 522abfa5e5c..ceb7a6955ad 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -1,8 +1,8 @@ --- - job: name: Beats/beats-tester - display-name: Beats Tester - description: Run the beats-tester + display-name: Beats Tester orchestrator + description: Orchestrate the beats-tester when packaging finished successfully view: Beats disabled: false project-type: multibranch diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 3d38d0d27a2..d7418c458fc 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -55,9 +55,29 @@ pipeline { stage('Checkout') { options { skipDefaultCheckout() } steps { - deleteDir() - gitCheckout(basedir: "${BASE_DIR}") + script { + if(isUpstreamTrigger()) { + try { + copyArtifacts(filter: 'packaging.properties', + flatten: true, + projectName: "Beats/beats/${env.JOB_BASE_NAME}", + selector: upstream(fallbackToLastSuccessful: true)) + def props = readProperties(file: 'packaging.properties') + gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT) + } catch(err) { + // Fallback to the head of the branch as used to be. + gitCheckout(basedir: "${BASE_DIR}") + } + } else { + gitCheckout(basedir: "${BASE_DIR}") + } + } setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + withMageEnv(){ + dir("${BASE_DIR}"){ + setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) + } + } stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } @@ -150,6 +170,17 @@ pipeline { } } } + post { + success { + writeFile(file: 'beats-tester.properties', + text: """\ + ## To be consumed by the beats-tester pipeline + COMMIT=${env.GIT_BASE_COMMIT} + BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT} + VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/ + archiveArtifacts artifacts: 'beats-tester.properties' + } + } } } } @@ -185,7 +216,7 @@ def pushCIDockerImages(){ } def tagAndPush(name){ - def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim() + def libbetaVer = env.BEAT_VERSION if("${env.SNAPSHOT}" == "true"){ libbetaVer += "-SNAPSHOT" } @@ -237,7 +268,7 @@ def publishPackages(baseDir){ uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) // Copy those files to another location with the sha commit to test them - // aftewords. + // afterward. bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}" uploadPackages("${bucketUri}/${beatsFolderName}", baseDir) } @@ -275,4 +306,4 @@ def withBeatsEnv(Closure body) { } } } -} +} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 08608d0139b..60b7d0bc2ca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -78,6 +78,7 @@ pipeline { withGithubNotify(context: "Lint") { withBeatsEnv(archive: false, id: "lint") { dumpVariables() + setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) cmd(label: "make check-python", script: "make check-python") cmd(label: "make check-go", script: "make check-go") cmd(label: "Check for changes", script: "make check-no-changes") @@ -124,6 +125,12 @@ pipeline { } } post { + success { + writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline +COMMIT=${env.GIT_BASE_COMMIT} +VERSION=${env.VERSION}-SNAPSHOT""") + archiveArtifacts artifacts: 'packaging.properties' + } always { deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") diff --git a/Makefile b/Makefile index 59a4d589bce..5396253543e 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,11 @@ python-env: test-apm: sh ./script/test_apm.sh +## get-version : Get the libbeat version +.PHONY: get-version +get-version: + @mage dumpVariables | grep 'beat_version' | cut -d"=" -f 2 | tr -d " " + ### Packaging targets #### ## snapshot : Builds a snapshot release.