-
Notifications
You must be signed in to change notification settings - Fork 8.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into ML-new-apm-jobs
- Loading branch information
Showing
1,239 changed files
with
18,967 additions
and
14,096 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| #!/bin/groovy | ||
|
|
||
| // This job effectively has two SCM configurations: | ||
| // one for kibana, used to check out this Jenkinsfile (which means it's the job's main SCM configuration), as well as kick-off the downstream verification job | ||
| // one for elasticsearch, used to check out the elasticsearch source before building it | ||
|
|
||
| // There are two parameters that drive which branch is checked out for each of these, but they will typically be the same | ||
| // 'branch_specifier' is for kibana / the job itself | ||
| // ES_BRANCH is for elasticsearch | ||
|
|
||
| library 'kibana-pipeline-library' | ||
| kibanaLibrary.load() | ||
|
|
||
| def ES_BRANCH = params.ES_BRANCH | ||
|
|
||
| if (!ES_BRANCH) { | ||
| error "Parameter 'ES_BRANCH' must be specified." | ||
| } | ||
|
|
||
| currentBuild.displayName += " - ${ES_BRANCH}" | ||
| currentBuild.description = "ES: ${ES_BRANCH}<br />Kibana: ${params.branch_specifier}" | ||
|
|
||
| def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION | ||
|
|
||
| timeout(time: 120, unit: 'MINUTES') { | ||
| timestamps { | ||
| ansiColor('xterm') { | ||
| node('linux && immutable') { | ||
| catchError { | ||
| def VERSION | ||
| def SNAPSHOT_ID | ||
| def DESTINATION | ||
|
|
||
| def scmVars = checkoutEs(ES_BRANCH) | ||
| def GIT_COMMIT = scmVars.GIT_COMMIT | ||
| def GIT_COMMIT_SHORT = sh(script: "git rev-parse --short ${GIT_COMMIT}", returnStdout: true).trim() | ||
|
|
||
| buildArchives('to-archive') | ||
|
|
||
| dir('to-archive') { | ||
| def now = new Date() | ||
| def date = now.format("yyyyMMdd-HHmmss") | ||
|
|
||
| def filesRaw = sh(script: "ls -1", returnStdout: true).trim() | ||
| def files = filesRaw | ||
| .split("\n") | ||
| .collect { filename -> | ||
| // Filename examples | ||
| // elasticsearch-oss-8.0.0-SNAPSHOT-linux-x86_64.tar.gz | ||
| // elasticsearch-8.0.0-SNAPSHOT-linux-x86_64.tar.gz | ||
| def parts = filename.replace("elasticsearch-oss", "oss").split("-") | ||
|
|
||
| VERSION = VERSION ?: parts[1] | ||
| SNAPSHOT_ID = SNAPSHOT_ID ?: "${date}_${GIT_COMMIT_SHORT}" | ||
| DESTINATION = DESTINATION ?: "${VERSION}/archives/${SNAPSHOT_ID}" | ||
|
|
||
| return [ | ||
| filename: filename, | ||
| checksum: filename + '.sha512', | ||
| url: "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${DESTINATION}/${filename}".toString(), | ||
| version: parts[1], | ||
| platform: parts[3], | ||
| architecture: parts[4].split('\\.')[0], | ||
| license: parts[0] == 'oss' ? 'oss' : 'default', | ||
| ] | ||
| } | ||
|
|
||
| sh 'find * -exec bash -c "shasum -a 512 {} > {}.sha512" \\;' | ||
|
|
||
| def manifest = [ | ||
| bucket: "kibana-ci-es-snapshots-daily/${DESTINATION}".toString(), | ||
| branch: ES_BRANCH, | ||
| sha: GIT_COMMIT, | ||
| sha_short: GIT_COMMIT_SHORT, | ||
| version: VERSION, | ||
| generated: now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC")), | ||
| archives: files, | ||
| ] | ||
| def manifestJson = toJSON(manifest).toString() | ||
| writeFile file: 'manifest.json', text: manifestJson | ||
|
|
||
| upload(DESTINATION, '*.*') | ||
|
|
||
| sh "cp manifest.json manifest-latest.json" | ||
| upload(VERSION, 'manifest-latest.json') | ||
| } | ||
|
|
||
| if (PROMOTE_WITHOUT_VERIFY) { | ||
| esSnapshots.promote(VERSION, SNAPSHOT_ID) | ||
|
|
||
| emailext( | ||
| to: 'build-kibana@elastic.co', | ||
| subject: "ES snapshot promoted without verification: ${params.ES_BRANCH}", | ||
| body: '${SCRIPT,template="groovy-html.template"}', | ||
| mimeType: 'text/html', | ||
| ) | ||
| } else { | ||
| build( | ||
| propagate: false, | ||
| wait: false, | ||
| job: 'elasticsearch+snapshots+verify', | ||
| parameters: [ | ||
| string(name: 'branch_specifier', value: branch_specifier), | ||
| string(name: 'SNAPSHOT_VERSION', value: VERSION), | ||
| string(name: 'SNAPSHOT_ID', value: SNAPSHOT_ID), | ||
| ] | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| kibanaPipeline.sendMail() | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| def checkoutEs(branch) { | ||
| retryWithDelay(8, 15) { | ||
| return checkout([ | ||
| $class: 'GitSCM', | ||
| branches: [[name: branch]], | ||
| doGenerateSubmoduleConfigurations: false, | ||
| extensions: [], | ||
| submoduleCfg: [], | ||
| userRemoteConfigs: [[ | ||
| credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', | ||
| url: 'git@github.com:elastic/elasticsearch', | ||
| ]], | ||
| ]) | ||
| } | ||
| } | ||
|
|
||
| def upload(destination, pattern) { | ||
| return googleStorageUpload( | ||
| credentialsId: 'kibana-ci-gcs-plugin', | ||
| bucket: "gs://kibana-ci-es-snapshots-daily/${destination}", | ||
| pattern: pattern, | ||
| sharedPublicly: false, | ||
| showInline: false, | ||
| ) | ||
| } | ||
|
|
||
| def buildArchives(destination) { | ||
| def props = readProperties file: '.ci/java-versions.properties' | ||
| withEnv([ | ||
| // Select the correct JDK for this branch | ||
| "PATH=/var/lib/jenkins/.java/${props.ES_BUILD_JAVA}/bin:${env.PATH}", | ||
|
|
||
| // These Jenkins env vars trigger some automation in the elasticsearch repo that we don't want | ||
| "BUILD_NUMBER=", | ||
| "JENKINS_URL=", | ||
| "BUILD_URL=", | ||
| "JOB_NAME=", | ||
| "NODE_NAME=", | ||
| ]) { | ||
| sh """ | ||
| ./gradlew -p distribution/archives assemble --parallel | ||
| mkdir -p ${destination} | ||
| find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\; | ||
| """ | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| #!/bin/groovy | ||
|
|
||
| if (!params.branches_yaml) { | ||
| error "'branches_yaml' parameter must be specified" | ||
| } | ||
|
|
||
| def branches = readYaml text: params.branches_yaml | ||
|
|
||
| branches.each { branch -> | ||
| build( | ||
| propagate: false, | ||
| wait: false, | ||
| job: 'elasticsearch+snapshots+build', | ||
| parameters: [ | ||
| string(name: 'branch_specifier', value: branch), | ||
| string(name: 'ES_BRANCH', value: branch), | ||
| ] | ||
| ) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| #!/bin/groovy | ||
|
|
||
| library 'kibana-pipeline-library' | ||
| kibanaLibrary.load() | ||
|
|
||
| def SNAPSHOT_VERSION = params.SNAPSHOT_VERSION | ||
| def SNAPSHOT_ID = params.SNAPSHOT_ID | ||
|
|
||
| if (!SNAPSHOT_VERSION) { | ||
| error "Parameter SNAPSHOT_VERSION must be specified" | ||
| } | ||
|
|
||
| if (!SNAPSHOT_ID) { | ||
| error "Parameter SNAPSHOT_ID must be specified" | ||
| } | ||
|
|
||
| currentBuild.displayName += " - ${SNAPSHOT_VERSION}" | ||
| currentBuild.description = "ES: ${SNAPSHOT_VERSION}<br />Kibana: ${params.branch_specifier}" | ||
|
|
||
| def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" | ||
|
|
||
| timeout(time: 120, unit: 'MINUTES') { | ||
| timestamps { | ||
| ansiColor('xterm') { | ||
| catchError { | ||
| withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { | ||
| parallel([ | ||
| // TODO we just need to run integration tests from intake? | ||
| 'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'), | ||
| 'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'), | ||
| 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ | ||
| 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), | ||
| 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), | ||
| 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), | ||
| 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), | ||
| 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), | ||
| 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), | ||
| 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), | ||
| 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), | ||
| 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), | ||
| 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), | ||
| 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), | ||
| 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), | ||
| ]), | ||
| 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ | ||
| 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), | ||
| 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), | ||
| 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), | ||
| 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), | ||
| 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), | ||
| 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), | ||
| 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), | ||
| 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), | ||
| 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), | ||
| 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), | ||
| ]), | ||
| ]) | ||
| } | ||
|
|
||
| promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID) | ||
| } | ||
|
|
||
| kibanaPipeline.sendMail() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| def promoteSnapshot(snapshotVersion, snapshotId) { | ||
| node('linux && immutable') { | ||
| esSnapshots.promote(snapshotVersion, snapshotId) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Validating CODEOWNERS rules …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.