Skip to content

Commit

Permalink
CI: retry the step only (#26736)
Browse files Browse the repository at this point in the history
(cherry picked from commit 6b0c391)
  • Loading branch information
v1v authored and mergify-bot committed Jul 6, 2021
1 parent 0592c19 commit 7f7a1fb
Showing 1 changed file with 40 additions and 5 deletions.
45 changes: 40 additions & 5 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -554,14 +554,23 @@ def target(Map args = [:]) {
def isE2E = args.e2e?.get('enabled', false)
def isPackaging = args.get('package', false)
def dockerArch = args.get('dockerArch', 'amd64')
def enableRetry = args.get('enableRetry', false)
withNode(labels: args.label, forceWorkspace: true){
withGithubNotify(context: "${context}") {
withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
dumpVariables()
// make commands use -C <folder> while mage commands require the dir(folder)
// let's support this scenario with the location variable.
dir(isMage ? directory : '') {
cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
if (enableRetry) {
// Retry the same command to bypass any kind of flakiness.
// Downside: genuine failures will be repeated.
retry(3) {
cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
}
} else {
cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
}
}
// TODO:
// Packaging should happen only after the e2e?
Expand Down Expand Up @@ -1023,11 +1032,35 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
public run(Map args = [:]){
steps.stageStatusCache(args){
def withModule = args.content.get('withModule', false)
//
// What's the retry policy for fighting the flakiness:
// 1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim
// 2) All the remaining stages will retry the command within the same worker/workspace if any failure
//
// NOTE: stage: lint uses target function while cloud and k8sTest use a different function
//
def enableRetry = (args.content.get('stage', 'enabled').toLowerCase().equals('lint') ||
args?.content?.containsKey('packaging-arm') ||
args?.content?.containsKey('packaging-linux')) ? false : true
if(args?.content?.containsKey('make')) {
steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
steps.target(context: args.context,
command: args.content.make,
directory: args.project,
label: args.label,
withModule: withModule,
isMage: false,
id: args.id,
enableRetry: enableRetry)
}
if(args?.content?.containsKey('mage')) {
steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
steps.target(context: args.context,
command: args.content.mage,
directory: args.project,
label: args.label,
withModule: withModule,
isMage: true,
id: args.id,
enableRetry: enableRetry)
}
if(args?.content?.containsKey('packaging-arm')) {
steps.packagingArm(context: args.context,
Expand All @@ -1038,7 +1071,8 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
id: args.id,
e2e: args.content.get('e2e'),
package: true,
dockerArch: 'arm64')
dockerArch: 'arm64',
enableRetry: enableRetry)
}
if(args?.content?.containsKey('packaging-linux')) {
steps.packagingLinux(context: args.context,
Expand All @@ -1049,7 +1083,8 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
id: args.id,
e2e: args.content.get('e2e'),
package: true,
dockerArch: 'amd64')
dockerArch: 'amd64',
enableRetry: enableRetry)
}
if(args?.content?.containsKey('k8sTest')) {
steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
Expand Down

0 comments on commit 7f7a1fb

Please sign in to comment.