Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Premerge databricks CI autotrigger [skip ci] #5067

Merged
merged 5 commits into from
Mar 30, 2022
Merged

Premerge databricks CI autotrigger [skip ci] #5067

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6d0f1c6
Patterns to auto-trigger databricks build in premerge
pxLi Mar 28, 2022
afab5b1
also remove non-existed shims sourcePattern
pxLi Mar 28, 2022
cb46516
remove dup unstash
pxLi Mar 28, 2022
66a295f
aborted status change
pxLi Mar 28, 2022
ddab091
avoid list all changed files
pxLi Mar 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 75 additions & 75 deletions jenkins/Jenkinsfile-blossom.premerge
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,6 @@ def PREMERGE_CI_2_ARGUMENT // argument for 'spark-premerge-build.sh' running fro
def TestResult1 // keep ci_1 pytest result files that's in JUnit-XML style
def TestResult2 // keep ci_2 pytest result files that's in JUnit-XML style

def sourcePattern30X = 'shims/spark301db/src/main/scala/,' +
'shims/spark301/src/main/scala/,' +
'shims/spark302/src/main/scala/,' +
'shims/spark303/src/main/scala/,' +
'shims/spark304/src/main/scala/'
def sourcePattern31X = 'shims/spark311/src/main/scala/,' +
'shims/spark312/src/main/scala/,' +
'shims/spark313/src/main/scala/,' +
'shims/spark314/src/main/scala/'
def sourcePattern32X = 'shims/spark320/src/main/scala/,' +
'shims/spark321/src/main/scala/,' +
'shims/spark322/src/main/scala/'
def sourcePattern33X = 'shims/spark330/src/main/scala/'
def sourcePattern31XPlus = sourcePattern31X + ',' + sourcePattern32X + ',' + sourcePattern33X
def sourcePattern = 'shuffle-plugin/src/main/scala/,udf-compiler/src/main/scala/,' +
'sql-plugin/src/main/java/,sql-plugin/src/main/scala/'

Expand Down Expand Up @@ -174,14 +160,51 @@ pipeline {
skipped = true
return
}
checkoutCode(githubHelper.getCloneUrl(), githubHelper.getMergedSHA())
// check if need trigger databricks CI build
if (title ==~ /.*\[databricks\].*/) {
if (title ==~ /.*\[databricks\].*/ || databricksCodeChanged()) {
db_build = true
}
}
}
} // end of Init githubHelper

stage("Determine Project Version") {
when {
expression {
!skipped
}
}

steps {
script {
// Retrieve PROJECT_VER from version-def.sh, e.g, '<major>.<minor>.<patch>-SNAPSHOT'
PROJECT_VER = sh(returnStdout: true, script: "bash $JENKINS_ROOT/version-def.sh | cut -d ',' -f 3 | cut -d ' ' -f 3")
PROJECT_VER = PROJECT_VER.split('-')[0] // Remove trailing '-SNAPSHOT'
echo PROJECT_VER

def versions = PROJECT_VER.split('\\.')
major_ver = versions[0].toInteger()
minor_ver = versions[1].toInteger()

// TODO: remove major version 21.XX pre-merge support
if (major_ver == 21) {
if (minor_ver == 8) {
PREMERGE_CI_2_ARGUMENT = "unit_test" // for '21.08' version
} else if (minor_ver >= 10) {
PREMERGE_CI_2_ARGUMENT = "ci_2" // for '21.10' or later version
}
} else if (major_ver >= 22) {
PREMERGE_CI_2_ARGUMENT = "ci_2"
} else {
error("Unsupported major version: $major_ver")
}

echo PREMERGE_CI_2_ARGUMENT
}
}
}

stage('Build docker image') {
when {
beforeAgent true
Expand All @@ -203,21 +226,7 @@ pipeline {
steps {
script {
githubHelper.updateCommitStatus("$BUILD_URL", "Running", GitHubCommitState.PENDING)
checkout(
changelog: false,
poll: true,
scm: [
$class : 'GitSCM', branches: [[name: githubHelper.getMergedSHA()]],
submoduleCfg : [],
userRemoteConfigs: [[
credentialsId: 'github-token',
url : githubHelper.getCloneUrl(),
refspec : '+refs/pull/*/merge:refs/remotes/origin/pr/*']]
]
)

stash(name: "source_tree", includes: "**")

unstash "source_tree"
container('docker-build') {
// check if pre-merge dockerfile modified
def dockerfileModified = sh(returnStdout: true,
Expand All @@ -244,48 +253,6 @@ pipeline {
}
} // end of Build docker image

stage("Determine Project Version") {
when {
expression {
!skipped
}
}

steps {
script {
unstash "source_tree"
// Retrieve PROJECT_VER from version-def.sh, e.g, '<major>.<minor>.<patch>-SNAPSHOT'
PROJECT_VER = sh(returnStdout: true, script: "bash $JENKINS_ROOT/version-def.sh | cut -d ',' -f 3 | cut -d ' ' -f 3")
PROJECT_VER = PROJECT_VER.split('-')[0] // Remove trailing '-SNAPSHOT'
echo PROJECT_VER

def versions = PROJECT_VER.split('\\.')
major_ver = versions[0].toInteger()
minor_ver = versions[1].toInteger()

sourcePattern = sourcePattern + ',' + sourcePattern31XPlus
// TODO: remove major version 21.XX pre-merge support
if (major_ver == 21) {
if (minor_ver == 8) {
PREMERGE_CI_2_ARGUMENT = "unit_test" // for '21.08' version
} else if (minor_ver >= 10) {
PREMERGE_CI_2_ARGUMENT = "ci_2" // for '21.10' or later version
}
sourcePattern = sourcePattern + ',' + sourcePattern30X
} else if (major_ver >= 22) {
PREMERGE_CI_2_ARGUMENT = "ci_2"
if (major_ver == 22 && minor_ver < 4) { // 22.02-
sourcePattern = sourcePattern + ',' + sourcePattern30X
}
} else {
error("Unsupported major version: $major_ver")
}

echo PREMERGE_CI_2_ARGUMENT
}
}
}

stage('Init DB') {
when {
beforeAgent true
Expand All @@ -302,8 +269,9 @@ pipeline {
}
steps {
script {
githubHelper.updateCommitStatus("$BUILD_URL", "Running - includes databricks", GitHubCommitState.PENDING)
unstash "source_tree"
container('cpu') {
unstash "source_tree"
sh """
bash -c 'dbfs mkdirs $INIT_SCRIPTS_DIR'
bash -c 'dbfs cp --overwrite jenkins/databricks/init_cudf_udf.sh $INIT_SCRIPTS_DIR'
Expand Down Expand Up @@ -390,7 +358,6 @@ pipeline {
steps {
script {
unstash "source_tree"

container('gpu') {
timeout(time: 4, unit: 'HOURS') {
try {
Expand Down Expand Up @@ -535,7 +502,9 @@ pipeline {
guardWords.add("nvidia-smi(?s)(.*?)(?=jenkins/version-def.sh)") // hide GPU info
githubHelper.uploadParallelLogs(this, env.JOB_NAME, env.BUILD_NUMBER, null, guardWords)

githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)
if (currentBuild.currentResult != "ABORTED") { // skip ABORTED result to avoid status overwrite
githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)
}
}

if (db_build) {
Expand Down Expand Up @@ -685,3 +654,34 @@ String stashPytestResult(String name) {

return result
}

void checkoutCode(String url, String sha) {
checkout(
changelog: false,
poll: true,
scm: [
$class : 'GitSCM', branches: [[name: sha]],
submoduleCfg : [],
userRemoteConfigs: [[
credentialsId: 'github-token',
url : url,
refspec : '+refs/pull/*/merge:refs/remotes/origin/pr/*']]
]
)

stash(name: "source_tree", includes: "**,.git/**", useDefaultExcludes: false)
}

boolean databricksCodeChanged() {
def output = sh(script: '''
# get merge BASE from merged pull request. Log message e.g. "Merge HEAD into BASE"
BASE_REF=$(git --no-pager log --oneline -1 | awk '{ print $NF }')
git --no-pager diff --name-only ${BASE_REF} HEAD | grep -E 'sql-plugin/src/main/.*[0-9x-]db/|databricks || true'
jlowe marked this conversation as resolved.
Show resolved Hide resolved
''', returnStdout: true).trim()

if (output) {
echo "Databricks-related changed files: \n${output}"
return true
}
return false
}