Skip to content

Commit

Permalink
Address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@driazati
driazati committed May 19, 2022
1 parent 2217e2c commit 726dfcd
Show file tree
Hide file tree
Showing 7 changed files with 380 additions and 191 deletions.
228 changes: 121 additions & 107 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
// Generated at 2022-05-18T08:50:34.445369
// Generated at 2022-05-19T12:14:11.652883

import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
Expand Down Expand Up @@ -193,9 +193,9 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {

cancel_previous_build()

def lint(is_first_run) {
stage('Lint') {
parallel(
def run_lint() {
stage('Lint') {
parallel(
'Lint 1 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
Expand All @@ -205,54 +205,10 @@ stage('Lint') {
withEnv([
'TVM_NUM_SHARDS=2',
'TVM_SHARD_INDEX=0'], {
ci_arm = params.ci_arm_param ?: ci_arm
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
ci_i386 = params.ci_i386_param ?: ci_i386
ci_lint = params.ci_lint_param ?: ci_lint
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_wasm = params.ci_wasm_param ?: ci_wasm

sh (script: """
echo "Docker images being used in this build:"
echo " ci_arm = ${ci_arm}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_hexagon = ${ci_hexagon}"
echo " ci_i386 = ${ci_i386}"
echo " ci_lint = ${ci_lint}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_wasm = ${ci_wasm}"
""", label: 'Docker image names')

is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
if (is_first_run) {
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/should_rebuild_docker.py',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
})
}
}
Expand All @@ -267,67 +223,70 @@ stage('Lint') {
withEnv([
'TVM_NUM_SHARDS=2',
'TVM_SHARD_INDEX=1'], {
ci_arm = params.ci_arm_param ?: ci_arm
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
ci_i386 = params.ci_i386_param ?: ci_i386
ci_lint = params.ci_lint_param ?: ci_lint
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_wasm = params.ci_wasm_param ?: ci_wasm

sh (script: """
echo "Docker images being used in this build:"
echo " ci_arm = ${ci_arm}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_hexagon = ${ci_hexagon}"
echo " ci_i386 = ${ci_i386}"
echo " ci_lint = ${ci_lint}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_wasm = ${ci_wasm}"
""", label: 'Docker image names')

is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
if (is_first_run) {
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/should_rebuild_docker.py',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
})
}
}
}
},
)
)
}
}

def prepare() {
stage('Prepare') {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
init_git()
ci_arm = params.ci_arm_param ?: ci_arm
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
ci_i386 = params.ci_i386_param ?: ci_i386
ci_lint = params.ci_lint_param ?: ci_lint
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_wasm = params.ci_wasm_param ?: ci_wasm

sh (script: """
echo "Docker images being used in this build:"
echo " ci_arm = ${ci_arm}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_hexagon = ${ci_hexagon}"
echo " ci_i386 = ${ci_i386}"
echo " ci_lint = ${ci_lint}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_wasm = ${ci_wasm}"
""", label: 'Docker image names')

is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/should_rebuild_docker.py',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
}
}
}
}

// [note: method size]
// This has to be extracted into a method due to JVM limitations on the size of
// a method (so the code can't all be inlined)
lint(true)
prepare()

def ecr_push(full_name) {
aws_account_id = sh(
Expand Down Expand Up @@ -496,14 +455,11 @@ if (rebuild_docker_images) {
},
)
}
// TODO: Once we are able to use the built images, enable this step
// If the docker images changed, we need to run the image build before the lint
// can run since it requires a base docker image. Most of the time the images
// aren't build though so it's faster to use the same node that checks for
// docker changes to run the lint in the usual case.
lint(false)
}

// Run the lint with the new Docker image before continuing to builds
run_lint()

// Run make. First try to do an incremental make from a previous workspace in hope to
// accelerate the compilation. If something is wrong, clean the workspace and then
// build from scratch.
Expand Down Expand Up @@ -611,7 +567,9 @@ def cpp_unittest(image) {
}

def docker_init(image) {
if (image.contains("amazon")) {
if (image.contains("amazonaws.com")) {
// If this string is in the image name it's from ECR and needs to be pulled
// with the right credentials
ecr_pull(image)
} else {
sh(
Expand Down Expand Up @@ -1574,6 +1532,25 @@ def deploy_docs() {
}
}


def update_docker(ecr_image, hub_image) {
if (!ecr_image.contains("amazonaws.com")) {
sh("echo Skipping '${ecr_image}' since it doesn't look like an ECR image")
return
}
sh(
script: """
set -eux
docker pull ${ecr_image}
docker tag \
${ecr_image} \
${hub_image}
docker push ${hub_image}
""",
label: "Update ${hub_image} on Docker Hub",
)
}

stage('Deploy') {
if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') {
node('CPU') {
Expand All @@ -1583,4 +1560,41 @@ stage('Deploy') {
}
}
}
// if (env.BRANCH_NAME == 'main' && rebuild_docker_images) {
if (env.BRANCH_NAME == 'PR-11329' && rebuild_docker_images && upstream_revision != null) {
node('CPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docker") {
try {
withCredentials([string(
credentialsId: 'dockerhub-tlcpackstaging-key',
variable: 'DOCKERHUB_KEY',
)]) {
sh(
script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}',
label: 'Log in to Docker Hub',
)
}
def date_Ymd_HMS = sh(
script: 'python -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'',
label: 'Determine date',
returnStdout: true,
).trim()
def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}"
update_docker(ci_arm, "tlcpackstaging/test_ci_arm:${tag}")
update_docker(ci_cpu, "tlcpackstaging/test_ci_cpu:${tag}")
update_docker(ci_gpu, "tlcpackstaging/test_ci_gpu:${tag}")
update_docker(ci_hexagon, "tlcpackstaging/test_ci_hexagon:${tag}")
update_docker(ci_i386, "tlcpackstaging/test_ci_i386:${tag}")
update_docker(ci_lint, "tlcpackstaging/test_ci_lint:${tag}")
update_docker(ci_qemu, "tlcpackstaging/test_ci_qemu:${tag}")
update_docker(ci_wasm, "tlcpackstaging/test_ci_wasm:${tag}")
} finally {
sh(
script: 'docker logout',
label: 'Clean up login credentials'
)
}
}
}
}
}
Loading

0 comments on commit 726dfcd

Please sign in to comment.