diff --git a/Jenkinsfile b/Jenkinsfile index 0205a1e7364fe..ec4cea52d67b3 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-06-02T14:03:43.284817 +// Generated at 2022-06-09T09:42:12.430625 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -97,6 +97,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { // Filenames for stashing between build and test steps s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" + // General note: Jenkins has limits on the size of a method (or top level code) // that are pretty strict, so most usage of groovy methods in these templates // are purely to satisfy the JVM @@ -171,6 +172,17 @@ def docker_init(image) { """, label: 'Clean old Docker images', ) + + if (image.contains("amazonaws.com")) { + // If this string is in the image name it's from ECR and needs to be pulled + // with the right credentials + ecr_pull(image) + } else { + sh( + script: "docker pull ${image}", + label: 'Pull docker image', + ) + } } def should_skip_slow_tests(pr_number) { @@ -273,16 +285,50 @@ def prepare() { } } } -def build_image(image_name) { - hash = sh( +def ecr_push(full_name) { + aws_account_id = sh( returnStdout: true, - script: 'git log -1 --format=\'%h\'' + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' ).trim() - def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" - sh( - script: "${docker_build} ${image_name} --spec ${full_name}", - label: 'Build docker image' - ) + + def ecr_name = "${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com/${full_name}" + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -x + docker tag ${full_name} \$AWS_ECR_REPO/${full_name} + docker push \$AWS_ECR_REPO/${full_name} + """, + label: 'Upload image to ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } + return ecr_name +} + +def ecr_pull(full_name) { aws_account_id = sh( returnStdout: true, script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', @@ -290,153 +336,144 @@ def build_image(image_name) { ).trim() try { - // Use a credential so Jenkins knows to scrub the AWS account ID which is nice - // (but so we don't have to rely it being hardcoded in Jenkins) - withCredentials([string( - credentialsId: 'aws-account-id', - variable: '_ACCOUNT_ID_DO_NOT_USE', - )]) { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2']) { - sh( - script: ''' - set -x - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -x - docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} - docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} - """, - label: 'Upload image to ECR' - ) - } + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) } } finally { - sh( - script: 'rm -f ~/.docker/config.json', - label: 'Clean up login credentials' - ) + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } } +} + +def build_image(image_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" sh( - script: "docker rmi ${full_name}", - label: 'Remove docker image' + script: "${docker_build} ${image_name} --spec ${full_name}", + label: 'Build docker image' ) + return ecr_push(full_name) } + def build_docker_images() { stage('Docker Image Build') { - // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_lint') + parallel( + 'ci_arm': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_arm = build_image('ci_arm') + build_image('ci_arm') + } } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_cpu') + }, + 'ci_cpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_cpu = build_image('ci_cpu') + build_image('ci_cpu') + } } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_gpu') + }, + 'ci_gpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_gpu = build_image('ci_gpu') + build_image('ci_gpu') + } } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_qemu') + }, + 'ci_hexagon': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_hexagon = build_image('ci_hexagon') + build_image('ci_hexagon') + } } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_i386') + }, + 'ci_i386': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_i386 = build_image('ci_i386') + build_image('ci_i386') + } } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_arm') + }, + 'ci_lint': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_lint = build_image('ci_lint') + build_image('ci_lint') + } } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_wasm') + }, + 'ci_qemu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_qemu = build_image('ci_qemu') + build_image('ci_qemu') + } } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_hexagon') + }, + 'ci_wasm': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // ci_wasm = build_image('ci_wasm') + build_image('ci_wasm') + } } - } - } - } - // // TODO: Once we are able to use the built images, enable this step - // // If the docker images changed, we need to run the image build before the lint - // // can run since it requires a base docker image. Most of the time the images - // // aren't build though so it's faster to use the same node that checks for - // // docker changes to run the lint in the usual case. - // stage('Sanity Check (re-run)') { - // timeout(time: max_time, unit: 'MINUTES') { - // node('CPU') { - // ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/sanity") { - // init_git() - // sh ( - // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - // label: 'Run lint', - // ) - // } - // } - // } - // } -} - -// Run make. First try to do an incremental make from a previous workspace in hope to -// accelerate the compilation. If something is wrong, clean the workspace and then -// build from scratch. -def make(docker_type, path, make_flag) { - timeout(time: max_time, unit: 'MINUTES') { - try { - cmake_build(docker_type, path, make_flag) - // always run cpp test when build - } catch (hudson.AbortException ae) { - // script exited due to user abort, directly throw instead of retry - if (ae.getMessage().contains('script returned exit code 143')) { - throw ae - } - echo 'Incremental compilation failed. Fall back to build from scratch' - sh ( - script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}", - label: 'Clear old cmake workspace', - ) - cmake_build(docker_type, path, make_flag) - } + }, + ) } } def lint() { @@ -531,6 +568,29 @@ def add_hexagon_permissions() { ) } +// Run make. First try to do an incremental make from a previous workspace in hope to +// accelerate the compilation. If something is wrong, clean the workspace and then +// build from scratch. +def make(docker_type, path, make_flag) { + timeout(time: max_time, unit: 'MINUTES') { + try { + cmake_build(docker_type, path, make_flag) + } catch (hudson.AbortException ae) { + // script exited due to user abort, directly throw instead of retry + if (ae.getMessage().contains('script returned exit code 143')) { + throw ae + } + echo 'Incremental compilation failed. Fall back to build from scratch' + sh ( + script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}", + label: 'Clear old cmake workspace', + ) + cmake_build(docker_type, path, make_flag) + } + } +} + + def build() { stage('Build') { environment { @@ -3239,6 +3299,25 @@ stage('Build packages') { } */ + +def update_docker(ecr_image, hub_image) { + if (!ecr_image.contains("amazonaws.com")) { + sh("echo Skipping '${ecr_image}' since it doesn't look like an ECR image") + return + } + docker_init(ecr_image) + sh( + script: """ + set -eux + docker tag \ + ${ecr_image} \ + ${hub_image} + docker push ${hub_image} + """, + label: "Update ${hub_image} on Docker Hub", + ) +} + def deploy_docs() { // Note: This code must stay in the Jenkinsfile to ensure that it runs // from a trusted context only @@ -3298,6 +3377,42 @@ def deploy() { } } } + if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { + node('CPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docker") { + try { + withCredentials([string( + credentialsId: 'dockerhub-tlcpackstaging-key', + variable: 'DOCKERHUB_KEY', + )]) { + sh( + script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}', + label: 'Log in to Docker Hub', + ) + } + def date_Ymd_HMS = sh( + script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'', + label: 'Determine date', + returnStdout: true, + ).trim() + def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}" + update_docker(ci_arm, "tlcpackstaging/test_ci_arm:${tag}") + update_docker(ci_cpu, "tlcpackstaging/test_ci_cpu:${tag}") + update_docker(ci_gpu, "tlcpackstaging/test_ci_gpu:${tag}") + update_docker(ci_hexagon, "tlcpackstaging/test_ci_hexagon:${tag}") + update_docker(ci_i386, "tlcpackstaging/test_ci_i386:${tag}") + update_docker(ci_lint, "tlcpackstaging/test_ci_lint:${tag}") + update_docker(ci_qemu, "tlcpackstaging/test_ci_qemu:${tag}") + update_docker(ci_wasm, "tlcpackstaging/test_ci_wasm:${tag}") + } finally { + sh( + script: 'docker logout', + label: 'Clean up login credentials' + ) + } + } + } + } } } diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2 index 62ccc94916048..fcde53f559395 100644 --- a/jenkins/Build.groovy.j2 +++ b/jenkins/Build.groovy.j2 @@ -52,6 +52,29 @@ def add_hexagon_permissions() { {% endfor %} } +// Run make. First try to do an incremental make from a previous workspace in hope to +// accelerate the compilation. If something is wrong, clean the workspace and then +// build from scratch. +def make(docker_type, path, make_flag) { + timeout(time: max_time, unit: 'MINUTES') { + try { + cmake_build(docker_type, path, make_flag) + } catch (hudson.AbortException ae) { + // script exited due to user abort, directly throw instead of retry + if (ae.getMessage().contains('script returned exit code 143')) { + throw ae + } + echo 'Incremental compilation failed. Fall back to build from scratch' + sh ( + script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}", + label: 'Clear old cmake workspace', + ) + cmake_build(docker_type, path, make_flag) + } + } +} + + def build() { stage('Build') { environment { diff --git a/jenkins/Deploy.groovy.j2 b/jenkins/Deploy.groovy.j2 index 917f71ded1ff3..3a049c5141dd9 100644 --- a/jenkins/Deploy.groovy.j2 +++ b/jenkins/Deploy.groovy.j2 @@ -16,6 +16,25 @@ stage('Build packages') { } */ + +def update_docker(ecr_image, hub_image) { + if (!ecr_image.contains("amazonaws.com")) { + sh("echo Skipping '${ecr_image}' since it doesn't look like an ECR image") + return + } + docker_init(ecr_image) + sh( + script: """ + set -eux + docker tag \ + ${ecr_image} \ + ${hub_image} + docker push ${hub_image} + """, + label: "Update ${hub_image} on Docker Hub", + ) +} + def deploy_docs() { // Note: This code must stay in the Jenkinsfile to ensure that it runs // from a trusted context only @@ -67,5 +86,36 @@ def deploy() { } } } + if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) { + node('CPU') { + ws({{ m.per_exec_ws('tvm/deploy-docker') }}) { + try { + withCredentials([string( + credentialsId: 'dockerhub-tlcpackstaging-key', + variable: 'DOCKERHUB_KEY', + )]) { + sh( + script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}', + label: 'Log in to Docker Hub', + ) + } + def date_Ymd_HMS = sh( + script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'', + label: 'Determine date', + returnStdout: true, + ).trim() + def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}" + {% for image in images %} + update_docker({{ image.name }}, "tlcpackstaging/test_{{ image.name }}:${tag}") + {% endfor %} + } finally { + sh( + script: 'docker logout', + label: 'Clean up login credentials' + ) + } + } + } + } } } diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2 index e9d80801a9d9c..a0ff666773f75 100644 --- a/jenkins/DockerBuild.groovy.j2 +++ b/jenkins/DockerBuild.groovy.j2 @@ -1,13 +1,47 @@ -def build_image(image_name) { - hash = sh( +def ecr_push(full_name) { + aws_account_id = sh( returnStdout: true, - script: 'git log -1 --format=\'%h\'' + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' ).trim() - def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" - sh( - script: "${docker_build} ${image_name} --spec ${full_name}", - label: 'Build docker image' - ) + + def ecr_name = "${aws_account_id}.{{ aws_ecr_url }}/${full_name}" + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -x + docker tag ${full_name} \$AWS_ECR_REPO/${full_name} + docker push \$AWS_ECR_REPO/${full_name} + """, + label: 'Upload image to ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } + return ecr_name +} + +def ecr_pull(full_name) { aws_account_id = sh( returnStdout: true, script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', @@ -15,152 +49,68 @@ def build_image(image_name) { ).trim() try { - // Use a credential so Jenkins knows to scrub the AWS account ID which is nice - // (but so we don't have to rely it being hardcoded in Jenkins) - withCredentials([string( - credentialsId: 'aws-account-id', - variable: '_ACCOUNT_ID_DO_NOT_USE', - )]) { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2']) { - sh( - script: ''' - set -x - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -x - docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} - docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name} - """, - label: 'Upload image to ECR' - ) - } + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) } } finally { - sh( - script: 'rm -f ~/.docker/config.json', - label: 'Clean up login credentials' - ) + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } } +} + +def build_image(image_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" sh( - script: "docker rmi ${full_name}", - label: 'Remove docker image' + script: "${docker_build} ${image_name} --spec ${full_name}", + label: 'Build docker image' ) + return ecr_push(full_name) } + def build_docker_images() { stage('Docker Image Build') { - // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_lint') - } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_cpu') + parallel( + {% for image in images %} + '{{ image.name }}': { + node('{{ image.platform }}') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + // We're purposefully not setting the built image here since they + // are not yet being uploaded to tlcpack + // {{ image.name }} = build_image('{{ image.name }}') + build_image('{{ image.name }}') + } } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_gpu') - } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_qemu') - } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_i386') - } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_arm') - } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_wasm') - } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - docker_init('none') - init_git() - build_image('ci_hexagon') - } - } - } - } - // // TODO: Once we are able to use the built images, enable this step - // // If the docker images changed, we need to run the image build before the lint - // // can run since it requires a base docker image. Most of the time the images - // // aren't build though so it's faster to use the same node that checks for - // // docker changes to run the lint in the usual case. - // stage('Sanity Check (re-run)') { - // timeout(time: max_time, unit: 'MINUTES') { - // node('CPU') { - // ws({{ m.per_exec_ws('tvm/sanity') }}) { - // init_git() - // sh ( - // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - // label: 'Run lint', - // ) - // } - // } - // } - // } -} - -// Run make. First try to do an incremental make from a previous workspace in hope to -// accelerate the compilation. If something is wrong, clean the workspace and then -// build from scratch. -def make(docker_type, path, make_flag) { - timeout(time: max_time, unit: 'MINUTES') { - try { - cmake_build(docker_type, path, make_flag) - // always run cpp test when build - } catch (hudson.AbortException ae) { - // script exited due to user abort, directly throw instead of retry - if (ae.getMessage().contains('script returned exit code 143')) { - throw ae - } - echo 'Incremental compilation failed. Fall back to build from scratch' - sh ( - script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}", - label: 'Clear old cmake workspace', - ) - cmake_build(docker_type, path, make_flag) - } + }, + {% endfor %} + ) } } diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index c165de964feb4..4e344c56d7f72 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -100,6 +100,9 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { {% set hexagon_api = ['build/hexagon_api_output',] %} s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" +{% set aws_default_region = "us-west-2" %} +{% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %} + // General note: Jenkins has limits on the size of a method (or top level code) // that are pretty strict, so most usage of groovy methods in these templates // are purely to satisfy the JVM diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2 index 40dad3aef7be3..3ede64301c935 100644 --- a/jenkins/Lint.groovy.j2 +++ b/jenkins/Lint.groovy.j2 @@ -2,11 +2,11 @@ def lint() { stage('Lint') { parallel( {% call m.sharded_lint_step( - name='Lint', - num_shards=2, - node='CPU-SMALL', - ws='tvm/lint', - docker_image='ci_lint', + name='Lint', + num_shards=2, + node='CPU-SMALL', + ws='tvm/lint', + docker_image='ci_lint', ) %} sh ( diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2 index 2900775f49452..894ddc72eeb7b 100644 --- a/jenkins/Prepare.groovy.j2 +++ b/jenkins/Prepare.groovy.j2 @@ -69,6 +69,17 @@ def docker_init(image) { """, label: 'Clean old Docker images', ) + + if (image.contains("amazonaws.com")) { + // If this string is in the image name it's from ECR and needs to be pulled + // with the right credentials + ecr_pull(image) + } else { + sh( + script: "docker pull ${image}", + label: 'Pull docker image', + ) + } } def should_skip_slow_tests(pr_number) { diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py index 042c109dd9d49..7ef2f0cd58452 100644 --- a/tests/python/ci/test_ci.py +++ b/tests/python/ci/test_ci.py @@ -18,9 +18,11 @@ import subprocess import sys import json +from tempfile import tempdir import textwrap import pytest import tvm.testing +from pathlib import Path from test_utils import REPO_ROOT @@ -29,11 +31,13 @@ class TempGit: def __init__(self, cwd): self.cwd = cwd - def run(self, *args): - proc = subprocess.run(["git"] + list(args), cwd=self.cwd) + def run(self, *args, **kwargs): + proc = subprocess.run(["git"] + list(args), encoding="utf-8", cwd=self.cwd, **kwargs) if proc.returncode != 0: raise RuntimeError(f"git command failed: '{args}'") + return proc + def test_cc_reviewers(tmpdir_factory): reviewers_script = REPO_ROOT / "tests" / "scripts" / "github_cc_reviewers.py" @@ -747,5 +751,94 @@ def run(type, data, check): ) +@pytest.mark.parametrize( + "changed_files,name,check,expected_code", + [ + d.values() + for d in [ + dict( + changed_files=[], + name="abc", + check="Image abc is not using new naming scheme", + expected_code=1, + ), + dict( + changed_files=[], name="123-123-abc", check="No extant hash found", expected_code=1 + ), + dict( + changed_files=[["test.txt"]], + name=None, + check="Did not find changes, no rebuild necessary", + expected_code=0, + ), + dict( + changed_files=[["test.txt"], ["docker/test.txt"]], + name=None, + check="Found docker changes", + expected_code=2, + ), + ] + ], +) +def test_should_rebuild_docker(tmpdir_factory, changed_files, name, check, expected_code): + tag_script = REPO_ROOT / "tests" / "scripts" / "should_rebuild_docker.py" + + git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) + git.run("init") + git.run("config", "user.name", "ci") + git.run("config", "user.email", "email@example.com") + git.run("checkout", "-b", "main") + git.run("remote", "add", "origin", "https://github.com/apache/tvm.git") + + git_path = Path(git.cwd) + for i, commits in enumerate(changed_files): + for filename in commits: + path = git_path / filename + path.parent.mkdir(exist_ok=True, parents=True) + path.touch() + git.run("add", filename) + + git.run("commit", "-m", f"message {i}") + + if name is None: + ref = "HEAD" + if len(changed_files) > 1: + ref = f"HEAD~{len(changed_files) - 1}" + proc = git.run("rev-parse", ref, stdout=subprocess.PIPE) + last_hash = proc.stdout.strip() + name = f"123-123-{last_hash}" + + docker_data = { + "repositories/tlcpack": { + "results": [ + { + "name": "ci-something", + }, + { + "name": "something-else", + }, + ], + }, + "repositories/tlcpack/ci-something/tags": { + "results": [{"name": name}, {"name": name + "old"}], + }, + } + + proc = subprocess.run( + [ + str(tag_script), + "--testing-docker-data", + json.dumps(docker_data), + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + cwd=git.cwd, + ) + + assert_in(check, proc.stdout) + assert proc.returncode == expected_code + + if __name__ == "__main__": tvm.testing.main() diff --git a/tests/scripts/cmd_utils.py b/tests/scripts/cmd_utils.py index 272086796e8df..771c3ee52dbd2 100644 --- a/tests/scripts/cmd_utils.py +++ b/tests/scripts/cmd_utils.py @@ -44,18 +44,21 @@ def init_log(): class Sh: - def __init__(self, env=None): + def __init__(self, env=None, cwd=None): self.env = os.environ.copy() if env is not None: self.env.update(env) + self.cwd = cwd def run(self, cmd: str, **kwargs): logging.info(f"+ {cmd}") - if "check" not in kwargs: - kwargs["check"] = True - if "shell" not in kwargs: - kwargs["shell"] = True - if "env" not in kwargs: - kwargs["env"] = self.env - - subprocess.run(cmd, **kwargs) + defaults = { + "check": True, + "shell": True, + "env": self.env, + "encoding": "utf-8", + "cwd": self.cwd, + } + defaults.update(kwargs) + + return subprocess.run(cmd, **defaults) diff --git a/tests/scripts/git_utils.py b/tests/scripts/git_utils.py index 267756d859050..c5ea8d85e0718 100644 --- a/tests/scripts/git_utils.py +++ b/tests/scripts/git_utils.py @@ -20,6 +20,7 @@ import subprocess import re import base64 +import logging from urllib import request from typing import Dict, Tuple, Any, Optional, List diff --git a/tests/scripts/http_utils.py b/tests/scripts/http_utils.py new file mode 100644 index 0000000000000..c14259479d3be --- /dev/null +++ b/tests/scripts/http_utils.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import logging +from urllib import request +from typing import Dict, Any, Optional + + +def get(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + logging.info(f"Requesting GET to {url}") + if headers is None: + headers = {} + req = request.Request(url, headers=headers) + with request.urlopen(req) as response: + response_headers = {k: v for k, v in response.getheaders()} + response = json.loads(response.read()) + + return response, response_headers diff --git a/tests/scripts/should_rebuild_docker.py b/tests/scripts/should_rebuild_docker.py new file mode 100755 index 0000000000000..dc12c38de8303 --- /dev/null +++ b/tests/scripts/should_rebuild_docker.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import argparse +import datetime +import json +import logging +import subprocess + +from typing import Dict, Any, List + + +from http_utils import get +from cmd_utils import Sh, init_log + + +DOCKER_API_BASE = "https://hub.docker.com/v2/" +PAGE_SIZE = 25 +TEST_DATA = None + + +def docker_api(url: str) -> Dict[str, Any]: + """ + Run a paginated fetch from the public Docker Hub API + """ + if TEST_DATA is not None: + return TEST_DATA[url] + pagination = f"?page_size={PAGE_SIZE}&page=1" + url = DOCKER_API_BASE + url + pagination + r, headers = get(url) + reset = headers.get("x-ratelimit-reset") + if reset is not None: + reset = datetime.datetime.fromtimestamp(int(reset)) + reset = reset.isoformat() + logging.info( + f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})" + ) + if "results" not in r: + raise RuntimeError(f"Error fetching data, no results found in: {r}") + return r + + +def any_docker_changes_since(hash: str) -> bool: + """ + Check the docker/ directory, return True if there have been any code changes + since the specified hash + """ + sh = Sh() + cmd = f"git diff {hash} -- docker/" + proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout = proc.stdout.strip() + return stdout != "", stdout + + +def does_commit_exist(hash: str) -> bool: + """ + Returns True if the hash exists in the repo + """ + sh = Sh() + cmd = f"git rev-parse -q {hash}" + proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + print(proc.stdout) + if proc.returncode == 0: + return True + + if "unknown revision or path not in the working tree" in proc.stdout: + return False + + raise RuntimeError(f"Unexpected failure when running: {cmd}") + + +def find_hash_for_tag(tag: Dict[str, Any]) -> str: + """ + Split the hash off of a name like -