From e554f1191931a934477ed6a784bd573c92a3f203 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Mon, 31 Jul 2023 08:07:15 -0500 Subject: [PATCH] native multi-arch --- .github/actions/compute-matrix/action.yml | 30 +++++ .../workflows/build-and-publish-images.yml | 109 ++++++++++-------- .github/workflows/build-image.yml | 64 ++++++++++ .github/workflows/{prs.yml => pr.yml} | 6 +- .github/workflows/{publish.yml => push.yml} | 8 +- ci/compute-arch.sh | 24 ---- ci/compute-matrix.jq | 61 ++++++++++ ci/compute-matrix.sh | 22 ++-- ci/create-multiarch-manifest.sh | 24 ++++ ci/remove-temp-images.sh | 25 ++++ axis.yaml => matrix.yaml | 2 + 11 files changed, 287 insertions(+), 88 deletions(-) create mode 100644 .github/actions/compute-matrix/action.yml create mode 100644 .github/workflows/build-image.yml rename .github/workflows/{prs.yml => pr.yml} (72%) rename .github/workflows/{publish.yml => push.yml} (71%) delete mode 100755 ci/compute-arch.sh create mode 100644 ci/compute-matrix.jq create mode 100755 ci/create-multiarch-manifest.sh create mode 100755 ci/remove-temp-images.sh rename axis.yaml => matrix.yaml (95%) diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml new file mode 100644 index 0000000..a86078a --- /dev/null +++ b/.github/actions/compute-matrix/action.yml @@ -0,0 +1,30 @@ +name: 'Compute Matrix Action' +description: 'Computes matrix' +outputs: + MATRIX: + description: "Computed matrix" + value: ${{ steps.compute-matrix.outputs.MATRIX }} + LATEST_LINUX_VER: + description: "Latest Linux version" + value: ${{ steps.latest-values.outputs.LATEST_LINUX_VER }} + LATEST_CUDA_VER: + description: "Latest CUDA version" + value: ${{ steps.latest-values.outputs.LATEST_CUDA_VER }} + LATEST_PYTHON_VER: + description: "Latest Python version" + value: ${{ steps.latest-values.outputs.LATEST_PYTHON_VER }} +runs: + using: "composite" + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Compute latest values + id: latest-values + run: ./ci/compute-latest-versions.sh + shell: bash + - name: Compute matrix + id: compute-matrix + run: | + MATRIX=$(ci/compute-matrix.sh) + echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT} + shell: bash diff --git a/.github/workflows/build-and-publish-images.yml b/.github/workflows/build-and-publish-images.yml index fc25a45..095252c 100644 --- a/.github/workflows/build-and-publish-images.yml +++ b/.github/workflows/build-and-publish-images.yml @@ -3,9 +3,9 @@ name: build and publish imgs workflow on: workflow_call: inputs: - push: + build_type: required: true - type: boolean + type: string defaults: run: @@ -15,71 +15,80 @@ jobs: compute-matrix: runs-on: ubuntu-latest outputs: - LATEST_LINUX_VER: ${{ steps.latest-values.outputs.LATEST_LINUX_VER }} - LATEST_CUDA_VER: ${{ steps.latest-values.outputs.LATEST_CUDA_VER }} - LATEST_PYTHON_VER: ${{ steps.latest-values.outputs.LATEST_PYTHON_VER }} MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }} + LATEST_LINUX_VER: ${{ steps.compute-matrix.outputs.LATEST_LINUX_VER }} + LATEST_CUDA_VER: ${{ steps.compute-matrix.outputs.LATEST_CUDA_VER }} + LATEST_PYTHON_VER: ${{ steps.compute-matrix.outputs.LATEST_PYTHON_VER }} steps: - name: Checkout uses: actions/checkout@v3 - name: Compute latest values id: latest-values - run: ./ci/compute-latest-versions.sh + run: ci/compute-latest-versions.sh - name: Compute matrix id: compute-matrix - run: ./ci/compute-matrix.sh + run: | + MATRIX=$(ci/compute-matrix.sh) + echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT} + env: + BUILD_TYPE: ${{ inputs.BUILD_TYPE }} docker: + name: docker needs: compute-matrix - runs-on: ubuntu-latest - env: - DOCKERHUB_USERNAME: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} strategy: matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} + fail-fast: false + secrets: inherit + uses: ./.github/workflows/build-image.yml + with: + ARCHES: ${{ toJSON(matrix.ARCHES) }} + CUDA_VER: ${{ matrix.CUDA_VER }} + LINUX_VER: ${{ matrix.LINUX_VER }} + PYTHON_VER: ${{ matrix.PYTHON_VER }} + IMAGE_REPO: ${{ matrix.IMAGE_REPO }} + IMAGE_NAME: ${{ matrix.IMAGE_NAME }} + + build-multiarch-manifest: + name: manifest (${{ matrix.CUDA_VER }}, ${{ matrix.PYTHON_VER }}, ${{ matrix.LINUX_VER }}, ${{ matrix.IMAGE_REPO }}) + needs: [docker, compute-matrix] + strategy: + matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} - name: Login to DockerHub + - name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Compute Tags - id: compute-tags - run: | - set -x - TAGS="rapidsai/ci:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PYTHON_VER }}" - - if [[ - "${{ needs.compute-matrix.outputs.LATEST_LINUX_VER }}" == "${{ matrix.LINUX_VER }}" && - "${{ needs.compute-matrix.outputs.LATEST_CUDA_VER }}" == "${{ matrix.CUDA_VER }}" && - "${{ needs.compute-matrix.outputs.LATEST_PYTHON_VER }}" == "${{ matrix.PYTHON_VER }}" - ]]; then - TAGS+=",rapidsai/ci:latest" - fi - echo "TAGS=${TAGS}" >> ${GITHUB_OUTPUT} - - name: Compute Platforms - id: compute-platforms - run: ./ci/compute-arch.sh + username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Create multiarch manifest env: + ARCHES: ${{ toJSON(matrix.ARCHES) }} CUDA_VER: ${{ matrix.CUDA_VER }} LINUX_VER: ${{ matrix.LINUX_VER }} - - name: Build and push - timeout-minutes: 20 - uses: docker/build-push-action@v4 - with: - context: context - file: Dockerfile - platforms: ${{ steps.compute-platforms.outputs.PLATFORMS }} - push: ${{ inputs.push }} - pull: true - build-args: | - CUDA_VER=${{ matrix.CUDA_VER }} - LINUX_VER=${{ matrix.LINUX_VER }} - PYTHON_VER=${{ matrix.PYTHON_VER }} - tags: ${{ steps.compute-tags.outputs.TAGS }} + PYTHON_VER: ${{ matrix.PYTHON_VER }} + IMAGE_REPO: ${{ matrix.IMAGE_REPO }} + BUILD_TYPE: ${{ inputs.build_type }} + IMAGE_NAME: ${{ matrix.IMAGE_NAME }} + LATEST_CUDA_VER: ${{ needs.compute-matrix.outputs.LATEST_CUDA_VER }} + LATEST_PYTHON_VER: ${{ needs.compute-matrix.outputs.LATEST_PYTHON_VER }} + LATEST_UBUNTU_VER: ${{ needs.compute-matrix.outputs.LATEST_LINUX_VER }} + run: ci/create-multiarch-manifest.sh + + delete-temp-images: + needs: [compute-matrix, build-multiarch-manifest] + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Remove temporary images + run: ci/remove-temp-images.sh + env: + GPUCIBOT_DOCKERHUB_USER: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + IMAGE_NAME: ${{ matrix.IMAGE_NAME }} + ARCHES: ${{ toJSON(matrix.ARCHES) }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 0000000..98dcacb --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,64 @@ +name: Build and push image variant + +on: + workflow_call: + inputs: + ARCHES: + required: true + type: string + CUDA_VER: + required: true + type: string + LINUX_VER: + required: true + type: string + PYTHON_VER: + required: true + type: string + IMAGE_REPO: + required: true + type: string + IMAGE_NAME: + required: true + type: string + +jobs: + run: + name: build (${{ matrix.CUDA_VER }}, ${{ matrix.PYTHON_VER }}, ${{ matrix.LINUX_VER }}, ${{ matrix.ARCH }}) + strategy: + matrix: + ARCH: ${{ fromJSON(inputs.ARCHES) }} + CUDA_VER: ["${{ inputs.CUDA_VER }}"] + LINUX_VER: ["${{ inputs.LINUX_VER }}"] + PYTHON_VER: ["${{ inputs.PYTHON_VER }}"] + fail-fast: false + runs-on: linux-${{ matrix.ARCH }}-cpu4 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Set up Docker Context for Buildx + run: | + docker context create builders + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver: docker + endpoint: builders + - name: Build image + uses: docker/build-push-action@v4 + with: + context: ./${{ inputs.IMAGE_REPO }} + file: ./${{ inputs.IMAGE_REPO }}/Dockerfile + push: true + pull: true + build-args: | + CUDA_VER=${{ inputs.CUDA_VER }} + LINUX_VER=${{ inputs.LINUX_VER }} + PYTHON_VER=${{ inputs.PYTHON_VER }} + TARGETPLATFORM=${{ matrix.ARCH }} + tags: ${{ inputs.IMAGE_NAME }}-${{ matrix.ARCH }} diff --git a/.github/workflows/prs.yml b/.github/workflows/pr.yml similarity index 72% rename from .github/workflows/prs.yml rename to .github/workflows/pr.yml index fcbe519..82ffc77 100644 --- a/.github/workflows/prs.yml +++ b/.github/workflows/pr.yml @@ -1,7 +1,9 @@ name: ci on: - pull_request: + push: + branches: + - "pull-request/[0-9]+" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -11,5 +13,5 @@ jobs: build-images: uses: ./.github/workflows/build-and-publish-images.yml with: - push: false + build_type: pull-request secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/push.yml similarity index 71% rename from .github/workflows/publish.yml rename to .github/workflows/push.yml index a289a1b..58ac40c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/push.yml @@ -1,18 +1,18 @@ name: publish on: + workflow_dispatch: push: branches: - - "main" - workflow_dispatch: + - main concurrency: - group: "${{ github.workflow }} @ ${{ github.ref }}" + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build-images: uses: ./.github/workflows/build-and-publish-images.yml with: - push: true + build_type: branch secrets: inherit diff --git a/ci/compute-arch.sh b/ci/compute-arch.sh deleted file mode 100755 index 286164f..0000000 --- a/ci/compute-arch.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Script to determine which image architectures should -# be built for different CUDA/OS variants. -# Example Usage: -# CUDA_VER=11.5.1 LINUX_VER=rockylinux8 ./ci/compute-arch.sh -set -eu - -PLATFORMS="linux/amd64" - -write_platforms() { - local PLATFORMS="${1}" - - # Use /dev/null to ensure the script can be tested locally - echo "PLATFORMS=${PLATFORMS}" | tee --append "${GITHUB_OUTPUT:-/dev/null}" -} - -if [[ - "${CUDA_VER}" > "11.2.2" && - "${LINUX_VER}" != "centos7" -]]; then - PLATFORMS+=",linux/arm64" -fi - -write_platforms "${PLATFORMS}" diff --git a/ci/compute-matrix.jq b/ci/compute-matrix.jq new file mode 100644 index 0000000..5a0ea53 --- /dev/null +++ b/ci/compute-matrix.jq @@ -0,0 +1,61 @@ +def compute_arch($x): + ["amd64"] | + if + ["ubuntu18.04", "centos7"] | index($x.LINUX_VER) != null + then + . + else + . + ["arm64"] + end | + $x + {ARCHES: .}; + +def compute_repo($x): + if + env.BUILD_TYPE == "pull-request" + then + "staging" + else + $x.IMAGE_REPO + end; + +def compute_tag_prefix($x): + if + env.BUILD_TYPE == "branch" + then + "" + else + $x.IMAGE_REPO + "-" + env.PR_NUM + "-" + end; + +def compute_image_name($x): + compute_repo($x) as $repo | + compute_tag_prefix($x) as $tag_prefix | + "rapidsai/" + $repo + ":" + $tag_prefix + "cuda" + $x.CUDA_VER + "-" + $x.LINUX_VER + "-" + "py" + $x.PYTHON_VER | + $x + {IMAGE_NAME: .}; + +# Checks the current entry to see if it matches the given exclude +def matches($entry; $exclude): + all($exclude | to_entries | .[]; $entry[.key] == .value); + +# Checks the current entry to see if it matches any of the excludes. +# If so, produce no output. Otherwise, output the entry. +def filter_excludes($entry; $excludes): + select(any($excludes[]; matches($entry; .)) | not); + +def lists2dict($keys; $values): + reduce range($keys | length) as $ind ({}; . + {($keys[$ind]): $values[$ind]}); + +def compute_matrix($input): + ($input.exclude // []) as $excludes | + $input | del(.exclude) | + keys_unsorted as $matrix_keys | + to_entries | + map(.value) | + [ + combinations | + lists2dict($matrix_keys; .) | + filter_excludes(.; $excludes) | + compute_arch(.) | + compute_image_name(.) + ] | + {include: .}; diff --git a/ci/compute-matrix.sh b/ci/compute-matrix.sh index 771292d..03afcba 100755 --- a/ci/compute-matrix.sh +++ b/ci/compute-matrix.sh @@ -1,10 +1,16 @@ #!/bin/bash -# Computes matrix based on "axis.yaml" values. Will also -# remove any keys (e.g. LATEST_VERSIONS) that are not used -# for the matrix build. -# Example Usage: -# ./ci/compute-matrix.sh -set -eu +set -euo pipefail -MATRIX=$(yq -o json '. | del(.LATEST_VERSIONS)' axis.yaml | jq -c) -echo "MATRIX=${MATRIX}" | tee --append ${GITHUB_OUTPUT:-/dev/null} +case "${BUILD_TYPE}" in + pull-request) + export PR_NUM="${GITHUB_REF_NAME##*/}" + ;; + branch) + ;; + *) + echo "Invalid build type: '${BUILD_TYPE}'" + exit 1 + ;; +esac + +yq -o json matrix.yaml | jq -c 'include "ci/compute-matrix"; compute_matrix(.)' diff --git a/ci/create-multiarch-manifest.sh b/ci/create-multiarch-manifest.sh new file mode 100755 index 0000000..c764876 --- /dev/null +++ b/ci/create-multiarch-manifest.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +source_tags=() +tag="${IMAGE_NAME}" +for arch in $(echo "${ARCHES}" | jq .[] -r); do + source_tags+=("${tag}-${arch}") +done + +docker manifest create "${tag}" "${source_tags[@]}" +docker manifest push "${tag}" +if [[ + "${LATEST_UBUNTU_VER}" == "${LINUX_VER}" && + "${LATEST_CUDA_VER}" == "${CUDA_VER}" && + "${LATEST_PYTHON_VER}" == "${PYTHON_VER}" +]]; then + # only create a 'latest' manifest if it is a non-PR workflow. + if [[ "${BUILD_TYPE}" != "pull-request" ]]; then + docker manifest create "rapidsai/${IMAGE_REPO}:latest" "${source_tags[@]}" + docker manifest push "rapidsai/${IMAGE_REPO}:latest" + else + echo "Skipping 'latest' manifest creation for PR workflow." + fi +fi diff --git a/ci/remove-temp-images.sh b/ci/remove-temp-images.sh new file mode 100755 index 0000000..3842d9e --- /dev/null +++ b/ci/remove-temp-images.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +HUB_TOKEN=$( + curl -s -H "Content-Type: application/json" \ + -X POST \ + -d "{\"username\": \"${GPUCIBOT_DOCKERHUB_USER}\", \"password\": \"${GPUCIBOT_DOCKERHUB_TOKEN}\"}" \ + https://hub.docker.com/v2/users/login/ | jq -r .token \ +) +echo "::add-mask::${HUB_TOKEN}" + +full_repo_name=${IMAGE_NAME%%:*} +tag=${IMAGE_NAME##*:} + +for arch in $(echo "$ARCHES" | jq .[] -r); do + curl --fail-with-body -i -X DELETE \ + -H "Accept: application/json" \ + -H "Authorization: JWT $HUB_TOKEN" \ + "https://hub.docker.com/v2/repositories/$full_repo_name/tags/$tag-$arch/" +done + +# Logout +curl -X POST \ + -H "Authorization: JWT $HUB_TOKEN" \ + "https://hub.docker.com/v2/logout/" diff --git a/axis.yaml b/matrix.yaml similarity index 95% rename from axis.yaml rename to matrix.yaml index 0d66246..0a01d4a 100644 --- a/axis.yaml +++ b/matrix.yaml @@ -13,6 +13,8 @@ LINUX_VER: - "ubuntu22.04" - "centos7" - "rockylinux8" +IMAGE_REPO: + - "ci" exclude: - LINUX_VER: "ubuntu22.04" CUDA_VER: "11.2.2"