diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 index 6b3dc8b..698b7a4 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml.j2 @@ -53,7 +53,7 @@ runs: ls -lahR ${{ env.ARTIFACTS_DIR }} <% endif %> - - if: github.repository_owner == 'nv-legate' + - if: github.repository_owner == 'nvidia' name: Get AWS credentials for sccache bucket uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/actions/download-artifacts/action.yml b/.github/actions/download-artifacts/action.yml index f6bd219..3dcaeec 100644 --- a/.github/actions/download-artifacts/action.yml +++ b/.github/actions/download-artifacts/action.yml @@ -36,7 +36,7 @@ runs: name: Cache conda artifacts uses: actions/cache@v4 with: - key: "nv-legate/${{ inputs.artifact-repo }}@${{ inputs.platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}" + key: "nvidia/{ inputs.artifact-repo }}@${{ inputs.platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}" path: ${{ inputs.dest-dir }} - if: steps.cache.outputs.cache-hit != 'true' @@ -45,7 +45,7 @@ runs: with: github_token: ${{ inputs.inter-repos-ro-access-token }} path: ${{ inputs.dest-dir }} - repo: nv-legate/${{ inputs.artifact-repo }} + repo: nvidia/${{ inputs.artifact-repo }} check_artifacts: true commit: ${{ inputs.git_sha }} workflow_conclusion: "" diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 00a17f3..cad554b 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -31,7 +31,14 @@ runs: pwd ls -lahR $ARTIFACTS_DIR + - name: Make test env + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/make-conda-env" test + - name: Run test / analysis shell: bash --noprofile --norc -xeuo pipefail {0} run: | + . conda-utils + activate_conda_env "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/test" ${{ inputs.test-options }} diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index 8759872..740ffe9 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -32,3 +32,93 @@ jobs: build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} secrets: inherit + + setup-test: + name: Setup test + if: inputs.upload-enabled == false + needs: + - build + runs-on: linux-amd64-cpu4 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + set -xeuo pipefail + MATRIX_JSON='{"include": [' + RUNNERS=('linux-amd64-gpu-v100-latest-1:gpu:linux') + TEST_CONFIGS=('Unit Tests:unit' 'Samples:samples') + + for RUNNER in "${RUNNERS[@]}"; do + IFS=':' read -ra RUNNER_INFO <<< "$RUNNER" + RUNNER_NAME=${RUNNER_INFO[0]} + TARGET_DEVICE=${RUNNER_INFO[1]} + RUNNER_PLATFORM=${RUNNER_INFO[2]} + + if [[ "$TARGET_DEVICE" == "${{ inputs.target-device }}" && "$RUNNER_PLATFORM" == "${{ inputs.platform }}" ]]; then + for TEST_CONFIG in "${TEST_CONFIGS[@]}"; do + IFS=':' read -ra CONFIG_INFO <<< "$TEST_CONFIG" + CONFIG_NAME=${CONFIG_INFO[0]} + CONFIG_SCOPE=${CONFIG_INFO[1]} + + MATRIX_JSON+="{\"runner\": {\"name\": \"$RUNNER_NAME\", \"type\": \"$TARGET_DEVICE\", \"platform\": \"$RUNNER_PLATFORM\"}, \"test-config\": {\"name\": \"$CONFIG_NAME\", \"test-scope\": \"$CONFIG_SCOPE\"}}," + done + fi + done + + MATRIX_JSON=$(echo "$MATRIX_JSON" | sed 's/,$//') # Remove the trailing comma + MATRIX_JSON+=']}' + echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + + test-within-container: + if: github.repository_owner == 'nvidia' && (inputs.platform == 'linux-x64' || inputs.platform == 'linux-aarch64') + needs: + - setup-test + + name: ${{ matrix.test-config.name }} (${{ inputs.platform }}, ${{ inputs.target-device }}, ${{ inputs.build-mode }}) + + strategy: + fail-fast: false + matrix: ${{fromJson(needs.setup-test.outputs.matrix)}} + + uses: + ./.github/workflows/gh-test-within-container.yml + with: + client-repo: ${{ github.event.repository.name }} + build-type: ${{ inputs.build-type }} + name: ${{ matrix.test-config.name }} + target-device: ${{ inputs.target-device }} + runs-on: ${{ matrix.runner.name }} + has-gpu: ${{ matrix.runner.type == 'gpu' }} + test-options: ${{ matrix.test-config.test-scope }} + platform: ${{ matrix.runner.platform }} + build-mode: ${{ inputs.build-mode }} + upload-enabled: ${{ inputs.upload-enabled }} + secrets: inherit + + + test-without-container: + if: github.repository_owner == 'nv-legate' && (inputs.platform != 'linux-x64' && inputs.platform != '-aarch64') + needs: + - setup-test + + name: ${{ matrix.test-config.name }} (${{ inputs.platform }}, ${{ inputs.target-device }}, ${{ inputs.build-mode }}) + + strategy: + fail-fast: false + matrix: ${{fromJson(needs.setup-test.outputs.matrix)}} + + uses: + ./.github/workflows/gh-test-without-container.yml + with: + client-repo: ${{ github.event.repository.name }} + build-type: ${{ inputs.build-type }} + name: ${{ matrix.test-config.name }} + target-device: ${{ inputs.target-device }} + runs-on: ${{ matrix.runner.name }} + has-gpu: ${{ matrix.runner.type == 'gpu' }} + test-options: ${{ matrix.test-config.test-scope }} + platform: ${{ matrix.runner.platform }} + build-mode: ${{ inputs.build-mode }} + upload-enabled: ${{ inputs.upload-enabled }} + secrets: inherit diff --git a/.github/workflows/gh-test-within-container.yml b/.github/workflows/gh-test-within-container.yml index efb928c..3242cb1 100644 --- a/.github/workflows/gh-test-within-container.yml +++ b/.github/workflows/gh-test-within-container.yml @@ -41,7 +41,7 @@ on: jobs: test: name: ${{ inputs.name }} - if: github.repository_owner == 'nv-legate' + if: github.repository_owner == 'nvidia runs-on: ${{ inputs.runs-on }} container: diff --git a/.github/workflows/gh-test-without-container.yml b/.github/workflows/gh-test-without-container.yml index 9f72e2d..edcb4b4 100644 --- a/.github/workflows/gh-test-without-container.yml +++ b/.github/workflows/gh-test-without-container.yml @@ -41,7 +41,7 @@ on: jobs: test: name: ${{ inputs.name }} - if: github.repository_owner == 'nv-legate' + if: github.repository_owner == 'nvidia' runs-on: ${{ inputs.runs-on }} defaults: diff --git a/.github/workflows/gh-upload.yml b/.github/workflows/gh-upload.yml index 7e71b0c..1c06e14 100644 --- a/.github/workflows/gh-upload.yml +++ b/.github/workflows/gh-upload.yml @@ -40,7 +40,7 @@ on: jobs: upload: name: ${{ inputs.name }} - if: github.repository_owner == 'nv-legate' + if: github.repository_owner == 'nvidia' runs-on: linux-amd64-gpu-v100-earliest-1 container: diff --git a/continuous_integration/environment.yml b/continuous_integration/build-environment.yml similarity index 100% rename from continuous_integration/environment.yml rename to continuous_integration/build-environment.yml diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env index 5d8e6f4..1737c61 100755 --- a/continuous_integration/scripts/make-conda-env +++ b/continuous_integration/scripts/make-conda-env @@ -3,7 +3,18 @@ set -x make_ci_env() { - mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" + mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/build-environment.yml" +} + +make_test_env() { + . conda-utils + + mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/test-environment.yml" + + activate_conda_env + + pip install "${ARTIFACTS_DIR}/*.whl" + } make_conda_env() { @@ -14,6 +25,7 @@ make_conda_env() { case "$1" in ci) make_ci_env;; + test) make_test_env;; *) return 1;; esac diff --git a/continuous_integration/scripts/test b/continuous_integration/scripts/test new file mode 100755 index 0000000..44d36a7 --- /dev/null +++ b/continuous_integration/scripts/test @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +run_test_or_analysis() { + set -x + + conda_info; + + set -xeuo pipefail + + case "$1" in + "unit") + echo "Executing unit tests..." + python -m pytest + ;; + "samples") + echo "Running samples..." + python -m pytest examples + ;; + + *) + echo "Invalid command: $1" + return 1 + ;; + esac + + return 0 +} + +(run_test_or_analysis "$@"); diff --git a/continuous_integration/scripts/vault-s3-init b/continuous_integration/scripts/vault-s3-init index 1209bd6..0823227 100755 --- a/continuous_integration/scripts/vault-s3-init +++ b/continuous_integration/scripts/vault-s3-init @@ -39,7 +39,7 @@ vault_s3_init() { echo "" local vault_token=null; - local user_orgs=nv-legate; + local user_orgs=nvidia; # Attempt to authenticate with GitHub eval "$(get_vault_token "${VAULT_HOST}" ${user_orgs} "$GH_TOKEN")"; diff --git a/continuous_integration/test-environment.yml b/continuous_integration/test-environment.yml new file mode 100644 index 0000000..1f4b879 --- /dev/null +++ b/continuous_integration/test-environment.yml @@ -0,0 +1,9 @@ +name: cupy +channels: + - defaults +dependencies: + - python=3.11 + - pytest>=6.2.4 + - pip + - pip: + - pytest-benchmark>=3.4.1