Forward-merge branch-24.10 into branch-24.12

.devcontainer/rapids.Dockerfile

@@ -47,7 +47,8 @@ ENV PYTHONDONTWRITEBYTECODE="1"
 
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
-ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
+ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
+
 ENV HISTFILE="/home/coder/.cache/._bash_history"
 
 ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"

.github/actions/build-and-test-feature/action.yml

@@ -6,6 +6,7 @@ inputs:
   args: {type: string, required: true}
   gh_token: {type: string, defaut: '', required: false}
   vault_host: {type: string, defaut: '', required: false}
+  aws_role_arn: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
 
@@ -30,5 +31,6 @@ runs:
         VAULT_S3_TTL: "900" # 15 minutes
         gh_token: "${{ inputs.gh_token }}"
         vault_host: "${{ inputs.vault_host }}"
+        aws_role_arn: "${{ inputs.aws_role_arn }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"

.github/workflows/build-and-test-feature.yml

@@ -38,5 +38,6 @@ jobs:
           args: "${{ inputs.args }}"
           gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}"
           vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
+          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"

features/src/rapids-build-utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.10.9",
+  "version": "24.10.12",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp.build.tmpl.sh

@@ -17,9 +17,6 @@ build_${CPP_LIB}_cpp() {
     local -;
     set -euo pipefail;
 
-    # Stop the sccache server in case we need to reload credentials before starting the next build
-    sccache --stop-server >/dev/null 2>&1 || true;
-
     eval "$(\
     PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)} \
     MAX_TOTAL_SYSTEM_MEMORY="${MAX_TOTAL_SYSTEM_MEMORY:-${CPP_MAX_TOTAL_SYSTEM_MEMORY}}" \

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.build.wheel.tmpl.sh

@@ -21,9 +21,6 @@ build_${PY_LIB}_python_wheel() {
     local -;
     set -euo pipefail;
 
-    # Stop the sccache server in case we need to reload credentials before starting the next build
-    sccache --stop-server >/dev/null 2>&1 || true;
-
     eval "$(                                          \
     PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}  \
         rapids-get-num-archs-jobs-and-load "$@"       \

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python.install.tmpl.sh

@@ -21,9 +21,6 @@ install_${PY_LIB}_python() {
     local -;
     set -euo pipefail;
 
-    # Stop the sccache server in case we need to reload credentials before starting the next build
-    sccache --stop-server >/dev/null 2>&1 || true;
-
     eval "$(                                          \
     PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)}  \
         rapids-get-num-archs-jobs-and-load "$@"       \

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh

@@ -51,7 +51,6 @@ clone_${NAME}() {
         --branch "${branch}"                    \
         --ssh-url "${ssh_url}"                  \
         --https-url "${https_url}"              \
-        --recurse-submodules                    \
         -j ${n_jobs:-$(nproc --all --ignore=1)} \
         -c checkout.defaultRemote=upstream      \
         "${OPTS[@]}"                            \

features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml

@@ -58,11 +58,11 @@ repos:
     - name: libkvikio
       sub_dir: python/libkvikio
       depends: [KvikIO]
-      args: {cmake: -DFIND_KVIKIO_CPP=ON, install: *rapids_build_backend_args}
+      args: {install: *rapids_build_backend_args}
     - name: kvikio
       sub_dir: python/kvikio
       depends: [KvikIO]
-      args: {cmake: -DFIND_KVIKIO_CPP=ON, install: *rapids_build_backend_args}
+      args: {install: *rapids_build_backend_args}
 
 - name: cudf
   path: cudf
@@ -71,24 +71,22 @@ repos:
     - name: cudf
       sub_dir: cpp
       depends: [KvikIO]
-      args:
-        cmake: |
-          $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo)
     - name: cudf_kafka
       sub_dir: cpp/libcudf_kafka
       depends: [cudf]
   python:
     - name: libcudf
       sub_dir: python/libcudf
       depends: [cudf]
+      args: {install: *rapids_build_backend_args}
     - name: pylibcudf
       sub_dir: python/pylibcudf
       depends: [cudf]
-      args: {cmake: -DFIND_CUDF_CPP=ON, install: *rapids_build_backend_args}
+      args: {install: *rapids_build_backend_args}
     - name: cudf
       sub_dir: python/cudf
       depends: [cudf]
-      args: {cmake: -DFIND_CUDF_CPP=ON, install: *rapids_build_backend_args}
+      args: {install: *rapids_build_backend_args}
     - name: dask_cudf
       sub_dir: python/dask_cudf
       args: {install: *rapids_build_backend_args}
@@ -217,9 +215,7 @@ repos:
     - name: cugraph_etl
       sub_dir: cpp/libcugraph_etl
       depends: [cudf, cugraph]
-      args:
-        cmake: $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo)
-        install: *rapids_build_backend_args
+      args: {install: *rapids_build_backend_args}
   python:
     - name: pylibcugraph
       sub_dir: python/pylibcugraph
@@ -259,6 +255,7 @@ repos:
     - name: libcuspatial
       sub_dir: python/libcuspatial
       depends: [cuspatial]
+      args: {install: *rapids_build_backend_args}
     - name: cuproj
       sub_dir: python/cuproj
       depends: [rmm]

features/src/utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "24.10.3",
+  "version": "24.10.5",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/utils/install.sh

@@ -67,9 +67,9 @@ chmod u+s "$(realpath -m "$(which cron)")";
 
 # shellcheck disable=SC2174
 mkdir -m 0775 -p /var/log/devcontainer-utils;
-touch /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-chmod 0664 /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-chgrp crontab /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
+touch /var/log/devcontainer-utils/creds-s3.log;
+chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
 
 # Install Devcontainer utility scripts to /opt/devcontainer
 cp -ar ./opt/devcontainer /opt/;
@@ -79,6 +79,15 @@ declare -a commands_and_sources=(
     "parse-args                         parse-args.sh"
     "parse-args-from-docstring          parse-args-from-docstring.sh"
     "bash-completion.tmpl               bash/completion.tmpl.sh"
+    "creds-s3-init                      creds/s3/init.sh"
+    "creds-s3-generate                  creds/s3/generate.sh"
+    "creds-s3-persist                   creds/s3/persist.sh"
+    "creds-s3-propagate                 creds/s3/propagate.sh"
+    "creds-s3-schedule                  creds/s3/schedule.sh"
+    "creds-s3-test                      creds/s3/test.sh"
+    "creds-s3-gh-generate               creds/s3/gh/generate.sh"
+    "creds-s3-vault-generate            creds/s3/vault/generate.sh"
+    "creds-s3-vault-github              creds/s3/vault/github.sh"
     "generate-bash-completion           bash/generate-bash-completion.sh"
     "shell-is-interactive               shell-is-interactive.sh"
     "post-create-command                post-create-command.sh"
@@ -107,10 +116,10 @@ declare -a commands_and_sources=(
 
 # Install alternatives
 for entry in "${commands_and_sources[@]}"; do
-    declare -a pair=(${entry});
+    declare -a pair="(${entry})";
     declare cmd="devcontainer-utils-${pair[0]}";
     declare src="/opt/devcontainer/bin/${pair[1]}";
-    update-alternatives --install /usr/bin/${cmd} ${cmd} ${src} 0;
+    update-alternatives --install "/usr/bin/${cmd}" "${cmd}" "${src}" 0;
 done
 
 declare -a commands="($(for pair in "${commands_and_sources[@]}"; do cut -d' ' -f1 <<< "${pair}"; done))";
@@ -182,6 +191,30 @@ find_non_root_user;
 
 if test -n "${USERNAME-}"; then
     USERHOME="$(bash -c "echo ~${USERNAME-}")";
+
+    if type gh >/dev/null 2>&1; then
+        mkdir -p -m 0755                                         \
+            "$USERHOME/.local"                                   \
+            "$USERHOME/.local/share"                             \
+            "$USERHOME/.local/share/gh"                          \
+            "$USERHOME/.local/share/gh/extensions"               \
+            "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws" \
+            ;
+        NV_GHA_AWS_VERSION=latest
+        find_version_from_git_tags NV_GHA_AWS_VERSION https://github.com/nv-gha-runners/gh-nv-gha-aws;
+        wget --no-hsts -q -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \
+            "https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')";
+        chmod 0755 "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws";
+        cat <<EOF >"$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/manifest.yml"
+owner: nv-gha-runners
+name: gh-nv-gha-aws
+host: github.com
+tag: v${NV_GHA_AWS_VERSION}
+ispinned: false
+path: $USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws
+EOF
+    fi
+
     # Add user to the crontab group
     usermod -aG crontab "${USERNAME}";
     # Allow user to edit the crontab

features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+_creds_s3_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-generate';
+
+    if test -z "${SCCACHE_BUCKET:-}"; then
+        exit 1;
+    fi
+
+    if test -n "${AWS_ROLE_ARN:-}" && gh nv-gha-aws --help >/dev/null 2>&1; then
+        # shellcheck disable=SC1091
+        devcontainer-utils-creds-s3-gh-generate;
+    elif test -n "${VAULT_HOST}"; then
+        # shellcheck disable=SC1091
+        devcontainer-utils-creds-s3-vault-generate;
+    fi
+}
+
+_creds_s3_generate "$@" <&0;

features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh

@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+
+_creds_github_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate';
+
+    if test -z "${AWS_ROLE_ARN:-}" \
+    || test -z "${SCCACHE_BUCKET:-}" \
+    || ! gh nv-gha-aws --help >/dev/null 2>&1; then
+        exit 1;
+    fi
+
+    # Remove existing credentials in case vault declines to issue new ones.
+    rm -rf ~/.aws/{stamp,config,credentials};
+
+    SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
+
+    devcontainer-utils-creds-s3-persist - <<< \
+        --bucket="${SCCACHE_BUCKET:-}"        \
+        --region="${SCCACHE_REGION:-}"        ;
+
+    # Initialize the GitHub CLI with the appropriate user scopes
+    # shellcheck disable=SC1091
+    . devcontainer-utils-init-github-cli;
+
+    # Check whether the user is in one of the allowed GitHub orgs
+    local allowed_orgs="${AWS_GITHUB_ORGS:-${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}}";
+    allowed_orgs="${allowed_orgs// /|}";
+    allowed_orgs="${allowed_orgs//;/|}";
+    allowed_orgs="${allowed_orgs//,/|}";
+
+    local -ra user_orgs="($(                         \
+        gh api user/orgs --jq '.[].login'            \
+            -H "Accept: application/vnd.github+json" \
+        | grep --color=never -iE "(${allowed_orgs})" \
+    ))";
+
+    if test "${#user_orgs[@]}" -eq 0; then
+        exit 1;
+    fi
+
+    local org;
+    local generated_at;
+    local -a nv_gha_aws_args=(
+        --profile default
+        --output creds-file
+        --role-arn "${AWS_ROLE_ARN}"
+        --aud "${AWS_AUDIENCE:-sts.amazonaws.com}"
+        --duration "${AWS_S3_TTL:-${VAULT_S3_TTL:-43200}}"
+        --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}"
+    );
+
+    for org in "${user_orgs[@]}"; do
+        generated_at="$(date '+%s')";
+        if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then
+            if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then
+                echo "${generated_at}" > ~/.aws/stamp;
+                return 0;
+            fi
+        fi
+    done
+
+}
+
+_creds_github_generate "$@" <&0;

features/src/utils/opt/devcontainer/bin/creds/s3/init.sh

@@ -0,0 +1,58 @@
+#! /usr/bin/env bash
+
+_s3_cred() {
+    sed -n "s/$1=//p" ~/.aws/credentials 2>/dev/null;
+}
+
+_s3_creds_init() {
+    local -
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-init';
+
+    if type sccache >/dev/null 2>&1; then
+        if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then
+            if grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}"     \
+            && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then
+                if ! grep -qE "^$" <<< "${VAULT_HOST:-${AWS_ROLE_ARN:-}}"; then
+                    # Generate S3 creds if they don't exist (or are expired)
+                    if devcontainer-utils-creds-s3-test \
+                    || devcontainer-utils-creds-s3-generate; then
+                        # Persist creds in ~/.aws dir
+                        devcontainer-utils-creds-s3-persist - <<< "                     \
+                            --bucket '${SCCACHE_BUCKET:-}'                              \
+                            --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}'       \
+                            --aws-access-key-id '$(_s3_cred aws_access_key_id)'         \
+                            --aws-session-token '$(_s3_cred aws_session_token)'         \
+                            --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
+                        ";
+                        # Install a crontab to refresh the credentials
+                        devcontainer-utils-creds-s3-schedule;
+                    else
+                        devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
+                    fi
+                elif devcontainer-utils-creds-s3-test; then
+                    devcontainer-utils-creds-s3-persist - <<< "                     \
+                        --bucket '${SCCACHE_BUCKET:-}'                              \
+                        --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}'       \
+                        --aws-access-key-id '$(_s3_cred aws_access_key_id)'         \
+                        --aws-session-token '$(_s3_cred aws_session_token)'         \
+                        --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
+                    ";
+                else
+                    # bucket is inaccessible
+                    devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
+                fi
+            elif ! devcontainer-utils-creds-s3-propagate; then
+                # bucket is inaccessible
+                devcontainer-utils-creds-s3-persist <<< "--no-bucket --no-region";
+            fi
+        fi
+    fi
+}
+
+_s3_creds_init "$@";
+
+# shellcheck disable=SC1090
+. /etc/profile.d/*-devcontainer-utils.sh;