Skip to content

Commit

Permalink
Merge branch 'branch-24.10' into fix-repo-clone-submodules
Browse files Browse the repository at this point in the history
  • Loading branch information
trxcllnt authored Sep 19, 2024
2 parents ce91628 + 3952e76 commit db40a78
Show file tree
Hide file tree
Showing 42 changed files with 203 additions and 165 deletions.
4 changes: 2 additions & 2 deletions .devcontainer/cuda11.8-conda/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "11.8",
"PYTHON_PACKAGE_MANAGER": "conda",
"BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-mambaforge-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-mambaforge-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda11.8-conda"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-conda"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda11.8-pip/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "11.8",
"PYTHON_PACKAGE_MANAGER": "pip",
"BASE": "rapidsai/devcontainers:24.08-cpp-cuda11.8-ucx1.15.0-openmpi-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-cuda11.8-ucx1.17.0-openmpi-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda11.8-pip"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda11.8-pip"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda12.0-conda/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "12.0",
"PYTHON_PACKAGE_MANAGER": "conda",
"BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.0-conda"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.0-conda"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda12.0-pip/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "12.0",
"PYTHON_PACKAGE_MANAGER": "pip",
"BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.0-ucx1.15.0-openmpi-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.0-ucx1.17.0-openmpi-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.0-pip"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.0-pip"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda12.2-conda/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "12.2",
"PYTHON_PACKAGE_MANAGER": "conda",
"BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.2-conda"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.2-conda"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda12.2-pip/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "12.2",
"PYTHON_PACKAGE_MANAGER": "pip",
"BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.2-ucx1.15.0-openmpi-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.2-ucx1.17.0-openmpi-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.2-pip"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.2-pip"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda12.5-conda/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "12.5",
"PYTHON_PACKAGE_MANAGER": "conda",
"BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-mambaforge-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-conda"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-conda"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
4 changes: 2 additions & 2 deletions .devcontainer/cuda12.5-pip/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
"args": {
"CUDA": "12.5",
"PYTHON_PACKAGE_MANAGER": "pip",
"BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ucx1.15.0-openmpi-ubuntu22.04"
"BASE": "rapidsai/devcontainers:24.10-cpp-cuda12.5-ucx1.17.0-openmpi-ubuntu22.04"
}
},
"runArgs": [
"--rm",
"--name",
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-pip"
"${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.10-cuda12.5-pip"
],
"hostRequirements": {"gpu": "optional"},
"features": {
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build-all-rapids-repos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
if: needs.check-event.outputs.ok == 'true'
needs: check-event
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08
uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10
permissions:
actions: read
packages: read
Expand Down
4 changes: 2 additions & 2 deletions USAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ The features that comprise the image are noted in the image tags. If no version

The pre-built images can be used as the `"image"`, or as the base of a Dockerfile in `"build"`, in `devcontainer.json`:

<details><summary>devcontainer.json using pre-built image</summary><pre>{<br/> "image": "rapidsai/devcontainers:24.08-cpp-llvm16-cuda12.0-nvhpc23.5-ubuntu22.04",<br/> "hostRequirements": { "gpu": true },<br/> "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",<br/> "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"<br/>}</pre></details>
<details><summary>devcontainer.json using pre-built image</summary><pre>{<br/> "image": "rapidsai/devcontainers:24.10-cpp-llvm16-cuda12.0-nvhpc23.5-ubuntu22.04",<br/> "hostRequirements": { "gpu": true },<br/> "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",<br/> "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"<br/>}</pre></details>

You can also build a custom devcontainer by composing individual features:

<details><summary>devcontainer.json using individual features</summary><pre>{<br/> "image": "ubuntu:22.04",<br/> "features": {<br/> "ghcr.io/rapidsai/devcontainers/features/cmake:24.8": {},<br/> "ghcr.io/rapidsai/devcontainers/features/ninja:24.8": {},<br/> "ghcr.io/rapidsai/devcontainers/features/sccache:24.8": {<br/> "version": "0.5.4"<br/> }<br/> },<br/> "overrideFeatureInstallOrder": [<br/> "ghcr.io/rapidsai/devcontainers/features/cmake",<br/> "ghcr.io/rapidsai/devcontainers/features/ninja",<br/> "ghcr.io/rapidsai/devcontainers/features/sccache"<br/> ],<br/> "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",<br/> "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"<br/>}</pre></details>
<details><summary>devcontainer.json using individual features</summary><pre>{<br/> "image": "ubuntu:22.04",<br/> "features": {<br/> "ghcr.io/rapidsai/devcontainers/features/cmake:24.10": {},<br/> "ghcr.io/rapidsai/devcontainers/features/ninja:24.10": {},<br/> "ghcr.io/rapidsai/devcontainers/features/sccache:24.10": {<br/> "version": "0.5.4"<br/> }<br/> },<br/> "overrideFeatureInstallOrder": [<br/> "ghcr.io/rapidsai/devcontainers/features/cmake",<br/> "ghcr.io/rapidsai/devcontainers/features/ninja",<br/> "ghcr.io/rapidsai/devcontainers/features/sccache"<br/> ],<br/> "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",<br/> "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind"<br/>}</pre></details>

> **NOTE:** Feature updates published since your most recent image build will invalidate your docker image layer cache, meaning it can take the [devcontainers CLI](https://github.com/devcontainers/cli) longer to initialize containers composed from individual features.
Expand Down
4 changes: 2 additions & 2 deletions ci/release/update-version.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}')
NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
NEXT_FULL_TAG=${NEXT_MAJOR}.${NEXT_MINOR}.${NEXT_PATCH}
NEXT_UCX_PY_VERSION="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG})"
NEXT_UCXX_SHORT_TAG="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG})"

# Need to distutils-normalize the versions for some use cases
CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}'))")
Expand All @@ -46,7 +46,7 @@ for file in $(find .devcontainer -name devcontainer.json); do
done

sed_runner "s/branch-[[:digit:]]\{2\}\.[[:digit:]]\+/branch-${NEXT_SHORT_TAG}/g" ./features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml
sed_runner "s/branch-0.[[:digit:]]\+/branch-${NEXT_UCX_PY_VERSION}/g" ./features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml
sed_runner "s/branch-0.[[:digit:]]\+/branch-${NEXT_UCXX_SHORT_TAG}/g" ./features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml

for file in $(find features -name devcontainer-feature.json); do
tmp=$(mktemp)
Expand Down
2 changes: 1 addition & 1 deletion features/src/cccl-dev/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "NVIDIA CCCL development utilities",
"id": "cccl-dev",
"version": "24.8.1",
"version": "24.10.0",
"description": "A feature to install NVIDIA CCCL development utilities",
"options": {
"litVersion": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/cmake/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "CMake",
"id": "cmake",
"version": "24.8.1",
"version": "24.10.0",
"description": "A feature to install CMake",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/cuda/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "CUDA Toolkit",
"id": "cuda",
"version": "24.8.2",
"version": "24.10.0",
"description": "A feature to install the NVIDIA CUDA Toolkit",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/gcc/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "GCC",
"id": "gcc",
"version": "24.8.1",
"version": "24.10.0",
"description": "A feature to install gcc",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/gitlab-cli/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"id": "gitlab-cli",
"version": "24.8.1",
"version": "24.10.0",
"name": "GitLab CLI",
"documentationURL": "https://github.com/rapidsai/devcontainers/features/tree/main/src/gitlab-cli",
"description": "Installs the GitLab CLI. Auto-detects latest version and installs needed dependencies.",
Expand Down
2 changes: 1 addition & 1 deletion features/src/llvm/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "LLVM compilers and tools",
"id": "llvm",
"version": "24.8.2",
"version": "24.10.1",
"description": "A feature to install LLVM compilers and tools",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/mambaforge/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "Mambaforge",
"id": "mambaforge",
"version": "24.8.1",
"version": "24.10.0",
"description": "A feature to install mambaforge",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/ninja/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "Ninja build",
"id": "ninja",
"version": "24.8.2",
"version": "24.10.0",
"description": "A feature to install ninja-build",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/nvhpc/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "NVHPC SDK",
"id": "nvhpc",
"version": "24.8.2",
"version": "24.10.0",
"description": "A feature to install the NVHPC SDK",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/oneapi/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "Intel oneapi toolchain",
"id": "oneapi",
"version": "24.8.1",
"version": "24.10.0",
"description": "A feature to install the Intel oneapi toolchain",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/openmpi/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "OpenMPI",
"id": "openmpi",
"version": "24.8.1",
"version": "24.10.0",
"description": "A feature to install OpenMPI with optional CUDA and UCX support",
"options": {
"version": {
Expand Down
2 changes: 1 addition & 1 deletion features/src/rapids-build-utils/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "NVIDIA RAPIDS devcontainer build utilities",
"id": "rapids-build-utils",
"version": "24.8.16",
"version": "24.10.10",
"description": "A feature to install the RAPIDS devcontainer build utilities",
"containerEnv": {
"BASH_ENV": "/etc/bash.bash_env"
Expand Down
19 changes: 14 additions & 5 deletions features/src/rapids-build-utils/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,22 @@ fi

source /etc/lsb-release;

if [[ ! "23.04" > "${DISTRIB_RELEASE}" ]]; then
BREAK_PACKAGES="--break-system-packages";
declare -a _PIP_INSTALL_ARGS=()
declare -a _PIP_UPGRADE_ARGS=(--upgrade)

if [[ "${DISTRIB_RELEASE}" > "22.04" ]]; then
_PIP_INSTALL_ARGS+=(--break-system-packages);
if [[ "${DISTRIB_RELEASE}" > "23.04" ]]; then
_PIP_UPGRADE_ARGS+=(--ignore-installed);
fi
fi

# Install the rapids dependency file generator and conda-merge
/usr/bin/python3 -m pip install $BREAK_PACKAGES -U pip;
/usr/bin/python3 -m pip install $BREAK_PACKAGES 'rapids-dependency-file-generator<1.14' conda-merge toml;
/usr/bin/python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" "${_PIP_UPGRADE_ARGS[@]}" pip;
# Install RAPIDS dependency file generator, conda-merge, and toml
/usr/bin/python3 -m pip install "${_PIP_INSTALL_ARGS[@]}" \
'rapids-dependency-file-generator<1.14' \
conda-merge \
toml;

# Install RAPIDS build utility scripts to /opt/

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ make_pip_dependencies() {
cuda="${cuda_version}"
cuda_suffixed=true
py="${python_version}"
use_cuda_wheels=false
);

# add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,16 @@ python_conda_pkg_names() {

if test -d ~/"${!repo_path:-}/.git"; then
# the regex will continue until morale improves
find ~/"${!repo_path}/" \
-type f -name 'meta.yaml' \
! -path '*/.conda/*' \
! -path '*/build/*' \
! -path '*/_skbuild/*' \
-exec grep -E 'name: ?' {} \; \
2>/dev/null \
| tr -d '[:blank:]' \
| cut -d':' -f2 \
find ~/"${!repo_path}/" \
-type f -name 'meta.yaml' \
! -path '*/.conda/*' \
! -path '*/build/*' \
! -path '*/_skbuild/*' \
! -path '*/target/cmake-build/*' \
-exec grep -E 'name: ?' {} \; \
2>/dev/null \
| tr -d '[:blank:]' \
| cut -d':' -f2 \
;
fi
done
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ build_${CPP_LIB}_cpp() {
local -;
set -euo pipefail;

# Stop the sccache server in case we need to reload credentials before starting the next build
sccache --stop-server >/dev/null 2>&1 || true;

eval "$(\
PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)} \
MAX_TOTAL_SYSTEM_MEMORY="${MAX_TOTAL_SYSTEM_MEMORY:-${CPP_MAX_TOTAL_SYSTEM_MEMORY}}" \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ build_${PY_LIB}_python_wheel() {
local -;
set -euo pipefail;

# Stop the sccache server in case we need to reload credentials before starting the next build
sccache --stop-server >/dev/null 2>&1 || true;

eval "$( \
PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)} \
rapids-get-num-archs-jobs-and-load "$@" \
Expand Down Expand Up @@ -96,9 +99,9 @@ build_${PY_LIB}_python_wheel() {
PARALLEL_LEVEL="${n_jobs}" \
CMAKE_ARGS="${cmake_args[*]@Q}" \
SKBUILD_BUILD_OPTIONS="${ninja_args[*]}" \
SKBUILD_BUILD_VERBOSE="${v:+True}" \
SKBUILD_LOGGING_LEVEL="${v:+INFO}" \
SKBUILD_INSTALL_STRIP="${strip:+True}" \
SKBUILD_CMAKE_VERBOSE="${v:+True}" \
SKBUILD_CMAKE_BUILD_TYPE="${build_type}" \
CMAKE_BUILD_PARALLEL_LEVEL="${n_jobs}" \
NVCC_APPEND_FLAGS="${nvcc_append_flags}" \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ install_${PY_LIB}_python() {
local -;
set -euo pipefail;

# Stop the sccache server in case we need to reload credentials before starting the next build
sccache --stop-server >/dev/null 2>&1 || true;

eval "$( \
PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc --all)} \
rapids-get-num-archs-jobs-and-load "$@" \
Expand Down Expand Up @@ -114,9 +117,9 @@ install_${PY_LIB}_python() {
PARALLEL_LEVEL="${n_jobs}" \
CMAKE_ARGS="${cmake_args[*]@Q}" \
SKBUILD_BUILD_OPTIONS="${ninja_args[*]}" \
SKBUILD_BUILD_VERBOSE="${v:+True}" \
SKBUILD_LOGGING_LEVEL="${v:+INFO}" \
SKBUILD_INSTALL_STRIP="${strip:+True}" \
SKBUILD_CMAKE_VERBOSE="${v:+True}" \
SKBUILD_CMAKE_BUILD_TYPE="${build_type}" \
CMAKE_BUILD_PARALLEL_LEVEL="${n_jobs}" \
NVCC_APPEND_FLAGS="${nvcc_append_flags}" \
Expand Down
Loading

0 comments on commit db40a78

Please sign in to comment.